- * 1) Changed the method signature to adopt the gremlin-server 3.5.1. + * 1) Changed the method signature to adopt the gremlin-server Context API. *
* public Optional> selectOther(RequestMessage requestMessage) * --> diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/Condition.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/Condition.java index e77f2f0b1e..d956a8ace7 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/Condition.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/query/Condition.java @@ -25,7 +25,6 @@ import java.util.Objects; import java.util.Set; import java.util.function.BiFunction; -import java.util.function.BiPredicate; import org.apache.commons.lang.ArrayUtils; import org.apache.hugegraph.backend.id.Id; @@ -37,6 +36,7 @@ import org.apache.hugegraph.util.DateUtil; import org.apache.hugegraph.util.E; import org.apache.hugegraph.util.NumericUtil; +import org.apache.tinkerpop.gremlin.process.traversal.PBiPredicate; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; @@ -51,7 +51,7 @@ public enum ConditionType { NOT } - public enum RelationType implements BiPredicate { + public enum RelationType implements PBiPredicate { EQ("==", RelationType::equals), @@ -165,6 +165,11 @@ public String string() { return this.operator; } + @Override + public String getPredicateName() { + return this.operator; + } + /** * Determine two values of any type equal * diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/SerializerFactory.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/SerializerFactory.java index db5fe0a8cd..213cf2b264 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/SerializerFactory.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/serializer/SerializerFactory.java @@ -72,13 +72,11 @@ public static void register(String name, String classPath) { "AbstractSerializer: '%s'", classPath); } - // Check exists - if (serializers.containsKey(name)) { + // Register atomically: identical re-registration is a no-op + Class registered = serializers.putIfAbsent(name, (Class) clazz); + if (registered != null && !registered.equals(clazz)) { throw new BackendException("Exists serializer: %s(Class '%s')", - name, serializers.get(name).getName()); + name, registered.getName()); } - - // Register class - serializers.put(name, (Class) clazz); } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendProviderFactory.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendProviderFactory.java index ac1e0018ce..2612ea3a07 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendProviderFactory.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/store/BackendProviderFactory.java @@ -114,12 +114,10 @@ public static void register(String name, String classPath) { BackendException.check(subclass, "Class '%s' is not a subclass of " + "class BackendStoreProvider", classPath); - // Check exists - BackendException.check(!providers.containsKey(name), + // Register atomically: identical re-registration is a no-op + Class registered = providers.putIfAbsent(name, (Class) clazz); + BackendException.check(registered == null || registered.equals(clazz), "Exists BackendStoreProvider: %s (%s)", - name, providers.get(name)); - - // Register class - providers.put(name, (Class) clazz); + name, registered); } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java index 0c962b11a2..95184fe67d 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/tx/GraphTransaction.java @@ -551,24 +551,26 @@ public Number queryNumber(Query query) { boolean hasUpdate = this.hasUpdate(); Aggregate aggregate = query.aggregateNotNull(); - // TODO: we can concat index-query results and tx uncommitted records. if (hasUpdate) { - E.checkArgument(!isConditionQuery, - "It's not allowed to query by index when " + - "there are uncommitted records."); + E.checkArgument(aggregate.func() == AggregateFunc.COUNT, + "The %s operator with uncommitted records " + + "is not supported", + aggregate.func().string()); + Query queryWithoutAggregate = query.copy(); + queryWithoutAggregate.aggregate(null); + Iterator results = queryWithoutAggregate.resultType().isVertex() ? + this.queryVertices(queryWithoutAggregate) : + this.queryEdges(queryWithoutAggregate); + return countAndClose(results); } QueryList queries = this.optimizeQueries(query, q -> { boolean isIndexQuery = q instanceof IdQuery; assert isIndexQuery || isConditionQuery || q == query; - // Need to fall back if there are uncommitted records - boolean fallback = hasUpdate; + boolean fallback = false; Number result; - if (fallback) { - // Here just ignore it, and do fall back later - result = null; - } else if (!isIndexQuery || !isConditionQuery) { + if (!isIndexQuery || !isConditionQuery) { // It's a sysprop-query, let parent tx do it assert !fallback; result = super.queryNumber(q); @@ -608,6 +610,19 @@ public Number queryNumber(Query query) { return aggregate.reduce(results.iterator()); } + private static long countAndClose(Iterator results) { + try { + long count = 0L; + while (results.hasNext()) { + results.next(); + count++; + } + return count; + } finally { + CloseableIterator.closeIterator(results); + } + } + @Watched(prefix = "graph") public HugeVertex addVertex(Object... keyValues) { return this.addVertex(this.constructVertex(true, keyValues)); @@ -834,7 +849,7 @@ public Iterator queryVertices() { public Iterator queryVertices(Query query) { if (this.hasUpdate()) { E.checkArgument(query.noLimitAndOffset(), - "It's not allowed to query with offser/limit " + + "It's not allowed to query with offset/limit " + "when there are uncommitted records."); // TODO: also add check: no SCAN, no OLAP E.checkArgument(!query.paging(), @@ -1000,7 +1015,7 @@ public Iterator queryEdges() { public Iterator queryEdges(Query query) { if (this.hasUpdate()) { E.checkArgument(query.noLimitAndOffset(), - "It's not allowed to query with offser/limit " + + "It's not allowed to query with offset/limit " + "when there are uncommitted records."); // TODO: also add check: no SCAN, no OLAP E.checkArgument(!query.paging(), diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/io/HugeGraphSONModule.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/io/HugeGraphSONModule.java index ddb7c1a981..f584046463 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/io/HugeGraphSONModule.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/io/HugeGraphSONModule.java @@ -57,7 +57,6 @@ import org.apache.hugegraph.util.SafeDateUtil; import org.apache.tinkerpop.gremlin.process.traversal.Path; import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree; -import org.apache.tinkerpop.gremlin.structure.Element; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONIo; import org.apache.tinkerpop.gremlin.structure.io.graphson.GraphSONTokens; import org.apache.tinkerpop.gremlin.structure.io.graphson.TinkerPopJacksonModule; @@ -103,12 +102,14 @@ public class HugeGraphSONModule extends TinkerPopJacksonModule { TYPE_DEFINITIONS = new ConcurrentHashMap<>(); TYPE_DEFINITIONS.put(Optional.class, "Optional"); + TYPE_DEFINITIONS.put(File.class, "File"); TYPE_DEFINITIONS.put(Date.class, "Date"); TYPE_DEFINITIONS.put(UUID.class, "UUID"); // HugeGraph id serializer TYPE_DEFINITIONS.put(StringId.class, "StringId"); TYPE_DEFINITIONS.put(LongId.class, "LongId"); + TYPE_DEFINITIONS.put(UuidId.class, "UuidId"); TYPE_DEFINITIONS.put(EdgeId.class, "EdgeId"); // HugeGraph schema serializer @@ -171,6 +172,7 @@ public static void registerCommonSerializers(SimpleModule module) { module.addSerializer(Shard.class, new ShardSerializer()); module.addSerializer(File.class, new FileSerializer()); + module.addDeserializer(File.class, new FileDeserializer()); boolean useTimestamp = false; module.addSerializer(Date.class, @@ -222,7 +224,9 @@ public static void registerGraphSerializers(SimpleModule module) { */ module.addSerializer(HugeVertex.class, new HugeVertexSerializer()); module.addSerializer(HugeEdge.class, new HugeEdgeSerializer()); + } + public static void registerTraversalSerializers(SimpleModule module) { module.addSerializer(Path.class, new PathSerializer()); module.addSerializer(Tree.class, new TreeSerializer()); } @@ -641,8 +645,8 @@ public T deserialize(JsonParser jsonParser, String idValue = ctxt.readValue(jsonParser, String.class); return (T) IdGenerator.of(idValue); } else if (clazz.equals(UuidId.class)) { - UUID idValue = ctxt.readValue(jsonParser, UUID.class); - return (T) IdGenerator.of(idValue); + String idValue = ctxt.readValue(jsonParser, String.class); + return (T) IdGenerator.of(UUID.fromString(idValue)); } else { assert clazz.equals(EdgeId.class); String idValue = ctxt.readValue(jsonParser, String.class); @@ -883,9 +887,8 @@ public TreeSerializer() { public void serialize(Tree tree, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException { jsonGenerator.writeStartArray(); - @SuppressWarnings("unchecked") - Set> set = tree.entrySet(); - for (Map.Entry entry : set) { + for (Object item : tree.entrySet()) { + Map.Entry entry = (Map.Entry) item; jsonGenerator.writeStartObject(); jsonGenerator.writeObjectField(GraphSONTokens.KEY, entry.getKey()); @@ -924,9 +927,65 @@ public FileSerializer() { public void serialize(File file, JsonGenerator jsonGenerator, SerializerProvider provider) throws IOException { jsonGenerator.writeStartObject(); - jsonGenerator.writeStringField("file", file.getName()); + this.writeFields(file, jsonGenerator); jsonGenerator.writeEndObject(); } + + @Override + public void serializeWithType(File file, + JsonGenerator jsonGenerator, + SerializerProvider provider, + TypeSerializer typeSer) + throws IOException { + WritableTypeId typeId = typeSer.typeId( + file, JsonToken.VALUE_EMBEDDED_OBJECT); + typeSer.writeTypePrefix(jsonGenerator, typeId); + this.serialize(file, jsonGenerator, provider); + typeSer.writeTypeSuffix(jsonGenerator, typeId); + } + + private void writeFields(File file, JsonGenerator jsonGenerator) + throws IOException { + jsonGenerator.writeStringField("file", file.getName()); + } + } + + private static class FileDeserializer extends StdDeserializer { + + public FileDeserializer() { + super(File.class); + } + + @Override + public File deserialize(JsonParser jsonParser, + DeserializationContext ctxt) + throws IOException { + JsonToken token = jsonParser.currentToken(); + if (token == null) { + token = jsonParser.nextToken(); + } + if (token == JsonToken.VALUE_STRING) { + return new File(jsonParser.getValueAsString()); + } + if (token == JsonToken.START_OBJECT) { + String file = null; + while (jsonParser.nextToken() != JsonToken.END_OBJECT) { + String field = jsonParser.currentName(); + jsonParser.nextToken(); + if ("file".equals(field)) { + file = jsonParser.getValueAsString(); + } else { + jsonParser.skipChildren(); + } + } + if (file == null) { + return (File) ctxt.handleUnexpectedToken(File.class, + jsonParser); + } + return new File(file); + } + return (File) ctxt.handleUnexpectedToken(File.class, jsonParser); + } } private static class BlobSerializer extends StdSerializer { diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/io/HugeGraphTypeSerializerRegistryBuilder.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/io/HugeGraphTypeSerializerRegistryBuilder.java new file mode 100644 index 0000000000..f46e4ee399 --- /dev/null +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/io/HugeGraphTypeSerializerRegistryBuilder.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.io; + +import java.io.IOException; + +import org.apache.hugegraph.backend.id.Id; +import org.apache.tinkerpop.gremlin.structure.io.Buffer; +import org.apache.tinkerpop.gremlin.structure.io.binary.GraphBinaryReader; +import org.apache.tinkerpop.gremlin.structure.io.binary.GraphBinaryWriter; +import org.apache.tinkerpop.gremlin.structure.io.binary.TypeSerializer; +import org.apache.tinkerpop.gremlin.structure.io.binary.TypeSerializerRegistry; +import org.apache.tinkerpop.gremlin.structure.io.binary.types.SimpleTypeSerializer; +import org.apache.tinkerpop.gremlin.structure.io.binary.types.TransformSerializer; + +public final class HugeGraphTypeSerializerRegistryBuilder + extends TypeSerializerRegistry.Builder { + + private static final TypeSerializer ID_TRANSFORM_SERIALIZER = + new IdTransformSerializer(); + + public HugeGraphTypeSerializerRegistryBuilder() { + this.withFallbackResolver(type -> { + if (Id.class.isAssignableFrom(type)) { + return ID_TRANSFORM_SERIALIZER; + } + return null; + }); + } + + private static final class IdTransformSerializer + extends SimpleTypeSerializer + implements TransformSerializer { + + private IdTransformSerializer() { + super(null); + } + + @Override + protected Id readValue(Buffer buffer, GraphBinaryReader context) + throws IOException { + throw new IOException("HugeGraph Id is written as a wire primitive"); + } + + @Override + protected void writeValue(Id value, Buffer buffer, + GraphBinaryWriter context) + throws IOException { + throw new IOException("HugeGraph Id is written as a wire primitive"); + } + + @Override + public Object transform(Id value) { + return value.asObject(); + } + } +} diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/GremlinLangRestrictionStrategy.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/GremlinLangRestrictionStrategy.java new file mode 100644 index 0000000000..0e33d6fb8a --- /dev/null +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/GremlinLangRestrictionStrategy.java @@ -0,0 +1,42 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.security; + +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy; + +public final class GremlinLangRestrictionStrategy + extends AbstractTraversalStrategy + implements TraversalStrategy.DecorationStrategy { + + private static final GremlinLangRestrictionStrategy INSTANCE = + new GremlinLangRestrictionStrategy(); + + private GremlinLangRestrictionStrategy() { + } + + public static GremlinLangRestrictionStrategy instance() { + return INSTANCE; + } + + @Override + public void apply(Traversal.Admin traversal) { + GremlinLangTextPredicateAdapter.restore(traversal); + } +} diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/GremlinLangTextPredicateAdapter.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/GremlinLangTextPredicateAdapter.java new file mode 100644 index 0000000000..ea0c91ef0f --- /dev/null +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/GremlinLangTextPredicateAdapter.java @@ -0,0 +1,479 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.security; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import javax.script.Bindings; +import javax.script.ScriptContext; + +import org.antlr.v4.runtime.BaseErrorListener; +import org.antlr.v4.runtime.CharStreams; +import org.antlr.v4.runtime.CommonTokenStream; +import org.antlr.v4.runtime.RecognitionException; +import org.antlr.v4.runtime.Recognizer; +import org.antlr.v4.runtime.Token; +import org.apache.commons.text.StringEscapeUtils; +import org.apache.hugegraph.traversal.optimize.ConditionP; +import org.apache.tinkerpop.gremlin.language.grammar.GremlinLexer; +import org.apache.tinkerpop.gremlin.process.traversal.Compare; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.step.GValue; +import org.apache.tinkerpop.gremlin.process.traversal.step.HasContainerHolder; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.HasContainer; +import org.apache.tinkerpop.gremlin.process.traversal.util.TraversalHelper; + +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; + +final class GremlinLangTextPredicateAdapter { + + private static final String TEXT = "Text"; + private static final String CONTAINS = "contains"; + private static final String RESERVED_BINDING_PREFIX = + "hugegraphTextContainsInternal"; + private static final long PLAN_CACHE_MAXIMUM_SIZE = 1024L; + private static final long PLAN_CACHE_EXPIRY_MINUTES = 10L; + + private final Cache plans; + + GremlinLangTextPredicateAdapter() { + this.plans = Caffeine.newBuilder() + .maximumSize(PLAN_CACHE_MAXIMUM_SIZE) + .expireAfterAccess(PLAN_CACHE_EXPIRY_MINUTES, + TimeUnit.MINUTES) + .build(); + } + + AdaptedScript adapt(String script, ScriptContext context) { + rejectReservedBindings(context); + if (script.contains(RESERVED_BINDING_PREFIX)) { + rejectReservedIdentifiers(tokens(script)); + } + if (!mightContainTextPredicate(script)) { + return AdaptedScript.identity(script); + } + + RewritePlan plan = this.plans.get( + script, GremlinLangTextPredicateAdapter::parse); + return plan.materialize(context); + } + + static void restore(Traversal.Admin traversal) { + TraversalHelper.applyTraversalRecursively( + GremlinLangTextPredicateAdapter::restoreCurrentTraversal, + traversal); + } + + private static boolean mightContainTextPredicate(String script) { + return script.contains(TEXT) && script.contains(CONTAINS); + } + + private static RewritePlan parse(String script) { + List tokens = tokens(script); + int[] codePointOffsets = codePointToUtf16Offsets(script); + + List occurrences = new ArrayList<>(); + for (int i = 0; i < tokens.size(); i++) { + if (!isTextContainsPrefix(tokens, i)) { + continue; + } + Occurrence occurrence = match(tokens, i, occurrences.size(), + codePointOffsets); + if (occurrence == null) { + throw unsupportedTextContains(); + } + occurrences.add(occurrence); + i += 5; + } + + if (occurrences.isEmpty()) { + return RewritePlan.identity(script); + } + rejectReservedIdentifiers(tokens); + + StringBuilder rewritten = new StringBuilder(script.length()); + int cursor = 0; + for (Occurrence occurrence : occurrences) { + rewritten.append(script, cursor, occurrence.start()); + rewritten.append(occurrence.internalBinding()); + cursor = occurrence.end(); + } + rewritten.append(script, cursor, script.length()); + return new RewritePlan(rewritten.toString(), occurrences); + } + + private static List tokens(String script) { + GremlinLexer lexer = new GremlinLexer(CharStreams.fromString(script)); + lexer.removeErrorListeners(); + lexer.addErrorListener(ThrowingErrorListener.INSTANCE); + CommonTokenStream tokenStream = new CommonTokenStream(lexer); + tokenStream.fill(); + + List tokens = new ArrayList<>(); + for (Token token : tokenStream.getTokens()) { + if (token.getType() != Token.EOF) { + tokens.add(token); + } + } + return tokens; + } + + private static int[] codePointToUtf16Offsets(String script) { + int codePointCount = script.codePointCount(0, script.length()); + int[] offsets = new int[codePointCount + 1]; + int utf16Offset = 0; + for (int i = 0; i < codePointCount; i++) { + offsets[i] = utf16Offset; + int codePoint = script.codePointAt(utf16Offset); + utf16Offset += Character.charCount(codePoint); + } + offsets[codePointCount] = script.length(); + return offsets; + } + + private static Occurrence match(List tokens, int index, + int occurrenceIndex, + int[] codePointOffsets) { + if (index == 0 || index + 6 >= tokens.size()) { + return null; + } + if (tokens.get(index - 1).getType() != GremlinLexer.COMMA || + tokens.get(index + 3).getType() != GremlinLexer.LPAREN || + tokens.get(index + 5).getType() != GremlinLexer.RPAREN || + tokens.get(index + 6).getType() != GremlinLexer.RPAREN) { + return null; + } + + Token argument = tokens.get(index + 4); + if (!isString(argument) && !isIdentifier(argument)) { + return null; + } + + int outerLeftParen = matchingLeftParen(tokens, index + 6); + if (outerLeftParen <= 0 || + tokens.get(outerLeftParen - 1).getType() != + GremlinLexer.K_HAS) { + return null; + } + int commas = topLevelCommas(tokens, outerLeftParen + 1, index); + if (commas != 1 && commas != 2) { + return null; + } + + String internalBinding = RESERVED_BINDING_PREFIX + occurrenceIndex; + String literal = isString(argument) ? + decodeStringLiteral(argument.getText()) : null; + String sourceBinding = isIdentifier(argument) ? + argument.getText() : null; + int start = codePointOffsets[tokens.get(index).getStartIndex()]; + int end = codePointOffsets[ + tokens.get(index + 5).getStopIndex() + 1]; + return new Occurrence(start, end, internalBinding, + literal, sourceBinding); + } + + private static int matchingLeftParen(List tokens, + int rightParen) { + int depth = 0; + for (int i = rightParen; i >= 0; i--) { + int type = tokens.get(i).getType(); + if (type == GremlinLexer.RPAREN) { + depth++; + } else if (type == GremlinLexer.LPAREN && --depth == 0) { + return i; + } + } + return -1; + } + + private static int topLevelCommas(List tokens, int start, + int end) { + int depth = 0; + int commas = 0; + for (int i = start; i < end; i++) { + int type = tokens.get(i).getType(); + if (type == GremlinLexer.LPAREN) { + depth++; + } else if (type == GremlinLexer.RPAREN) { + depth--; + } else if (type == GremlinLexer.COMMA && depth == 0) { + commas++; + } + } + return commas; + } + + private static boolean isTextContainsPrefix(List tokens, + int index) { + return index + 2 < tokens.size() && + isIdentifier(tokens.get(index), TEXT) && + tokens.get(index + 1).getType() == GremlinLexer.DOT && + isIdentifier(tokens.get(index + 2), CONTAINS); + } + + private static boolean isIdentifier(Token token, String value) { + return isIdentifier(token) && value.equals(token.getText()); + } + + private static boolean isIdentifier(Token token) { + return token.getType() == GremlinLexer.Identifier; + } + + private static boolean isString(Token token) { + return token.getType() == GremlinLexer.NonEmptyStringLiteral || + token.getType() == GremlinLexer.EmptyStringLiteral; + } + + private static String decodeStringLiteral(String literal) { + return StringEscapeUtils.unescapeJava( + literal.substring(1, literal.length() - 1)); + } + + private static void rejectReservedIdentifiers(List tokens) { + for (Token token : tokens) { + if (isIdentifier(token) && + token.getText().startsWith(RESERVED_BINDING_PREFIX)) { + throw new IllegalArgumentException( + "Gremlin query uses a reserved HugeGraph binding"); + } + } + } + + private static void rejectReservedBindings(ScriptContext context) { + rejectReservedBindings(context.getBindings( + ScriptContext.ENGINE_SCOPE)); + rejectReservedBindings(context.getBindings( + ScriptContext.GLOBAL_SCOPE)); + } + + private static void rejectReservedBindings(Bindings bindings) { + if (bindings == null) { + return; + } + for (String name : bindings.keySet()) { + if (name.startsWith(RESERVED_BINDING_PREFIX)) { + throw new IllegalArgumentException( + "Gremlin request contains a reserved " + + "HugeGraph binding"); + } + } + } + + private static IllegalArgumentException unsupportedTextContains() { + return new IllegalArgumentException( + "Text.contains() is only supported as the final argument " + + "of has(), with one String literal or String binding"); + } + + private static void restoreCurrentTraversal( + Traversal.Admin traversal) { + for (Object step : traversal.getSteps()) { + if (!(step instanceof HasContainerHolder)) { + continue; + } + HasContainerHolder holder = + (HasContainerHolder) step; + List containers = + new ArrayList<>(holder.getHasContainers()); + for (HasContainer current : containers) { + TextContainsMarker marker = marker(current.getPredicate(), + traversal); + if (marker == null) { + continue; + } + holder.removeHasContainer(current); + holder.addHasContainer(new HasContainer( + current.getKey(), + ConditionP.textContains(marker.value()))); + } + } + } + + private static TextContainsMarker marker( + P predicate, Traversal.Admin traversal) { + if (predicate.getBiPredicate() != Compare.eq) { + return null; + } + if (!predicate.isParameterized()) { + Object value = predicate.getValue(); + return value instanceof TextContainsMarker ? + (TextContainsMarker) value : null; + } + + for (GValue value : predicate.getGValues()) { + if (!value.isVariable() || + !value.getName().startsWith(RESERVED_BINDING_PREFIX)) { + continue; + } + TextContainsMarker current = currentMarker(traversal, + value.getName()); + if (current == null) { + throw new IllegalStateException( + "Missing internal Text.contains() binding"); + } + traversal.getGValueManager().pinVariable(value.getName()); + return current; + } + return null; + } + + private static TextContainsMarker currentMarker( + Traversal.Admin traversal, String name) { + for (GValue value : traversal.getGValueManager().getGValues()) { + if (value.isVariable() && name.equals(value.getName()) && + value.get() instanceof TextContainsMarker) { + return (TextContainsMarker) value.get(); + } + } + return null; + } + + static final class AdaptedScript { + + private final String script; + private final Map bindings; + + private AdaptedScript(String script, Map bindings) { + this.script = script; + this.bindings = bindings; + } + + static AdaptedScript identity(String script) { + return new AdaptedScript(script, Collections.emptyMap()); + } + + String script() { + return this.script; + } + + Map bindings() { + return this.bindings; + } + } + + private static final class RewritePlan { + + private final String script; + private final List occurrences; + + private RewritePlan(String script, List occurrences) { + this.script = script; + this.occurrences = List.copyOf(occurrences); + } + + static RewritePlan identity(String script) { + return new RewritePlan(script, Collections.emptyList()); + } + + AdaptedScript materialize(ScriptContext context) { + if (this.occurrences.isEmpty()) { + return AdaptedScript.identity(this.script); + } + Map bindings = new LinkedHashMap<>(); + for (Occurrence occurrence : this.occurrences) { + String value = occurrence.resolve(context); + bindings.put(occurrence.internalBinding(), + new TextContainsMarker(value)); + } + return new AdaptedScript(this.script, bindings); + } + } + + private static final class Occurrence { + + private final int start; + private final int end; + private final String internalBinding; + private final String literal; + private final String sourceBinding; + + private Occurrence(int start, int end, String internalBinding, + String literal, String sourceBinding) { + this.start = start; + this.end = end; + this.internalBinding = internalBinding; + this.literal = literal; + this.sourceBinding = sourceBinding; + } + + int start() { + return this.start; + } + + int end() { + return this.end; + } + + String internalBinding() { + return this.internalBinding; + } + + String resolve(ScriptContext context) { + if (this.sourceBinding == null) { + return this.literal; + } + Object value = context.getAttribute(this.sourceBinding); + if (!(value instanceof String)) { + throw new IllegalArgumentException(String.format( + "The Text.contains() binding '%s' must be a String", + this.sourceBinding)); + } + return (String) value; + } + } + + private static final class TextContainsMarker implements Serializable { + + private static final long serialVersionUID = 1L; + + private final String value; + + private TextContainsMarker(String value) { + this.value = value; + } + + String value() { + return this.value; + } + } + + private static final class ThrowingErrorListener + extends BaseErrorListener { + + private static final ThrowingErrorListener INSTANCE = + new ThrowingErrorListener(); + + @Override + public void syntaxError(Recognizer recognizer, + Object offendingSymbol, int line, + int charPositionInLine, String message, + RecognitionException exception) { + throw new IllegalArgumentException(String.format( + "Invalid Gremlin token at line %s, character %s: %s", + line, charPositionInLine, message), exception); + } + } +} diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/GremlinLangTraversalVerifier.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/GremlinLangTraversalVerifier.java new file mode 100644 index 0000000000..8cc319dda1 --- /dev/null +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/GremlinLangTraversalVerifier.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.security; + +import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.CallStepContract; +import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IoStep; + +public final class GremlinLangTraversalVerifier { + + private GremlinLangTraversalVerifier() { + } + + public static void verify(Traversal traversal) { + verify(traversal.asAdmin()); + } + + static void verify(Traversal.Admin traversal) { + for (Step step : traversal.getSteps()) { + if (step instanceof IoStep || step instanceof CallStepContract) { + throw new SecurityException(String.format( + "The traversal step '%s' is not allowed for remote " + + "Gremlin requests", step.getClass().getSimpleName())); + } + if (step instanceof TraversalParent) { + TraversalParent parent = (TraversalParent) step; + for (Traversal.Admin child : parent.getLocalChildren()) { + verify(child); + } + for (Traversal.Admin child : parent.getGlobalChildren()) { + verify(child); + } + } + } + } +} diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/GremlinLangVerificationStrategy.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/GremlinLangVerificationStrategy.java new file mode 100644 index 0000000000..62b6505ac2 --- /dev/null +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/GremlinLangVerificationStrategy.java @@ -0,0 +1,76 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.security; + +import java.util.Set; + +import org.apache.tinkerpop.gremlin.process.computer.traversal.strategy.decoration.VertexProgramStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ConnectiveStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.EventStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.HaltedTraverserStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.OptionsStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.PartitionStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.SeedStrategy; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.SubgraphStrategy; + +/* + * Keep this strategy in the decoration category. Optimization strategies such + * as PathRetractionStrategy can inspect CallStep requirements before standard + * verification strategies run. The explicit priors place this check after all + * decoration strategies that GremlinLang can construct, but before any + * optimization can access a forbidden step. + */ +public final class GremlinLangVerificationStrategy + extends AbstractTraversalStrategy + implements TraversalStrategy.DecorationStrategy { + + private static final GremlinLangVerificationStrategy INSTANCE = + new GremlinLangVerificationStrategy(); + private static final Set> PRIORS = + Set.of(GremlinLangRestrictionStrategy.class, + ConnectiveStrategy.class, + ElementIdStrategy.class, + EventStrategy.class, + HaltedTraverserStrategy.class, + OptionsStrategy.class, + PartitionStrategy.class, + SeedStrategy.class, + SubgraphStrategy.class, + VertexProgramStrategy.class); + + private GremlinLangVerificationStrategy() { + } + + public static GremlinLangVerificationStrategy instance() { + return INSTANCE; + } + + @Override + public Set> applyPrior() { + return PRIORS; + } + + @Override + public void apply(Traversal.Admin traversal) { + GremlinLangTraversalVerifier.verify(traversal); + } +} diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeGraphGremlinLangScriptEngine.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeGraphGremlinLangScriptEngine.java new file mode 100644 index 0000000000..1ae9bfd9f6 --- /dev/null +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeGraphGremlinLangScriptEngine.java @@ -0,0 +1,427 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.security; + +import java.io.IOException; +import java.io.Reader; +import java.util.ArrayList; +import java.util.Map; +import java.util.WeakHashMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.atomic.AtomicBoolean; + +import javax.script.AbstractScriptEngine; +import javax.script.Bindings; +import javax.script.ScriptContext; +import javax.script.ScriptException; +import javax.script.SimpleBindings; +import javax.script.SimpleScriptContext; + +import org.apache.tinkerpop.gremlin.jsr223.Customizer; +import org.apache.tinkerpop.gremlin.jsr223.GremlinLangCustomizer; +import org.apache.tinkerpop.gremlin.jsr223.GremlinLangScriptEngine; +import org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngine; +import org.apache.tinkerpop.gremlin.process.traversal.Bytecode; +import org.apache.tinkerpop.gremlin.process.traversal.Traversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; + +public class HugeGraphGremlinLangScriptEngine extends AbstractScriptEngine + implements GremlinScriptEngine { + + private static final String TRAVERSAL_SOURCE = "g"; + private static final String TINKERPOP_INITIALIZATION_PROBE = "1+1"; + private static final SourceRegistry SOURCES = new SourceRegistry(); + + private final HugeGraphGremlinLangScriptEngineFactory factory; + private final Customizer[] customizers; + private final GremlinLangTextPredicateAdapter textPredicateAdapter; + private final ConcurrentMap + delegates; + private final ConcurrentMap + protectedDelegates; + private final AtomicBoolean explicitRegistration; + private final AtomicBoolean initializationProbeAllowed; + + HugeGraphGremlinLangScriptEngine( + HugeGraphGremlinLangScriptEngineFactory factory, + Customizer... customizers) { + this.factory = factory; + this.customizers = withoutTraversalCache(customizers); + this.textPredicateAdapter = new GremlinLangTextPredicateAdapter(); + this.delegates = new ConcurrentHashMap<>(); + this.protectedDelegates = new ConcurrentHashMap<>(); + this.explicitRegistration = new AtomicBoolean(false); + this.initializationProbeAllowed = new AtomicBoolean(true); + } + + @Override + public Object eval(String script, ScriptContext context) + throws ScriptException { + /* + * Gremlin Server evaluates this fixed expression once for every + * configured engine whose registration name is not "gremlin-lang". + * HugeGraph uses a private registration name to avoid colliding with + * TinkerPop's factory, and the probe runs before traversal sources are + * injected. The fixed expression needs no graph access. + */ + if (TINKERPOP_INITIALIZATION_PROBE.equals(script) && + context.getAttribute(TRAVERSAL_SOURCE) == null && + this.initializationProbeAllowed.compareAndSet(true, false)) { + return 2; + } + Delegate delegate = this.delegate(traversalSource(context)); + GremlinLangTextPredicateAdapter.AdaptedScript adapted = + this.textPredicateAdapter.adapt(script, context); + try { + return verify(delegate.engine.eval( + adapted.script(), + guardedContext(context, delegate.traversalSource, + adapted.bindings()))); + } catch (ScriptException e) { + rethrowSecurityException(e); + throw e; + } + } + + @Override + public Object eval(Reader reader, ScriptContext context) + throws ScriptException { + try { + return this.eval(readFully(reader), context); + } catch (IOException e) { + throw new ScriptException(e); + } + } + + @Override + public Traversal.Admin eval(Bytecode bytecode, Bindings bindings, + String traversalSource) + throws ScriptException { + Object source = bindings.get(traversalSource); + if (!(source instanceof GraphTraversalSource)) { + throw new IllegalArgumentException(String.format( + "The binding '%s' must be a GraphTraversalSource", + traversalSource)); + } + Delegate delegate = this.delegate((GraphTraversalSource) source); + Bindings guardedBindings = new SimpleBindings(bindings); + guardedBindings.put(traversalSource, delegate.traversalSource); + Traversal.Admin traversal = delegate.engine.eval( + bytecode, guardedBindings, traversalSource); + GremlinLangTraversalVerifier.verify(traversal); + return traversal; + } + + @Override + public Bindings createBindings() { + return new SimpleBindings(); + } + + @Override + public HugeGraphGremlinLangScriptEngineFactory getFactory() { + return this.factory; + } + + public synchronized GraphTraversalSource add( + GraphTraversalSource traversalSource) { + if (traversalSource == null) { + throw new IllegalArgumentException( + "The traversal source can't be null"); + } + this.explicitRegistration.set(true); + Delegate delegate = this.delegates.computeIfAbsent( + traversalSource, this::newDelegate); + this.protectedDelegates.putIfAbsent(delegate.traversalSource, + delegate); + SOURCES.register(delegate.traversalSource, this); + return delegate.traversalSource; + } + + public synchronized void remove(GraphTraversalSource traversalSource) { + if (traversalSource == null) { + return; + } + Delegate delegate = this.delegates.get(traversalSource); + if (delegate == null) { + delegate = this.protectedDelegates.get(traversalSource); + } + if (delegate != null) { + if (this.explicitRegistration.get()) { + SOURCES.retire(delegate.traversalSource); + } else { + SOURCES.detach(delegate.traversalSource, this); + this.removeLocal(delegate.traversalSource); + } + } + } + + public synchronized void clear() { + if (this.explicitRegistration.get()) { + for (GraphTraversalSource source : + new ArrayList<>(this.protectedDelegates.keySet())) { + SOURCES.retire(source); + } + } else { + for (GraphTraversalSource source : + new ArrayList<>(this.protectedDelegates.keySet())) { + SOURCES.detach(source, this); + } + } + this.delegates.clear(); + this.protectedDelegates.clear(); + } + + public int traversalSourceCount() { + return this.delegates.size(); + } + + private Delegate delegate(GraphTraversalSource traversalSource) { + Delegate delegate = this.delegates.get(traversalSource); + if (delegate == null) { + delegate = this.protectedDelegates.get(traversalSource); + } + if (delegate == null && !this.explicitRegistration.get()) { + delegate = SOURCES.attach(traversalSource, this); + if (delegate == null && + (SOURCES.isRetired(traversalSource) || + isProtected(traversalSource))) { + throw new IllegalArgumentException( + "The protected 'g' binding must reference an active " + + "GraphTraversalSource"); + } + } + if (delegate == null) { + String requirement = this.explicitRegistration.get() ? + "registered" : "protected"; + throw new IllegalArgumentException( + "The 'g' binding must be a " + requirement + " " + + "GraphTraversalSource"); + } + return delegate; + } + + private Delegate attachLocal(GraphTraversalSource traversalSource) { + Delegate delegate = this.delegates.computeIfAbsent( + traversalSource, this::newProtectedDelegate); + this.protectedDelegates.putIfAbsent(delegate.traversalSource, + delegate); + return delegate; + } + + private void removeLocal(GraphTraversalSource traversalSource) { + Delegate delegate = this.protectedDelegates.get(traversalSource); + if (delegate == null) { + delegate = this.delegates.get(traversalSource); + } + if (delegate != null) { + this.delegates.remove(delegate.registrationSource, delegate); + this.protectedDelegates.remove(delegate.traversalSource, + delegate); + } + } + + private Delegate newDelegate(GraphTraversalSource traversalSource) { + GraphTraversalSource protectedSource = traversalSource; + if (!isProtected(protectedSource)) { + protectedSource = traversalSource.withStrategies( + GremlinLangRestrictionStrategy.instance(), + GremlinLangVerificationStrategy.instance()); + } + return new Delegate( + new GremlinLangScriptEngine(this.customizers), + traversalSource, protectedSource); + } + + private Delegate newProtectedDelegate( + GraphTraversalSource traversalSource) { + return new Delegate( + new GremlinLangScriptEngine(this.customizers), + traversalSource, traversalSource); + } + + private static boolean isProtected( + GraphTraversalSource traversalSource) { + return traversalSource.getStrategies().getStrategy( + GremlinLangRestrictionStrategy.class) + .isPresent() && + traversalSource.getStrategies().getStrategy( + GremlinLangVerificationStrategy.class) + .isPresent(); + } + + private static Customizer[] withoutTraversalCache( + Customizer[] customizers) { + Customizer[] safeCustomizers = customizers.clone(); + for (int i = 0; i < safeCustomizers.length; i++) { + if (!(safeCustomizers[i] instanceof GremlinLangCustomizer)) { + continue; + } + GremlinLangCustomizer gremlinLang = + (GremlinLangCustomizer) safeCustomizers[i]; + safeCustomizers[i] = new GremlinLangCustomizer( + false, gremlinLang.getCacheMaker()); + } + return safeCustomizers; + } + + private static GraphTraversalSource traversalSource( + ScriptContext context) { + Object source = context.getAttribute(TRAVERSAL_SOURCE); + if (!(source instanceof GraphTraversalSource)) { + throw new IllegalArgumentException( + "The 'g' binding must be a GraphTraversalSource"); + } + return (GraphTraversalSource) source; + } + + private static Object verify(Object result) { + if (result instanceof Traversal) { + GremlinLangTraversalVerifier.verify((Traversal) result); + } + return result; + } + + private static void rethrowSecurityException(ScriptException exception) { + Throwable cause = exception; + while (cause != null) { + if (cause instanceof SecurityException) { + throw (SecurityException) cause; + } + cause = cause.getCause(); + } + } + + private static ScriptContext guardedContext( + ScriptContext context, + GraphTraversalSource traversalSource, + Map additionalBindings) { + SimpleScriptContext guarded = new SimpleScriptContext(); + guarded.setReader(context.getReader()); + guarded.setWriter(context.getWriter()); + guarded.setErrorWriter(context.getErrorWriter()); + + Bindings engineBindings = new SimpleBindings(); + Bindings original = context.getBindings(ScriptContext.ENGINE_SCOPE); + if (original != null) { + engineBindings.putAll(original); + } + engineBindings.putAll(additionalBindings); + engineBindings.put(TRAVERSAL_SOURCE, traversalSource); + guarded.setBindings(engineBindings, ScriptContext.ENGINE_SCOPE); + + Bindings global = context.getBindings(ScriptContext.GLOBAL_SCOPE); + if (global != null) { + guarded.setBindings(global, ScriptContext.GLOBAL_SCOPE); + } + return guarded; + } + + private static String readFully(Reader reader) throws IOException { + StringBuilder script = new StringBuilder(); + char[] buffer = new char[8192]; + int length; + while ((length = reader.read(buffer)) != -1) { + script.append(buffer, 0, length); + } + return script.toString(); + } + + private static final class Delegate { + + private final GremlinLangScriptEngine engine; + private final GraphTraversalSource registrationSource; + private final GraphTraversalSource traversalSource; + + private Delegate(GremlinLangScriptEngine engine, + GraphTraversalSource registrationSource, + GraphTraversalSource traversalSource) { + this.engine = engine; + this.registrationSource = registrationSource; + this.traversalSource = traversalSource; + } + } + + private static final class SourceRegistry { + + private final Map sources; + private final Map retiredSources; + + private SourceRegistry() { + this.sources = new WeakHashMap<>(); + this.retiredSources = new WeakHashMap<>(); + } + + private synchronized void register(GraphTraversalSource source, + HugeGraphGremlinLangScriptEngine + engine) { + SourceEntry entry = this.sources.computeIfAbsent( + source, key -> new SourceEntry()); + this.retiredSources.remove(source); + entry.engines.put(engine, Boolean.TRUE); + } + + private synchronized Delegate attach( + GraphTraversalSource source, + HugeGraphGremlinLangScriptEngine engine) { + SourceEntry entry = this.sources.get(source); + if (entry == null) { + return null; + } + Delegate delegate = engine.attachLocal(source); + entry.engines.put(engine, Boolean.TRUE); + return delegate; + } + + private synchronized boolean isRetired( + GraphTraversalSource source) { + return this.retiredSources.containsKey(source); + } + + private synchronized void detach( + GraphTraversalSource source, + HugeGraphGremlinLangScriptEngine engine) { + SourceEntry entry = this.sources.get(source); + if (entry != null) { + entry.engines.remove(engine); + } + } + + private synchronized void retire(GraphTraversalSource source) { + SourceEntry entry = this.sources.remove(source); + this.retiredSources.put(source, Boolean.TRUE); + if (entry == null) { + return; + } + for (HugeGraphGremlinLangScriptEngine engine : + new ArrayList<>(entry.engines.keySet())) { + engine.removeLocal(source); + } + entry.engines.clear(); + } + } + + private static final class SourceEntry { + + private final Map engines; + + private SourceEntry() { + this.engines = new WeakHashMap<>(); + } + } +} diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeGraphGremlinLangScriptEngineFactory.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeGraphGremlinLangScriptEngineFactory.java new file mode 100644 index 0000000000..f8bf30bff8 --- /dev/null +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/security/HugeGraphGremlinLangScriptEngineFactory.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.security; + +import java.util.List; + +import org.apache.tinkerpop.gremlin.jsr223.AbstractGremlinScriptEngineFactory; +import org.apache.tinkerpop.gremlin.jsr223.Customizer; +import org.apache.tinkerpop.gremlin.jsr223.GremlinLangScriptEngineFactory; + +public class HugeGraphGremlinLangScriptEngineFactory + extends AbstractGremlinScriptEngineFactory { + + public static final String ENGINE_NAME = "gremlin-lang"; + public static final String INTERNAL_ENGINE_NAME = + "hugegraph-gremlin-lang"; + + private static final GremlinLangScriptEngineFactory BASE_FACTORY = + new GremlinLangScriptEngineFactory(); + + private final Customizer[] fixedCustomizers; + private volatile HugeGraphGremlinLangScriptEngine engine; + + public HugeGraphGremlinLangScriptEngineFactory() { + super(INTERNAL_ENGINE_NAME, BASE_FACTORY.getLanguageName(), + List.of(), List.of()); + this.fixedCustomizers = null; + } + + public HugeGraphGremlinLangScriptEngineFactory( + Customizer... customizers) { + super(INTERNAL_ENGINE_NAME, BASE_FACTORY.getLanguageName(), + List.of(), List.of()); + this.fixedCustomizers = customizers.clone(); + } + + @Override + public synchronized HugeGraphGremlinLangScriptEngine getScriptEngine() { + if (this.engine == null) { + Customizer[] customizers = this.customizers(); + this.engine = new HugeGraphGremlinLangScriptEngine(this, + customizers); + } + return this.engine; + } + + @Override + public List getNames() { + return List.of(INTERNAL_ENGINE_NAME); + } + + @Override + public String getMethodCallSyntax(String object, String method, + String... args) { + return BASE_FACTORY.getMethodCallSyntax(object, method, args); + } + + @Override + public String getOutputStatement(String value) { + return BASE_FACTORY.getOutputStatement(value); + } + + private Customizer[] customizers() { + if (this.fixedCustomizers != null) { + return this.fixedCustomizers.clone(); + } + if (this.manager == null) { + return new Customizer[0]; + } + List customizers = this.manager.getCustomizers( + ENGINE_NAME); + return customizers.toArray(new Customizer[0]); + } +} diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeFeatures.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeFeatures.java index f8bdb8c75c..49be0ecbe9 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeFeatures.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/structure/HugeFeatures.java @@ -186,7 +186,7 @@ public boolean supportsSerializableValues() { @Override public boolean supportsUniformListValues() { - return true; + return false; } @Override @@ -225,7 +225,7 @@ public boolean supportsSerializableValues() { @Override public boolean supportsUniformListValues() { - return true; + return false; } } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/ConditionP.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/ConditionP.java index e41a0df706..2ef93114f1 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/ConditionP.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/ConditionP.java @@ -17,16 +17,23 @@ package org.apache.hugegraph.traversal.optimize; -import java.util.function.BiPredicate; - import org.apache.hugegraph.backend.query.Condition; import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.PBiPredicate; +/** + * A HugeGraph-local predicate used by server-side traversal processing. + * + * This type relies on HugeGraph {@link Condition.RelationType} predicates and + * has no registered GraphSON or GraphBinary wire serializer. Remote clients + * should use supported TinkerPop predicates or server-side query APIs instead + * of sending {@code ConditionP} instances directly. + */ public class ConditionP extends P { private static final long serialVersionUID = 9094970577400072902L; - private ConditionP(final BiPredicate predicate, + private ConditionP(final PBiPredicate predicate, Object value) { super(predicate, value); } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeCountStep.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeCountStep.java index 403bf5be83..f12a84e40e 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeCountStep.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeCountStep.java @@ -55,12 +55,18 @@ public boolean equals(Object obj) { HugeCountStep other = (HugeCountStep) obj; return Objects.equals(this.originGraphStep, - other.originGraphStep) && this.done == other.done; + other.originGraphStep); } @Override public int hashCode() { - return Objects.hash(super.hashCode(), this.originGraphStep, this.done); + return Objects.hash(super.hashCode(), this.originGraphStep); + } + + @Override + public void reset() { + super.reset(); + this.done = false; } @Override diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeCountStepStrategy.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeCountStepStrategy.java index ef380d36b2..60035f83f4 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeCountStepStrategy.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeCountStepStrategy.java @@ -30,8 +30,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.CountGlobalStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.NoOpBarrierStep; -import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.AggregateGlobalStep; -import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.AggregateLocalStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.AggregateStep; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.IdentityStep; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SideEffectStep; import org.apache.tinkerpop.gremlin.process.traversal.step.util.CollectingBarrierStep; @@ -81,8 +80,7 @@ public void apply(Traversal.Admin traversal) { (step instanceof TraversalParent && TraversalHelper.anyStepRecursively(s -> { return s instanceof SideEffectStep || - s instanceof AggregateGlobalStep || - s instanceof AggregateLocalStep; + s instanceof AggregateStep; }, (TraversalParent) step))) { return; } diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeCountStrategy.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeCountStrategy.java index c3a1542f87..661e4de34d 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeCountStrategy.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeCountStrategy.java @@ -20,16 +20,16 @@ import java.util.Collection; import java.util.Collections; -import java.util.EnumSet; import java.util.HashMap; import java.util.Map; import java.util.Objects; import java.util.Set; -import java.util.function.BiPredicate; import org.apache.tinkerpop.gremlin.process.traversal.Compare; import org.apache.tinkerpop.gremlin.process.traversal.Contains; +import org.apache.tinkerpop.gremlin.process.traversal.NotP; import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.PBiPredicate; import org.apache.tinkerpop.gremlin.process.traversal.Step; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; @@ -62,13 +62,18 @@ public final class HugeCountStrategy extends AbstractTraversalStrategy implements TraversalStrategy.OptimizationStrategy { - private static final Map RANGE_PREDICATES = - new HashMap() {{ + private static final Map RANGE_PREDICATES = + new HashMap() {{ put(Contains.within, 1L); put(Contains.without, 0L); }}; - private static final Set INCREASED_OFFSET_SCALAR_PREDICATES = - EnumSet.of(Compare.eq, Compare.neq, Compare.lte, Compare.gt); + private static final Set> + INCREASED_OFFSET_SCALAR_PREDICATES = + Set.of(Compare.eq, Compare.neq, Compare.lte, Compare.gt, + new NotP.NotPBiPredicate<>(Compare.eq), + new NotP.NotPBiPredicate<>(Compare.neq), + new NotP.NotPBiPredicate<>(Compare.lte), + new NotP.NotPBiPredicate<>(Compare.gt)); private static final HugeCountStrategy INSTANCE = new HugeCountStrategy(); @@ -99,7 +104,7 @@ public void apply(final Traversal.Admin traversal) { ((ConnectiveP) isStepPredicate).getPredicates() : Collections.singletonList(isStepPredicate)) { final Object value = p.getValue(); - final BiPredicate predicate = p.getBiPredicate(); + final PBiPredicate predicate = p.getBiPredicate(); if (value instanceof Number) { final long highRangeOffset = INCREASED_OFFSET_SCALAR_PREDICATES.contains(predicate) ? diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeGraphStep.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeGraphStep.java index bdfb9e0b66..bc3984fd8c 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeGraphStep.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeGraphStep.java @@ -37,7 +37,7 @@ import org.slf4j.Logger; public final class HugeGraphStep - extends GraphStep implements QueryHolder { + extends GraphStep implements QueryHolder { private static final long serialVersionUID = -679873894532085972L; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugePrimaryKeyStrategy.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugePrimaryKeyStrategy.java index e6fa880837..5c645c40dc 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugePrimaryKeyStrategy.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugePrimaryKeyStrategy.java @@ -19,17 +19,17 @@ import java.util.LinkedList; import java.util.List; +import java.util.Map; import org.apache.tinkerpop.gremlin.process.traversal.Step; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy.ProviderOptimizationStrategy; -import org.apache.tinkerpop.gremlin.process.traversal.step.Mutating; import org.apache.tinkerpop.gremlin.process.traversal.step.map.AddVertexStartStep; import org.apache.tinkerpop.gremlin.process.traversal.step.map.AddVertexStep; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.AddVertexStepContract; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.AddPropertyStep; import org.apache.tinkerpop.gremlin.process.traversal.strategy.AbstractTraversalStrategy; -import org.apache.tinkerpop.gremlin.structure.T; import org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality; public class HugePrimaryKeyStrategy @@ -47,17 +47,17 @@ public static HugePrimaryKeyStrategy instance() { public void apply(Traversal.Admin traversal) { List removeSteps = new LinkedList<>(); - Mutating curAddStep = null; + AddVertexStepContract curAddStep = null; List stepList = traversal.getSteps(); for (int i = 0, s = stepList.size(); i < s; i++) { Step step = stepList.get(i); if (i == 0 && step instanceof AddVertexStartStep) { - curAddStep = (Mutating) step; + curAddStep = (AddVertexStepContract) step; continue; - } else if (curAddStep == null && (step) instanceof AddVertexStep) { - curAddStep = (Mutating) step; + } else if (curAddStep == null && step instanceof AddVertexStep) { + curAddStep = (AddVertexStepContract) step; continue; } @@ -70,29 +70,18 @@ public void apply(Traversal.Admin traversal) { continue; } - AddPropertyStep propertyStep = (AddPropertyStep) step; + AddPropertyStep propertyStep = (AddPropertyStep) step; if (propertyStep.getCardinality() == Cardinality.single || propertyStep.getCardinality() == null) { - Object[] kvs = new Object[2]; - List kvList = new LinkedList<>(); - - propertyStep.getParameters().getRaw().forEach((k, v) -> { - if (T.key.equals(k)) { - kvs[0] = v.get(0); - } else if (T.value.equals(k)) { - kvs[1] = v.get(0); - } else { - kvList.add(k.toString()); - kvList.add(v.get(0)); + curAddStep.addProperty(propertyStep.getKey(), + propertyStep.getValue()); + for (Map.Entry> entry : + propertyStep.getProperties().entrySet()) { + for (Object value : entry.getValue()) { + curAddStep.addProperty(entry.getKey(), value); } - }); - - curAddStep.configure(kvs); - - if (!kvList.isEmpty()) { - curAddStep.configure(kvList.toArray(new Object[0])); } removeSteps.add(step); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeVertexStep.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeVertexStep.java index bd2e1388c8..33ee8570b5 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeVertexStep.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/HugeVertexStep.java @@ -42,7 +42,7 @@ import org.slf4j.Logger; public class HugeVertexStep - extends VertexStep implements QueryHolder { + extends VertexStep implements QueryHolder { private static final long serialVersionUID = -7850636388424382454L; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/QueryHolder.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/QueryHolder.java index 917f777b95..eaa03ecc3a 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/QueryHolder.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/QueryHolder.java @@ -25,7 +25,8 @@ import org.apache.tinkerpop.gremlin.process.traversal.Order; import org.apache.tinkerpop.gremlin.process.traversal.step.HasContainerHolder; -public interface QueryHolder extends HasContainerHolder, Metadatable { +public interface QueryHolder + extends HasContainerHolder, Metadatable { String SYSPROP_PAGE = "~page"; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtil.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtil.java index e6a56027a1..a20f04201f 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtil.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtil.java @@ -24,7 +24,6 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.function.BiPredicate; import java.util.function.Function; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -57,9 +56,12 @@ import org.apache.hugegraph.util.JsonUtil; import org.apache.tinkerpop.gremlin.process.traversal.Compare; import org.apache.tinkerpop.gremlin.process.traversal.Contains; +import org.apache.tinkerpop.gremlin.process.traversal.NotP; import org.apache.tinkerpop.gremlin.process.traversal.Order; import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.PBiPredicate; import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.TextP; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.step.HasContainerHolder; @@ -177,7 +179,8 @@ public static void extractHasContainer(HugeGraphStep newStep, while (step instanceof HasStep || step instanceof NoOpBarrierStep) { Step nextStep = step.getNextStep(); if (step instanceof HasStep) { - HasContainerHolder holder = (HasContainerHolder) step; + HasContainerHolder holder = + (HasContainerHolder) step; boolean connectiveLabelStep = removeConnectiveLabelStep(step); /* @@ -318,7 +321,7 @@ private static boolean collectPositiveLabelValues( private static void addPositiveLabelValues(HasContainer has, List labels) { P predicate = has.getPredicate(); - BiPredicate bp = predicate.getBiPredicate(); + PBiPredicate bp = predicate.getBiPredicate(); if (bp == Compare.eq) { labels.add(predicate.getValue()); } else { @@ -328,7 +331,7 @@ private static void addPositiveLabelValues(HasContainer has, } private static boolean hasLabelAfterUnusablePredicate(HugeGraphStep step, - HasContainerHolder holder) { + HasContainerHolder holder) { HugeGraph graph = tryGetGraph(step); boolean seenUnusablePredicate = false; for (HasContainer has : holder.getHasContainers()) { @@ -344,7 +347,7 @@ private static boolean hasLabelAfterUnusablePredicate(HugeGraphStep step, } private static boolean hasUnsupportedLabelContainer( - HasContainerHolder holder) { + HasContainerHolder holder) { for (HasContainer has : holder.getHasContainers()) { if (isLabelContainer(has) && !isPositiveLabelContainer(has)) { return true; @@ -366,7 +369,7 @@ private static boolean removeConnectiveLabelStep(Step step) { } private static List extractLabelHasContainers( - HugeGraphStep step, HasContainerHolder holder) { + HugeGraphStep step, HasContainerHolder holder) { List extracted = new ArrayList<>(); for (HasContainer has : holder.getHasContainers()) { if (!isPositiveLabelContainer(has)) { @@ -385,12 +388,12 @@ private static boolean isLabelContainer(HasContainer has) { } static boolean isPositiveLabelContainer(HasContainer has) { - if (!isLabelContainer(has)) { + if (!isLabelContainer(has) || hasNullLabelValue(has)) { return false; } P predicate = has.getPredicate(); - BiPredicate bp = predicate.getBiPredicate(); + PBiPredicate bp = predicate.getBiPredicate(); if (bp == Compare.eq) { return true; } @@ -404,7 +407,7 @@ static boolean isPositiveLabelContainer(HasContainer has) { } private static boolean hasMatchIndexSensitivePredicate( - HasContainerHolder holder) { + HasContainerHolder holder) { for (HasContainer has : holder.getHasContainers()) { if (hasMatchIndexSensitivePredicate(has)) { return true; @@ -414,7 +417,7 @@ private static boolean hasMatchIndexSensitivePredicate( } private static boolean hasUnusableMatchPredicate(HugeGraphStep step, - HasContainerHolder holder) { + HasContainerHolder holder) { HugeGraph graph = tryGetGraph(step); for (HasContainer has : holder.getHasContainers()) { if (!hasMatchIndexSensitivePredicate(has)) { @@ -428,7 +431,7 @@ private static boolean hasUnusableMatchPredicate(HugeGraphStep step, } private static List extractUsableHasContainers( - HugeGraphStep step, HasContainerHolder holder) { + HugeGraphStep step, HasContainerHolder holder) { List extracted = new ArrayList<>(); HugeGraph graph = tryGetGraph(step); for (HasContainer has : holder.getHasContainers()) { @@ -454,7 +457,7 @@ private static boolean hasMatchIndexSensitivePredicate(HasContainer has) { List> predicates = new ArrayList<>(); collectPredicates(predicates, ImmutableList.of(has.getPredicate())); for (P pred : predicates) { - BiPredicate bp = pred.getBiPredicate(); + PBiPredicate bp = pred.getBiPredicate(); if (bp == Compare.neq || bp == Compare.gt || bp == Compare.gte || bp == Compare.lt || bp == Compare.lte) { @@ -541,6 +544,26 @@ private static boolean hasNullPredicate(HasContainer has) { return false; } + private static boolean hasNullLabelValue(HasContainer has) { + if (!isLabelContainer(has)) { + return false; + } + + List> predicates = new ArrayList<>(); + collectPredicates(predicates, ImmutableList.of(has.getPredicate())); + for (P pred : predicates) { + Object value = pred.getValue(); + if (value == null) { + return true; + } + if (value instanceof Collection && + ((Collection) value).contains(null)) { + return true; + } + } + return false; + } + private static boolean hasBooleanIndex(HugeGraph graph, SchemaLabel schemaLabel, PropertyKey pkey) { @@ -591,7 +614,7 @@ private static boolean hasOnlyRangePredicates(HasContainer has) { List> predicates = new ArrayList<>(); collectPredicates(predicates, ImmutableList.of(has.getPredicate())); for (P pred : predicates) { - BiPredicate bp = pred.getBiPredicate(); + PBiPredicate bp = pred.getBiPredicate(); if (bp != Compare.gt && bp != Compare.gte && bp != Compare.lt && bp != Compare.lte) { return false; @@ -607,7 +630,8 @@ public static void extractHasContainer(HugeVertexStep newStep, Step nextStep = step.getNextStep(); if (step instanceof HasStep) { removeConnectiveLabelStep(step); - HasContainerHolder holder = (HasContainerHolder) step; + HasContainerHolder holder = + (HasContainerHolder) step; if (extractHasContainers(newStep, holder)) { TraversalHelper.copyLabels(step, step.getPreviousStep(), false); traversal.removeStep(step); @@ -618,33 +642,69 @@ public static void extractHasContainer(HugeVertexStep newStep, } private static boolean extractHasContainers(HugeGraphStep newStep, - HasContainerHolder holder) { + HasContainerHolder holder) { HugeGraph graph = TraversalUtil.tryGetGraph(newStep); - if (!canExtractHasContainers(graph, holder)) { + if (canExtractHasContainers(graph, holder)) { + for (HasContainer has : holder.getHasContainers()) { + if (!GraphStep.processHasContainerIds(newStep, has)) { + newStep.addHasContainer(has); + } + } + return true; + } + if (!canPartiallyExtractWithLocalTextPropertyPredicates(graph, + holder) || + hasUnsupportedLabelContainer(holder)) { return false; } + + List extracted = new ArrayList<>(); for (HasContainer has : holder.getHasContainers()) { + if (!canExtractHasContainer(graph, has)) { + continue; + } + if (!isSysProp(has.getKey()) && + !hasUsablePartialIndex(graph, newStep, holder, has)) { + continue; + } if (!GraphStep.processHasContainerIds(newStep, has)) { newStep.addHasContainer(has); } + extracted.add(has); } - return true; + removeExtractedHasContainers(holder, extracted); + return holder.getHasContainers().isEmpty(); } private static boolean extractHasContainers(HugeVertexStep newStep, - HasContainerHolder holder) { + HasContainerHolder holder) { HugeGraph graph = TraversalUtil.tryGetGraph(newStep); - if (!canExtractHasContainers(graph, holder)) { + if (canExtractHasContainers(graph, holder)) { + for (HasContainer has : holder.getHasContainers()) { + newStep.addHasContainer(has); + } + return true; + } + if (!canPartiallyExtractWithLocalTextPropertyPredicates(graph, + holder) || + hasUnsupportedLabelContainer(holder)) { return false; } + + List extracted = new ArrayList<>(); for (HasContainer has : holder.getHasContainers()) { + if (!canExtractHasContainer(graph, has)) { + continue; + } newStep.addHasContainer(has); + extracted.add(has); } - return true; + removeExtractedHasContainers(holder, extracted); + return holder.getHasContainers().isEmpty(); } private static boolean canExtractHasContainers(HugeGraph graph, - HasContainerHolder holder) { + HasContainerHolder holder) { for (HasContainer has : holder.getHasContainers()) { if (!canExtractHasContainer(graph, has)) { return false; @@ -653,8 +713,178 @@ private static boolean canExtractHasContainers(HugeGraph graph, return true; } + private static boolean canPartiallyExtractWithLocalTextPropertyPredicates( + HugeGraph graph, HasContainerHolder holder) { + boolean seenLocalTextPropertyPredicate = false; + for (HasContainer has : holder.getHasContainers()) { + if (canExtractHasContainer(graph, has)) { + continue; + } + if (!isLocalTextPropertyPredicate(graph, has)) { + return false; + } + seenLocalTextPropertyPredicate = true; + } + return seenLocalTextPropertyPredicate; + } + + private static boolean isLocalTextPropertyPredicate(HugeGraph graph, + HasContainer has) { + if (graph == null || has.getKey() == null || + has.getPredicate() == null || isSysProp(has.getKey()) || + hasNullPredicate(has)) { + return false; + } + + try { + PropertyKey pkey = graph.propertyKey(has.getKey()); + return pkey != null && pkey.dataType().isText(); + } catch (NotFoundException e) { + return false; + } + } + + private static boolean hasUsablePartialIndex(HugeGraph graph, + HugeGraphStep step, + HasContainerHolder holder, + HasContainer has) { + if (graph == null || hasNonIndexablePredicate(has)) { + return false; + } + + PropertyKey pkey; + try { + pkey = graph.propertyKey(has.getKey()); + } catch (NotFoundException e) { + return false; + } + + Collection schemaLabels = + partialQuerySchemaLabels(graph, step, holder); + boolean seen = false; + for (SchemaLabel schemaLabel : schemaLabels) { + if (!schemaLabel.properties().contains(pkey.id())) { + continue; + } + seen = true; + if (!hasSingleFieldQueryIndex(graph, schemaLabel, pkey, has)) { + return false; + } + } + return seen; + } + + private static boolean hasNonIndexablePredicate(HasContainer has) { + List> predicates = new ArrayList<>(); + collectPredicates(predicates, ImmutableList.of(has.getPredicate())); + for (P predicate : predicates) { + PBiPredicate bp = predicate.getBiPredicate(); + if (bp == Compare.neq || bp == Contains.without) { + return true; + } + } + return false; + } + + private static Collection partialQuerySchemaLabels( + HugeGraph graph, HugeGraphStep step, + HasContainerHolder holder) { + List labels = new ArrayList<>(); + collectPositiveLabelValues(step, labels); + collectPositiveLabelValues(holder, labels); + if (labels.isEmpty()) { + List schemaLabels = new ArrayList<>(); + if (step.returnsVertex()) { + schemaLabels.addAll(graph.vertexLabels()); + } else { + schemaLabels.addAll(graph.edgeLabels()); + } + return schemaLabels; + } + + List schemaLabels = new ArrayList<>(); + try { + for (Object label : labels) { + SchemaLabel schemaLabel; + if (label instanceof Id) { + schemaLabel = step.returnsVertex() ? + graph.vertexLabel((Id) label) : + graph.edgeLabel((Id) label); + } else if (label instanceof String) { + schemaLabel = step.returnsVertex() ? + graph.vertexLabel((String) label) : + graph.edgeLabel((String) label); + } else { + return ImmutableList.of(); + } + if (schemaLabel == null) { + return ImmutableList.of(); + } + schemaLabels.add(schemaLabel); + } + } catch (IllegalArgumentException e) { + return ImmutableList.of(); + } + return schemaLabels; + } + + private static void collectPositiveLabelValues( + HasContainerHolder holder, List labels) { + for (HasContainer has : holder.getHasContainers()) { + if (isPositiveLabelContainer(has)) { + addPositiveLabelValues(has, labels); + } + } + } + + private static boolean hasSingleFieldQueryIndex(HugeGraph graph, + SchemaLabel schemaLabel, + PropertyKey pkey, + HasContainer has) { + boolean requireRange = hasRangePredicate(has); + for (Id id : schemaLabel.indexLabels()) { + IndexLabel indexLabel = indexLabelOrNull(graph, id); + if (indexLabel == null || + !indexLabel.status().ok() || + !matchSingleFieldIndex(indexLabel, pkey)) { + continue; + } + if (requireRange ? indexLabel.indexType().isNumeric() : + !indexLabel.indexType().isSearch()) { + return true; + } + } + return false; + } + + private static boolean hasRangePredicate(HasContainer has) { + List> predicates = new ArrayList<>(); + collectPredicates(predicates, ImmutableList.of(has.getPredicate())); + for (P predicate : predicates) { + PBiPredicate bp = predicate.getBiPredicate(); + if (bp == Compare.gt || bp == Compare.gte || + bp == Compare.lt || bp == Compare.lte) { + return true; + } + } + return false; + } + + private static void removeExtractedHasContainers( + HasContainerHolder holder, + List extracted) { + for (HasContainer has : extracted) { + holder.removeHasContainer(has); + } + } + static boolean canExtractHasContainer(HugeGraph graph, HasContainer has) { + if (has.getKey() == null || has.getPredicate() == null || + hasNullLabelValue(has) || hasNotPredicate(has) || + hasTextPredicate(has)) { + return false; + } if (isSysProp(has.getKey())) { return true; } @@ -678,7 +908,7 @@ static boolean canExtractHasContainer(HugeGraph graph, List> predicates = new ArrayList<>(); collectPredicates(predicates, ImmutableList.of(has.getPredicate())); for (P pred : predicates) { - BiPredicate bp = pred.getBiPredicate(); + PBiPredicate bp = pred.getBiPredicate(); if (bp == Compare.gt || bp == Compare.gte || bp == Compare.lt || bp == Compare.lte) { return false; @@ -687,6 +917,28 @@ static boolean canExtractHasContainer(HugeGraph graph, return true; } + private static boolean hasNotPredicate(HasContainer has) { + List> predicates = new ArrayList<>(); + collectPredicates(predicates, ImmutableList.of(has.getPredicate())); + for (P predicate : predicates) { + if (predicate instanceof NotP) { + return true; + } + } + return false; + } + + private static boolean hasTextPredicate(HasContainer has) { + List> predicates = new ArrayList<>(); + collectPredicates(predicates, ImmutableList.of(has.getPredicate())); + for (P predicate : predicates) { + if (TextP.class.isInstance(predicate)) { + return true; + } + } + return false; + } + public static void extractOrder(Step newStep, Traversal.Admin traversal) { Step step = newStep; @@ -840,7 +1092,7 @@ public static void fillConditionQuery(ConditionQuery query, public static Condition convHas2Condition(HasContainer has, HugeType type, HugeGraph graph) { P p = has.getPredicate(); E.checkArgument(p != null, "The predicate of has(%s) is null", has); - BiPredicate bp = p.getBiPredicate(); + PBiPredicate bp = p.getBiPredicate(); Condition condition; if (keyForContainsKeyOrValue(has.getKey())) { condition = convContains2Relation(graph, has); @@ -913,7 +1165,7 @@ private static Condition convCompare2Relation(HugeGraph graph, HugeType type, HasContainer has) { assert type.isGraph(); - BiPredicate bp = has.getPredicate().getBiPredicate(); + PBiPredicate bp = has.getPredicate().getBiPredicate(); assert bp instanceof Compare; return isSysProp(has.getKey()) ? @@ -924,7 +1176,7 @@ private static Condition convCompare2Relation(HugeGraph graph, private static Condition.Relation convCompare2SyspropRelation(HugeGraph graph, HugeType type, HasContainer has) { - BiPredicate bp = has.getPredicate().getBiPredicate(); + PBiPredicate bp = has.getPredicate().getBiPredicate(); assert bp instanceof Compare; HugeKeys key = token2HugeKey(has.getKey()); @@ -952,7 +1204,7 @@ private static Condition.Relation convCompare2SyspropRelation(HugeGraph graph, private static Condition convCompare2UserpropRelation(HugeGraph graph, HugeType type, HasContainer has) { - BiPredicate bp = has.getPredicate().getBiPredicate(); + PBiPredicate bp = has.getPredicate().getBiPredicate(); assert bp instanceof Compare; String key = has.getKey(); @@ -1012,7 +1264,7 @@ private static Condition convRelationType2Relation(HugeGraph graph, HugeType type, HasContainer has) { assert type.isGraph(); - BiPredicate bp = has.getPredicate().getBiPredicate(); + PBiPredicate bp = has.getPredicate().getBiPredicate(); assert bp instanceof Condition.RelationType; String key = has.getKey(); @@ -1025,7 +1277,7 @@ private static Condition convRelationType2Relation(HugeGraph graph, public static Condition convIn2Relation(HugeGraph graph, HugeType type, HasContainer has) { - BiPredicate bp = has.getPredicate().getBiPredicate(); + PBiPredicate bp = has.getPredicate().getBiPredicate(); assert bp instanceof Contains; Collection values = (Collection) has.getValue(); @@ -1068,7 +1320,7 @@ public static Condition convIn2Relation(HugeGraph graph, public static Condition convContains2Relation(HugeGraph graph, HasContainer has) { // Convert contains-key or contains-value - BiPredicate bp = has.getPredicate().getBiPredicate(); + PBiPredicate bp = has.getPredicate().getBiPredicate(); E.checkArgument(bp == Compare.eq, "CONTAINS query with relation " + "'%s' is not supported", bp); @@ -1097,6 +1349,9 @@ public static HugeKeys string2HugeKey(String key) { } public static HugeKeys token2HugeKey(String key) { + if (key == null) { + return null; + } if (key.equals(T.label.getAccessor())) { return HugeKeys.LABEL; } else if (key.equals(T.id.getAccessor())) { @@ -1178,7 +1433,7 @@ public static void convAllHasSteps(Traversal.Admin traversal) { } public static void convHasStep(HugeGraph graph, HasStep step) { - HasContainerHolder holder = step; + HasContainerHolder holder = step; for (HasContainer has : holder.getHasContainers()) { convPredicateValue(graph, has); } @@ -1187,7 +1442,7 @@ public static void convHasStep(HugeGraph graph, HasStep step) { private static void convPredicateValue(HugeGraph graph, HasContainer has) { // No need to convert if key is sys-prop - if (isSysProp(has.getKey())) { + if (has.getKey() == null || isSysProp(has.getKey())) { return; } PropertyKey pkey = graph.propertyKey(has.getKey()); @@ -1198,8 +1453,7 @@ private static void updatePredicateValue(P predicate, PropertyKey pkey) { List> leafPredicates = new ArrayList<>(); collectPredicates(leafPredicates, ImmutableList.of(predicate)); for (P pred : leafPredicates) { - if (pred.getBiPredicate() == Compare.neq && - pred.getValue() == null) { + if (isNullInequalityPredicate(pred)) { continue; } Object value = validPropertyValue(pred.getValue(), pkey); @@ -1207,6 +1461,19 @@ private static void updatePredicateValue(P predicate, PropertyKey pkey) { } } + private static boolean isNullInequalityPredicate(P predicate) { + if (predicate.getValue() != null) { + return false; + } + if (predicate.getBiPredicate() == Compare.neq) { + return true; + } + if (!(predicate instanceof NotP)) { + return false; + } + return ((NotP) predicate).negate().getBiPredicate() == Compare.eq; + } + private static boolean isSysProp(String key) { if (QueryHolder.SYSPROP_PAGE.equals(key)) { return true; diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/JsonUtil.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/JsonUtil.java index fd00816f66..b2fce8da57 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/JsonUtil.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/JsonUtil.java @@ -52,6 +52,7 @@ public final class JsonUtil { HugeGraphSONModule.registerServiceSerializers(module); HugeGraphSONModule.registerGraphSpaceSerializers(module); HugeGraphSONModule.registerGraphSerializers(module); + HugeGraphSONModule.registerTraversalSerializers(module); MAPPER.registerModule(module); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/Reflection.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/Reflection.java index 6e5fd07527..14e7b77120 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/Reflection.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/util/Reflection.java @@ -20,6 +20,8 @@ import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.util.Arrays; +import java.util.LinkedHashSet; +import java.util.Set; import org.apache.hugegraph.HugeException; import org.apache.hugegraph.exception.NotSupportException; @@ -42,14 +44,14 @@ public class Reflection { registerFieldsToFilterMethodTemp = reflectionClazzTemp.getMethod("registerFieldsToFilter", - Class.class, String[].class); + Class.class, Set.class); registerMethodsToFilterMethodTemp = reflectionClazzTemp.getMethod("registerMethodsToFilter", - Class.class, String[].class); + Class.class, Set.class); } catch (ClassNotFoundException e) { LOG.error("Can't find jdk.internal.reflect.Reflection class, " + - "please ensure you are using Java 11", e); + "please ensure you are using Java 17", e); } catch (NoSuchMethodException e) { LOG.error("Can't find reflection filter methods", e); } @@ -62,34 +64,59 @@ public class Reflection { public static void registerFieldsToFilter(Class containingClass, String... fieldNames) { if (REGISTER_FILEDS_TO_FILTER_METHOD == null) { throw new NotSupportException("Reflection.registerFieldsToFilter() - " + - "requires Java 11 or higher"); + "requires Java 17 or higher"); } try { REGISTER_FILEDS_TO_FILTER_METHOD.setAccessible(true); - REGISTER_FILEDS_TO_FILTER_METHOD.invoke(REFLECTION_CLAZZ, containingClass, fieldNames); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new HugeException("Failed to register class '%s' fields to filter: %s", - containingClass, Arrays.toString(fieldNames)); + REGISTER_FILEDS_TO_FILTER_METHOD.invoke(REFLECTION_CLAZZ, containingClass, + toFilterSet(fieldNames)); + } catch (IllegalAccessException e) { + throw new HugeException( + "Failed to register class '%s' fields to filter: %s", + e, containingClass, Arrays.toString(fieldNames)); + } catch (InvocationTargetException e) { + throwInvocationTargetException(e, containingClass, fieldNames, "fields"); } } public static void registerMethodsToFilter(Class containingClass, String... methodNames) { if (REGISTER_METHODS_TO_FILTER_METHOD == null) { throw new NotSupportException("Reflection.registerMethodsToFilter() - " + - "requires Java 11 or higher"); + "requires Java 17 or higher"); } try { REGISTER_METHODS_TO_FILTER_METHOD.setAccessible(true); REGISTER_METHODS_TO_FILTER_METHOD.invoke(REFLECTION_CLAZZ, containingClass, - methodNames); - } catch (IllegalAccessException | InvocationTargetException e) { - throw new HugeException("Failed to register class '%s' methods to filter: %s", - containingClass, Arrays.toString(methodNames)); + toFilterSet(methodNames)); + } catch (IllegalAccessException e) { + throw new HugeException( + "Failed to register class '%s' methods to filter: %s", + e, containingClass, Arrays.toString(methodNames)); + } catch (InvocationTargetException e) { + throwInvocationTargetException(e, containingClass, methodNames, "methods"); } } + private static Set toFilterSet(String... members) { + return new LinkedHashSet<>(Arrays.asList(members)); + } + + private static void throwInvocationTargetException(InvocationTargetException exception, + Class containingClass, + String[] members, + String type) { + Throwable cause = exception.getCause(); + if (cause instanceof IllegalArgumentException) { + throw (IllegalArgumentException) cause; + } + Throwable failure = cause != null ? cause : exception; + throw new HugeException( + "Failed to register class '%s' %s to filter: %s", + failure, containingClass, type, Arrays.toString(members)); + } + public static Class loadClass(String clazz) { try { return Class.forName(clazz); diff --git a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/version/CoreVersion.java b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/version/CoreVersion.java index 2eff71487a..00d27843c2 100644 --- a/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/version/CoreVersion.java +++ b/hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/version/CoreVersion.java @@ -31,7 +31,7 @@ public class CoreVersion { /** * Update it when the gremlin version changed, search "tinkerpop.version" in pom */ - public static final String GREMLIN_VERSION = "3.5.1"; + public static final String GREMLIN_VERSION = "3.8.1"; static { // Check versions of the dependency packages diff --git a/hugegraph-server/hugegraph-core/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngineFactory b/hugegraph-server/hugegraph-core/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngineFactory new file mode 100644 index 0000000000..19ffbfa6f5 --- /dev/null +++ b/hugegraph-server/hugegraph-core/src/main/resources/META-INF/services/org.apache.tinkerpop.gremlin.jsr223.GremlinScriptEngineFactory @@ -0,0 +1 @@ +org.apache.hugegraph.security.HugeGraphGremlinLangScriptEngineFactory diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh index caffedc482..2c93aa085c 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/hugegraph-server.sh @@ -63,11 +63,12 @@ ensure_path_writable "$PLUGINS" # The maximum and minimum heap memory that service can use MAX_MEM=$((32 * 1024)) MIN_MEM=$((1 * 512)) -MIN_JAVA_VERSION=11 +MIN_JAVA_VERSION=17 # JDK 24 removed the Security Manager (JEP 486): "-Djava.security.manager=allow" # is a fatal VM initialization error there and System.setSecurityManager() always # throws, so HugeSecurityManager cannot be installed on newer runtimes. MAX_SECURITY_JAVA_VERSION=23 +JVM_MODULE_OPTIONS="${BIN}/jvm-module.options" # Add the slf4j-log4j12 binding CP=$(find -L $LIB -name 'log4j-slf4j-impl*.jar' | sort | tr '\n' ':') @@ -114,6 +115,11 @@ if [[ -z $JAVA_VERSION || $JAVA_VERSION -lt $MIN_JAVA_VERSION ]]; then exit 1 fi +if [[ ! -r ${JVM_MODULE_OPTIONS} ]]; then + echo "Missing or unreadable JVM module options file: ${JVM_MODULE_OPTIONS}" >> "${OUTPUT}" + exit 1 +fi + # Set Java options if [ "$JAVA_OPTIONS" = "" ]; then XMX=$(calc_xmx $MIN_MEM $MAX_MEM) @@ -128,12 +134,6 @@ if [ "$JAVA_OPTIONS" = "" ]; then # -Xloggc:./logs/gc.log -XX:+PrintHeapAtGC -XX:+PrintGCDetails -XX:+PrintGCDateStamps" fi -if [[ $JAVA_VERSION -gt 9 ]]; then - JAVA_OPTIONS="${JAVA_OPTIONS} --add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED \ - --add-modules=jdk.unsupported \ - --add-exports=java.base/sun.nio.ch=ALL-UNNAMED " -fi - # Using G1GC as the default garbage collector (Recommended for large memory machines) # mention: zgc is only available on ARM-Mac with java > 13 case "$GC_OPTION" in @@ -144,7 +144,7 @@ case "$GC_OPTION" in -XX:G1RSetUpdatingPauseTimePercent=5" ;; zgc|ZGC) - echo "Using ZGC as the default garbage collector (Only support Java 11+)" + echo "Using ZGC as the default garbage collector (requires Java 17 or later)" JAVA_OPTIONS="${JAVA_OPTIONS} -XX:+UseZGC -XX:+UnlockExperimentalVMOptions \ -XX:ConcGCThreads=2 -XX:ParallelGCThreads=6 \ -XX:ZCollectionInterval=120 -XX:ZAllocationSpikeTolerance=5 \ @@ -258,12 +258,12 @@ fi # Turn on security check if [[ "${STDOUT_MODE:-false}" == "true" ]]; then - exec ${JAVA} -Dname="HugeGraphServer" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + exec ${JAVA} @"${JVM_MODULE_OPTIONS}" -Dname="HugeGraphServer" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ ${SECURITY_MANAGER_OPTION} -cp ${CLASSPATH}: \ org.apache.hugegraph.bootstrap.HugeGraphServerBootstrap \ ${OPEN_SECURITY_CHECK} ${GREMLIN_SERVER_CONF} ${REST_SERVER_CONF} else - exec ${JAVA} -Dname="HugeGraphServer" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ + exec ${JAVA} @"${JVM_MODULE_OPTIONS}" -Dname="HugeGraphServer" ${JVM_OPTIONS} ${JAVA_OPTIONS} \ ${SECURITY_MANAGER_OPTION} -cp ${CLASSPATH}: \ org.apache.hugegraph.bootstrap.HugeGraphServerBootstrap \ ${OPEN_SECURITY_CHECK} ${GREMLIN_SERVER_CONF} ${REST_SERVER_CONF} \ diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh index 74ec0bb731..d934192fad 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/init-store.sh @@ -30,6 +30,7 @@ TOP="$(cd "${BIN}"/../ && pwd)" CONF="$TOP/conf" LIB="$TOP/lib" PLUGINS="$TOP/plugins" +JVM_MODULE_OPTIONS="${BIN}/jvm-module.options" . "${BIN}"/util.sh @@ -38,15 +39,16 @@ ensure_path_writable "${PLUGINS}" if [[ -n "$JAVA_HOME" ]]; then JAVA="$JAVA_HOME"/bin/java - EXT="$JAVA_HOME/jre/lib/ext:$LIB:$PLUGINS" else JAVA=java - EXT="$LIB:$PLUGINS" fi cd "${TOP}" || exit -DEFAULT_JAVA_OPTIONS="--add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED" +if [[ ! -r ${JVM_MODULE_OPTIONS} ]]; then + echo "Missing or unreadable JVM module options file: ${JVM_MODULE_OPTIONS}" >&2 + exit 1 +fi echo "Initializing HugeGraph Store..." @@ -54,7 +56,7 @@ echo "Initializing HugeGraph Store..." CP=$(find -L "${LIB}" -name 'hugegraph*.jar' | sort | tr '\n' ':') CP="$CP":$(find -L "${LIB}" -name '*.jar' \! -name 'hugegraph*' | sort | tr '\n' ':') CP="$CP":$(find -L "${PLUGINS}" -name '*.jar' | sort | tr '\n' ':') -$JAVA -cp $CP ${DEFAULT_JAVA_OPTIONS} \ +"${JAVA}" @"${JVM_MODULE_OPTIONS}" -cp "$CP" \ org.apache.hugegraph.cmd.InitStore "${CONF}"/rest-server.properties INIT_STORE_STATUS=$? if [[ ${INIT_STORE_STATUS} -ne 0 ]]; then diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/jvm-module.options b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/jvm-module.options new file mode 100644 index 0000000000..eeb6114c5c --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/jvm-module.options @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +--add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED +--add-modules=jdk.unsupported +--add-exports=java.base/sun.nio.ch=ALL-UNNAMED diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh index 2c99238327..46044e932b 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-hugegraph.sh @@ -99,6 +99,22 @@ if [[ $PRELOAD == "true" ]]; then sed -i -e '/registerBackends/d; /serverStarted/d' "${SCRIPTS}/${EXAMPLE_SCRIPT}" fi +function forward_signal_and_wait() { + local signal="$1" + local exit_code="$2" + + trap - SIGHUP SIGINT SIGQUIT SIGTERM + if kill -0 "$PID" 2>/dev/null; then + kill "-$signal" "$PID" 2>/dev/null || true + # The foreground wait is interrupted before the trap runs, so retry + # until the child has exited and been reaped. + while kill -0 "$PID" 2>/dev/null; do + wait "$PID" 2>/dev/null || true + done + fi + exit "$exit_code" +} + if [[ $DAEMON == "true" ]]; then echo "Starting HugeGraphServer in daemon mode..." "${BIN}"/hugegraph-server.sh "${CONF}/${GREMLIN_SERVER_CONF}" "${CONF}"/rest-server.properties \ @@ -133,7 +149,14 @@ else PID="$!" # Write pid to file echo "$PID" > "$PID_FILE" - trap 'kill $PID; wait $PID; exit $?' SIGHUP SIGINT SIGQUIT SIGTERM + trap 'forward_signal_and_wait HUP 129' SIGHUP + # The background JVM can inherit an ignored SIGINT disposition, so use + # SIGTERM to guarantee that Ctrl-C shuts it down while retaining exit 130. + trap 'forward_signal_and_wait TERM 130' SIGINT + # Forward TERM instead of QUIT: the JVM only dumps threads on SIGQUIT + # and keeps running, which would leave the wait loop below stuck. + trap 'forward_signal_and_wait TERM 131' SIGQUIT + trap 'forward_signal_and_wait TERM 143' SIGTERM wait $PID exit $? fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-monitor.sh b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-monitor.sh index 570d07b545..ffe1a2243d 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-monitor.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/bin/start-monitor.sh @@ -31,7 +31,7 @@ TOP="$(cd $BIN/../ && pwd)" . $BIN/util.sh if [ "$JAVA_HOME" == "" ]; then - echo "Must set JAVA_HOME environment variable and installed jdk >= 1.8" + echo "Must set JAVA_HOME environment variable and install JDK >= 17" exit 1 fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/gremlin-driver-settings.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/gremlin-driver-settings.yaml index 55f38ab97d..e9a28f0aa9 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/gremlin-driver-settings.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/gremlin-driver-settings.yaml @@ -17,9 +17,15 @@ hosts: [localhost] port: 8182 serializer: { - className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, + className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV1, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } +# Typed GraphSON fallback for clients that require @type/@value for +# File/Id/simple typed values: +# use org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV1 in +# place of GraphSONUntypedMessageSerializerV1, and keep HugeGraphIoRegistry. +# Full HugeGraph internal object typed GraphSON for schema/Vertex/Edge/Path +# remains follow-up scope; keep untyped serializers for normal remote objects. diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/gremlin-server.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/gremlin-server.yaml index 32135163fd..b1991fd8cc 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/gremlin-server.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/gremlin-server.yaml @@ -21,11 +21,24 @@ # timeout in ms of gremlin query evaluationTimeout: 30000 -channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer +channelizer: org.apache.hugegraph.auth.HugeGraphWsAndHttpChannelizer # don't set graph at here, this happens after support for dynamically adding graph graphs: { } scriptEngines: { + # Internal registration name; remote clients still use gremlin-lang. + hugegraph-gremlin-lang: { + plugins: { + org.apache.tinkerpop.gremlin.jsr223.GremlinLangPlugin: { + # Parsed traversals can retain mutable request bindings. + # HugeGraph caches only immutable Text.contains() rewrite plans. + cacheEnabled: false + }, + org.apache.tinkerpop.gremlin.jsr223.VariableResolverPlugin: { + resolver: DefaultVariableResolver + } + } + }, gremlin-groovy: { staticImports: [ org.opencypher.gremlin.process.traversal.CustomPredicates.*', @@ -82,30 +95,54 @@ scriptEngines: { } } serializers: - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1, + config: { + serializeResultToString: false, + builder: org.apache.hugegraph.io.HugeGraphTypeSerializerRegistryBuilder, + ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] + } + } + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV1, + config: { + serializeResultToString: false, + ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] + } + } + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV2, + config: { + serializeResultToString: false, + ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] + } + } + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV3, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, + # Keep untyped GraphSON before typed GraphSON so application/json stays + # mapped to the untyped V1 serializer while explicit typed MIME requests work. + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV1, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV2, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV3, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } +# Typed GraphSON remains fallback scope for File/Id/simple typed values. +# Full HugeGraph internal object typed GraphSON for schema/Vertex/Edge/Path +# remains follow-up scope; keep untyped serializers for normal remote objects. metrics: { consoleReporter: {enabled: false, interval: 180000}, csvReporter: {enabled: false, interval: 180000, fileName: ./metrics/gremlin-server-metrics.csv}, diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/remote-objects.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/remote-objects.yaml index 39679d8c30..1ab52aa3c8 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/remote-objects.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/remote-objects.yaml @@ -17,7 +17,7 @@ hosts: [localhost] port: 8182 serializer: { - className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, + className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV1, config: { serializeResultToString: false, # The duplication of HugeGraphIoRegistry is meant to fix a bug in the @@ -28,3 +28,9 @@ serializer: { ] } } +# Typed GraphSON fallback for clients that require @type/@value for +# File/Id/simple typed values: +# use org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV1 in +# place of GraphSONUntypedMessageSerializerV1, and keep HugeGraphIoRegistry. +# Full HugeGraph internal object typed GraphSON for schema/Vertex/Edge/Path +# remains follow-up scope; keep untyped serializers for normal remote objects. diff --git a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/remote.yaml b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/remote.yaml index 55f38ab97d..e9a28f0aa9 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/static/conf/remote.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/static/conf/remote.yaml @@ -17,9 +17,15 @@ hosts: [localhost] port: 8182 serializer: { - className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, + className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV1, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } +# Typed GraphSON fallback for clients that require @type/@value for +# File/Id/simple typed values: +# use org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV1 in +# place of GraphSONUntypedMessageSerializerV1, and keep HugeGraphIoRegistry. +# Full HugeGraph internal object typed GraphSON for schema/Vertex/Edge/Path +# remains follow-up scope; keep untyped serializers for normal remote objects. diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/ci-service-utils.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/ci-service-utils.sh new file mode 100644 index 0000000000..934db1f171 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/ci-service-utils.sh @@ -0,0 +1,211 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +function dump_service_diagnostics() { + local service_dir="$1" + local service_name="$2" + local log_dir="${service_dir}/logs" + + echo "::group::${service_name} diagnostics" + echo "[ci] service dir: ${service_dir}" + echo "[ci] java processes:" + ps -ef | grep -E "HugeGraph|hg-|java" | grep -v grep || true + echo "[ci] listening tcp ports:" + (ss -ltnp || netstat -ltnp || true) 2>&1 + + if [ -d "${log_dir}" ]; then + find "${log_dir}" -maxdepth 2 -type f | sort | while read -r log_file; do + echo "--- tail -n 200 ${log_file} ---" + tail -n 200 "${log_file}" || true + done + else + echo "[ci] log dir not found: ${log_dir}" + fi + echo "::endgroup::" +} + +function wait_for_tcp_port() { + local service_name="$1" + local host="$2" + local port="$3" + local pid_file="$4" + local service_dir="$5" + local timeout_seconds="${6:-90}" + + echo "[ci] waiting for ${service_name} at ${host}:${port}" + for second in $(seq 1 "${timeout_seconds}"); do + if bash -c "echo > /dev/tcp/${host}/${port}" >/dev/null 2>&1; then + echo "[ci] ${service_name} is listening on ${host}:${port}" + return 0 + fi + + if [ -f "${pid_file}" ]; then + local pid + pid="$(cat "${pid_file}")" + if [ -n "${pid}" ] && ! kill -0 "${pid}" >/dev/null 2>&1; then + echo "[ci] ${service_name} process ${pid} exited before readiness" + dump_service_diagnostics "${service_dir}" "${service_name}" + return 1 + fi + fi + + if [ "$((second % 10))" -eq 0 ]; then + echo "[ci] still waiting for ${service_name} (${second}s)" + fi + sleep 1 + done + + echo "[ci] timeout waiting for ${service_name} at ${host}:${port}" + dump_service_diagnostics "${service_dir}" "${service_name}" + return 1 +} + +function http_status_is_accepted() { + local status="$1" + local accepted_statuses="$2" + + case ",${accepted_statuses}," in + *",${status},"*) + return 0 + ;; + *) + return 1 + ;; + esac +} + +function wait_for_http_status() { + local service_name="$1" + local url="$2" + local pid_file="$3" + local service_dir="$4" + local timeout_seconds="${5:-90}" + local accepted_statuses="${6:-200}" + local connect_timeout_seconds=2 + local max_request_seconds=5 + local started_at="${SECONDS}" + local deadline=$((started_at + timeout_seconds)) + local next_log_at=10 + + echo "[ci] waiting for ${service_name} HTTP readiness at ${url}" + echo "[ci] accepted HTTP statuses: ${accepted_statuses}" + while (( SECONDS < deadline )); do + local remaining=$((deadline - SECONDS)) + local request_timeout="${max_request_seconds}" + if (( remaining < request_timeout )); then + request_timeout="${remaining}" + fi + if (( request_timeout < 1 )); then + break + fi + + local status + status="$(curl -s -o /dev/null -w "%{http_code}" \ + --connect-timeout "${connect_timeout_seconds}" \ + --max-time "${request_timeout}" \ + "${url}" 2>/dev/null)" || status="000" + if http_status_is_accepted "${status}" "${accepted_statuses}"; then + echo "[ci] ${service_name} is HTTP ready at ${url}" \ + "(status ${status})" + return 0 + fi + + if [ -f "${pid_file}" ]; then + local pid + pid="$(cat "${pid_file}")" + if [ -n "${pid}" ] && ! kill -0 "${pid}" >/dev/null 2>&1; then + echo "[ci] ${service_name} process ${pid} exited before" \ + "HTTP readiness" + dump_service_diagnostics "${service_dir}" "${service_name}" + return 1 + fi + fi + + local elapsed=$((SECONDS - started_at)) + if (( elapsed >= next_log_at )); then + echo "[ci] still waiting for ${service_name} HTTP readiness" \ + "(${elapsed}s, last status ${status})" + next_log_at=$((next_log_at + 10)) + fi + if (( SECONDS >= deadline )); then + break + fi + sleep 1 + done + + echo "[ci] timeout waiting for ${service_name} HTTP readiness at ${url}" + dump_service_diagnostics "${service_dir}" "${service_name}" + return 1 +} + +function process_is_running() { + local pid="$1" + local state + + if [[ ! "${pid}" =~ ^[0-9]+$ ]]; then + return 1 + fi + + if ! kill -0 "${pid}" 2>/dev/null; then + return 1 + fi + + state="$(ps -o stat= -p "${pid}" 2>/dev/null | tr -d '[:space:]')" || + state="" + [[ "${state}" != Z* ]] +} + +function wait_for_process_exit() { + local pid="$1" + local timeout_seconds="${2:-10}" + local deadline=$((SECONDS + timeout_seconds)) + + while process_is_running "${pid}"; do + if (( SECONDS >= deadline )); then + return 1 + fi + sleep 1 + done + return 0 +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + command="$1" + shift || true + case "${command}" in + dump) + dump_service_diagnostics "$@" + exit $? + ;; + wait) + wait_for_tcp_port "$@" + exit $? + ;; + wait-http) + wait_for_http_status "$@" + exit $? + ;; + *) + echo "Usage: $0 dump SERVICE_DIR SERVICE_NAME" + echo " $0 wait SERVICE_NAME HOST PORT PID_FILE SERVICE_DIR [TIMEOUT_SECONDS]" + echo " $0 wait-http SERVICE_NAME URL PID_FILE SERVICE_DIR" \ + "[TIMEOUT_SECONDS] [ACCEPTED_STATUSES]" + exit 2 + ;; + esac +fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/conf-raft1/gremlin-server.yaml b/hugegraph-server/hugegraph-dist/src/assembly/travis/conf-raft1/gremlin-server.yaml index 7e10eb52b0..22889b1bca 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/conf-raft1/gremlin-server.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/conf-raft1/gremlin-server.yaml @@ -21,11 +21,24 @@ port: 8181 # timeout in ms of gremlin query evaluationTimeout: 60000 -channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer +channelizer: org.apache.hugegraph.auth.HugeGraphWsAndHttpChannelizer # don't set graph at here, this happens after support for dynamically adding graph graphs: { } scriptEngines: { + # Internal registration name; remote clients still use gremlin-lang. + hugegraph-gremlin-lang: { + plugins: { + org.apache.tinkerpop.gremlin.jsr223.GremlinLangPlugin: { + # Parsed traversals can retain mutable request bindings. + # HugeGraph caches only immutable Text.contains() rewrite plans. + cacheEnabled: false + }, + org.apache.tinkerpop.gremlin.jsr223.VariableResolverPlugin: { + resolver: DefaultVariableResolver + } + } + }, gremlin-groovy: { plugins: { org.apache.hugegraph.plugin.HugeGraphGremlinPlugin: {}, @@ -74,25 +87,44 @@ scriptEngines: { } } serializers: - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1, + config: { + serializeResultToString: false, + builder: org.apache.hugegraph.io.HugeGraphTypeSerializerRegistryBuilder, + ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] + } + } + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV1, + config: { + serializeResultToString: false, + ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] + } + } + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV2, + config: { + serializeResultToString: false, + ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] + } + } + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV3, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV1, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV2, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV3, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/conf-raft2/gremlin-server.yaml b/hugegraph-server/hugegraph-dist/src/assembly/travis/conf-raft2/gremlin-server.yaml index 5f097f91bc..18bb617a9e 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/conf-raft2/gremlin-server.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/conf-raft2/gremlin-server.yaml @@ -21,11 +21,24 @@ port: 8182 # timeout in ms of gremlin query evaluationTimeout: 60000 -channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer +channelizer: org.apache.hugegraph.auth.HugeGraphWsAndHttpChannelizer # don't set graph at here, this happens after support for dynamically adding graph graphs: { } scriptEngines: { + # Internal registration name; remote clients still use gremlin-lang. + hugegraph-gremlin-lang: { + plugins: { + org.apache.tinkerpop.gremlin.jsr223.GremlinLangPlugin: { + # Parsed traversals can retain mutable request bindings. + # HugeGraph caches only immutable Text.contains() rewrite plans. + cacheEnabled: false + }, + org.apache.tinkerpop.gremlin.jsr223.VariableResolverPlugin: { + resolver: DefaultVariableResolver + } + } + }, gremlin-groovy: { plugins: { org.apache.hugegraph.plugin.HugeGraphGremlinPlugin: {}, @@ -74,25 +87,44 @@ scriptEngines: { } } serializers: - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1, + config: { + serializeResultToString: false, + builder: org.apache.hugegraph.io.HugeGraphTypeSerializerRegistryBuilder, + ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] + } + } + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV1, + config: { + serializeResultToString: false, + ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] + } + } + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV2, + config: { + serializeResultToString: false, + ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] + } + } + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV3, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV1, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV2, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV3, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/conf-raft3/gremlin-server.yaml b/hugegraph-server/hugegraph-dist/src/assembly/travis/conf-raft3/gremlin-server.yaml index 3f50c64778..29a64513f3 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/conf-raft3/gremlin-server.yaml +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/conf-raft3/gremlin-server.yaml @@ -21,11 +21,24 @@ port: 8183 # timeout in ms of gremlin query evaluationTimeout: 60000 -channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer +channelizer: org.apache.hugegraph.auth.HugeGraphWsAndHttpChannelizer # don't set graph at here, this happens after support for dynamically adding graph graphs: { } scriptEngines: { + # Internal registration name; remote clients still use gremlin-lang. + hugegraph-gremlin-lang: { + plugins: { + org.apache.tinkerpop.gremlin.jsr223.GremlinLangPlugin: { + # Parsed traversals can retain mutable request bindings. + # HugeGraph caches only immutable Text.contains() rewrite plans. + cacheEnabled: false + }, + org.apache.tinkerpop.gremlin.jsr223.VariableResolverPlugin: { + resolver: DefaultVariableResolver + } + } + }, gremlin-groovy: { plugins: { org.apache.hugegraph.plugin.HugeGraphGremlinPlugin: {}, @@ -74,25 +87,44 @@ scriptEngines: { } } serializers: - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1, + config: { + serializeResultToString: false, + builder: org.apache.hugegraph.io.HugeGraphTypeSerializerRegistryBuilder, + ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] + } + } + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV1, + config: { + serializeResultToString: false, + ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] + } + } + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV2, + config: { + serializeResultToString: false, + ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] + } + } + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV3, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV1d0, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV1, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV2d0, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV2, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] } } - - {className: org.apache.tinkerpop.gremlin.driver.ser.GraphSONMessageSerializerV3d0, + - {className: org.apache.tinkerpop.gremlin.util.ser.GraphSONMessageSerializerV3, config: { serializeResultToString: false, ioRegistries: [org.apache.hugegraph.io.HugeGraphIoRegistry] diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-gremlin-console-smoke-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-gremlin-console-smoke-test.sh index c2b693aff5..089732a9bd 100644 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-gremlin-console-smoke-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-gremlin-console-smoke-test.sh @@ -59,16 +59,67 @@ EOF cat > "$SMOKE_SCRIPT" <&2 exit 1 fi @@ -97,8 +99,10 @@ if [[ "$ACTUAL_ARCH" == "riscv64" ]]; then echo "Expected RISC-V Server VM, got $JAVA_VM_NAME" >&2 exit 1 fi - if [[ "$JAVA_VM_VENDOR" != "Alibaba" ]]; then - echo "Expected RISC-V Java vendor Alibaba, got $JAVA_VM_VENDOR" >&2 + if [[ -n "$EXPECTED_RISCV64_JAVA_VENDOR" && \ + "$JAVA_VM_VENDOR" != "$EXPECTED_RISCV64_JAVA_VENDOR" ]]; then + echo "Expected RISC-V Java vendor $EXPECTED_RISCV64_JAVA_VENDOR," \ + "got $JAVA_VM_VENDOR" >&2 exit 1 fi if [[ "$JAVA_VM_INFO" != *"mixed mode"* ]]; then diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-server-e2e-smoke-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-server-e2e-smoke-test.sh index 9762e4fa26..59fd7cfa04 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-server-e2e-smoke-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-server-e2e-smoke-test.sh @@ -137,7 +137,7 @@ verify_graph() { request POST /gremlin 200 \ "$(jq -cn --arg query "g.V().hasLabel('$VERTEX_LABEL').count()" \ - '{gremlin:$query, bindings:{}, language:"gremlin-groovy", + '{gremlin:$query, bindings:{}, aliases:{g:"__g_DEFAULT-hugegraph"}}')" assert_json '.result.data == [2]' } diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-tinkerpop-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-tinkerpop-test.sh index 87d60c8880..f37ba07049 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-tinkerpop-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-tinkerpop-test.sh @@ -17,13 +17,74 @@ # set -ev +if [[ $# -ne 2 ]]; then + echo "Usage: $0 BACKEND {structure|process|process-standard|process-feature|tinkerpop}" + exit 2 +fi + BACKEND=$1 SUITE=$2 +REPORT_DIR=hugegraph-server/hugegraph-test/target/surefire-reports -if [[ "$SUITE" == "structure" || "$SUITE" == "tinkerpop" ]]; then +function run_structure_test() { mvn test -pl hugegraph-server/hugegraph-test -am -P tinkerpop-structure-test,$BACKEND -fi +} -if [[ "$SUITE" == "process" || "$SUITE" == "tinkerpop" ]]; then +function run_process_test() { mvn test -pl hugegraph-server/hugegraph-test -am -P tinkerpop-process-test,$BACKEND -fi +} + +function run_selected_process_test() { + local tests=$1 + shift + if [[ $# -eq 0 ]]; then + echo "At least one expected Surefire report is required" + exit 2 + fi + local expected_reports=("$@") + local expected_report + local report + + for expected_report in "${expected_reports[@]}"; do + report="$REPORT_DIR/TEST-org.apache.hugegraph.tinkerpop.$expected_report.xml" + rm -f "$report" + done + mvn test -pl hugegraph-server/hugegraph-test -am \ + -P tinkerpop-process-test,$BACKEND \ + -Dtest="$tests" \ + -Dsurefire.failIfNoSpecifiedTests=false + + for expected_report in "${expected_reports[@]}"; do + report="$REPORT_DIR/TEST-org.apache.hugegraph.tinkerpop.$expected_report.xml" + if [[ ! -s "$report" ]] || ! grep -Eq 'tests="[1-9][0-9]*"' "$report"; then + echo "Expected a non-empty Surefire report: $report" + exit 1 + fi + done +} + +case "$SUITE" in + structure) + run_structure_test + ;; + process) + run_process_test + ;; + process-standard) + run_selected_process_test \ + "ProcessStandardTest,HugeGraphProviderLifecycleTest" \ + "ProcessStandardTest" \ + "HugeGraphProviderLifecycleTest" + ;; + process-feature) + run_selected_process_test "HugeGraphFeatureTest" "HugeGraphFeatureTest" + ;; + tinkerpop) + run_structure_test + run_process_test + ;; + *) + echo "Unsupported TinkerPop suite: $SUITE" + exit 2 + ;; +esac diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh index 35e82ade40..5ebb6c53a5 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-pd.sh @@ -30,8 +30,12 @@ else fi PD_DIR=$HOME_DIR/hugegraph-pd/apache-hugegraph-pd-$VersionInBash +TRAVIS_DIR=$(dirname "$0") + +source "$TRAVIS_DIR"/ci-service-utils.sh pushd $PD_DIR . bin/start-hugegraph-pd.sh -sleep 10 +wait_for_http_status HugeGraphPD http://127.0.0.1:8620/v1/health \ + "$PD_DIR"/bin/pid "$PD_DIR" 90 200,401 popd diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh index 3e876ce9a0..cb54a03efe 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-store.sh @@ -30,8 +30,12 @@ else fi STORE_DIR=$HOME_DIR/hugegraph-store/apache-hugegraph-store-$VersionInBash +TRAVIS_DIR=$(dirname "$0") + +source "$TRAVIS_DIR"/ci-service-utils.sh pushd $STORE_DIR . bin/start-hugegraph-store.sh -sleep 10 +wait_for_http_status HugeGraphStore http://127.0.0.1:8520/v1/health \ + "$STORE_DIR"/bin/pid "$STORE_DIR" 90 200,401 popd diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-ci-service-utils.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-ci-service-utils.sh new file mode 100644 index 0000000000..a776591a1e --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-ci-service-utils.sh @@ -0,0 +1,69 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +UTILS="${1:-${SCRIPT_DIR}/ci-service-utils.sh}" +ACTIVE_PID="" + +cleanup() { + if [[ -n "${ACTIVE_PID}" ]]; then + kill "${ACTIVE_PID}" 2>/dev/null || true + wait "${ACTIVE_PID}" 2>/dev/null || true + fi +} +trap cleanup EXIT + +source "${UTILS}" + +if ! declare -F process_is_running >/dev/null || + ! declare -F wait_for_process_exit >/dev/null; then + echo "FAIL: process exit helpers are not available" + exit 1 +fi + +sleep 10 & +ACTIVE_PID=$! +if wait_for_process_exit "${ACTIVE_PID}" 1; then + echo "FAIL: a running process was reported as exited" + exit 1 +fi +kill "${ACTIVE_PID}" 2>/dev/null || true +wait "${ACTIVE_PID}" 2>/dev/null || true +ACTIVE_PID="" + +sleep 1 & +ACTIVE_PID=$! +if ! wait_for_process_exit "${ACTIVE_PID}" 5; then + echo "FAIL: a terminated process was reported as running" + exit 1 +fi +wait "${ACTIVE_PID}" 2>/dev/null || true +ACTIVE_PID="" + +ps() { + echo "Z" +} +if process_is_running "$$"; then + echo "FAIL: a zombie process was reported as running" + exit 1 +fi +unset -f ps + +echo "PASS: process exit helpers handle running, terminated, and zombie states" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh index 796d69c83b..f009d91ece 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java-security-properties.sh @@ -17,11 +17,13 @@ set -euo pipefail -SERVER_ROOT_INPUT="${1:?Usage: $0 PATH_TO_SERVER_DIST}" +SERVER_ROOT_INPUT="${1:?Usage: $0 PATH_TO_SERVER_DIST [SOURCE_ROOT]}" +SOURCE_ROOT_INPUT="${2:-}" SERVER_ROOT=$(cd "$SERVER_ROOT_INPUT" && pwd) SERVER_SCRIPT="${SERVER_ROOT}/bin/hugegraph-server.sh" CONF="${SERVER_ROOT}/conf" SECURITY_PROPERTIES="${CONF}/java-security.properties" +JVM_MODULE_OPTIONS="${SERVER_ROOT}/bin/jvm-module.options" fail() { echo "FAIL: $1" >&2 @@ -43,12 +45,136 @@ assert_no_argument() { fi } +assert_source_consumer() { + local source_file="$1" + local expected="$2" + [[ -f "$source_file" ]] || fail "source consumer is missing: $source_file" + grep -Fq -- "$expected" "$source_file" || + fail "JVM module options consumer is not wired: $source_file" +} + +assert_surefire_arg_lines() { + local pom="$1" + local expected="$2" + local total + local wired + local jacoco_wired + read -r total wired jacoco_wired < <( + awk -v expected="$expected" ' + /maven-surefire-plugin<\/artifactId>/ { + in_surefire = 1 + } + in_surefire && /]*)?>/ { + in_arg_line = 1 + arg_line = "" + } + in_arg_line { + arg_line = arg_line $0 + } + in_arg_line && /<\/argLine>/ { + total++ + if (index(arg_line, expected) != 0) { + wired++ + } + if (index(arg_line, "@{argLine}") != 0) { + jacoco_wired++ + } + in_arg_line = 0 + } + in_surefire && /<\/plugin>/ { + in_surefire = 0 + } + END { + print total + 0, wired + 0, jacoco_wired + 0 + } + ' "$pom" + ) + if [[ "$total" -eq 0 || "$wired" -ne "$total" ]]; then + fail "all Surefire argLine values must use jvm-module.options: $pom" + fi + if [[ "$jacoco_wired" -ne "$total" ]]; then + fail "all Surefire argLine values must preserve @{argLine}: $pom" + fi +} + +assert_no_inline_module_options() { + local pattern + local source_file + pattern="--add-(exports|opens)([[:space:]]+|=)[\"']?java\\.base/|" + pattern="${pattern}--add-modules([[:space:]]+|=)[\"']?jdk\.unsupported" + for source_file in "$@"; do + [[ -f "$source_file" ]] || fail "source consumer is missing: $source_file" + done + if grep -En -- "$pattern" "$@"; then + fail "JVM module options must only be declared in jvm-module.options" + fi +} + if [[ ! -x "$SERVER_SCRIPT" ]]; then fail "server script is not executable: $SERVER_SCRIPT" fi if [[ ! -f "$SECURITY_PROPERTIES" ]]; then fail "security properties file is missing: $SECURITY_PROPERTIES" fi +if [[ ! -f "$JVM_MODULE_OPTIONS" ]]; then + fail "JVM module options file is missing: $JVM_MODULE_OPTIONS" +fi + +assert_argument "--add-exports=java.base/jdk.internal.reflect=ALL-UNNAMED" \ + "$JVM_MODULE_OPTIONS" +assert_argument "--add-modules=jdk.unsupported" "$JVM_MODULE_OPTIONS" +assert_argument "--add-exports=java.base/sun.nio.ch=ALL-UNNAMED" \ + "$JVM_MODULE_OPTIONS" + +if [[ -n "$SOURCE_ROOT_INPUT" ]]; then + if [[ ! -d "$SOURCE_ROOT_INPUT" ]]; then + fail "source root is not a directory: $SOURCE_ROOT_INPUT" + fi + SOURCE_ROOT=$(cd "$SOURCE_ROOT_INPUT" && pwd) + SERVER_DIST_SOURCE="${SOURCE_ROOT}/hugegraph-server/hugegraph-dist" + CLUSTER_SOURCE="${SOURCE_ROOT}/hugegraph-cluster-test/"\ +"hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct" + SERVER_LAUNCHER_SOURCE="${SERVER_DIST_SOURCE}/src/assembly/static/bin/"\ +"hugegraph-server.sh" + INIT_STORE_SOURCE="${SERVER_DIST_SOURCE}/src/assembly/static/bin/init-store.sh" + SUREFIRE_POM="${SOURCE_ROOT}/hugegraph-server/hugegraph-test/pom.xml" + TEST_JVM_MODULE_OPTIONS="${SOURCE_ROOT}/hugegraph-server/hugegraph-test/"\ +"conf/jvm-test-module.options" + COMMONS_POM="${SOURCE_ROOT}/hugegraph-commons/pom.xml" + CLUSTER_WRAPPER="${CLUSTER_SOURCE}/node/ServerNodeWrapper.java" + SERVER_DOCKERFILE="${SOURCE_ROOT}/hugegraph-server/Dockerfile" + HSTORE_DOCKERFILE="${SOURCE_ROOT}/hugegraph-server/Dockerfile-hstore" + SERVER_WORKFLOW="${SOURCE_ROOT}/.github/workflows/server-ci.yml" + DOCKER_WORKFLOW="${SOURCE_ROOT}/.github/workflows/docker-build-ci.yml" + UPGRADE_CONTRACT_SCRIPT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/"\ +"test-java17-upgrade-contracts.sh" + + [[ -x "$UPGRADE_CONTRACT_SCRIPT" ]] || \ + fail "Java 17 upgrade contract script is missing: $UPGRADE_CONTRACT_SCRIPT" + "$UPGRADE_CONTRACT_SCRIPT" "$SERVER_ROOT" "$SOURCE_ROOT" + + assert_source_consumer "$SERVER_LAUNCHER_SOURCE" '@"${JVM_MODULE_OPTIONS}"' + assert_source_consumer "$INIT_STORE_SOURCE" '@"${JVM_MODULE_OPTIONS}"' + assert_surefire_arg_lines "$SUREFIRE_POM" \ + '@${project.basedir}/../hugegraph-dist/src/assembly/static/bin/jvm-module.options' + [[ -f "$TEST_JVM_MODULE_OPTIONS" ]] || \ + fail "JVM test module options file is missing: $TEST_JVM_MODULE_OPTIONS" + assert_argument \ + "--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED" \ + "$TEST_JVM_MODULE_OPTIONS" + assert_argument "--add-opens=java.base/java.lang=ALL-UNNAMED" \ + "$TEST_JVM_MODULE_OPTIONS" + assert_surefire_arg_lines "$SUREFIRE_POM" \ + '@${project.basedir}/conf/jvm-test-module.options' + assert_surefire_arg_lines "$COMMONS_POM" \ + '@${project.parent.basedir}/../hugegraph-server/hugegraph-test/conf/jvm-test-module.options' + assert_source_consumer "$CLUSTER_WRAPPER" \ + '"@" + Paths.get(SERVER_PACKAGE_PATH, BIN_DIR,' + assert_no_inline_module_options \ + "$SERVER_LAUNCHER_SOURCE" "$INIT_STORE_SOURCE" "$SUREFIRE_POM" \ + "$COMMONS_POM" "$CLUSTER_WRAPPER" "$SERVER_DOCKERFILE" \ + "$HSTORE_DOCKERFILE" "$SERVER_WORKFLOW" "$DOCKER_WORKFLOW" +fi if [[ -n "${JAVA_HOME:-}" ]]; then JAVA_BIN="${JAVA_HOME}/bin/java" @@ -409,7 +535,7 @@ if [[ " $* " == *" -version "* ]]; then if [[ -n "${MOCK_JAVA_PREAMBLE:-}" ]]; then echo "${MOCK_JAVA_PREAMBLE}" >&2 fi - echo "openjdk version \"${MOCK_JAVA_VERSION:-11}.0.0\"" >&2 + echo "openjdk version \"${MOCK_JAVA_VERSION:-17}.0.0\"" >&2 exit 0 fi printf '%s\n' "$@" > "$CAPTURE_FILE" @@ -425,6 +551,7 @@ CAPTURE_FILE="$ENABLED_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ assert_argument \ "-Djava.security.properties=${SECURITY_PROPERTIES}" "$ENABLED_CAPTURE" +assert_argument "@${JVM_MODULE_OPTIONS}" "$ENABLED_CAPTURE" assert_no_argument '^-Djava\.security\.manager=' "$ENABLED_CAPTURE" assert_argument \ "org.apache.hugegraph.bootstrap.HugeGraphServerBootstrap" "$ENABLED_CAPTURE" @@ -503,10 +630,10 @@ assert_argument "-Djava.security.manager=allow" "$AGENT_JDK21_CAPTURE" # ... and trip the JDK 24+ security guard when the agent version is high. HIGH_AGENT_PREAMBLE=$'Picked up JAVA_TOOL_OPTIONS: -javaagent:apm-agent.jar\nAPM agent version "24.0.1" is starting' -HIGH_AGENT_CAPTURE="${TEMP_DIR}/agent-preamble-jdk11.args" -HIGH_AGENT_ERROR="${TEMP_DIR}/agent-preamble-jdk11.err" +HIGH_AGENT_CAPTURE="${TEMP_DIR}/agent-preamble-jdk17.args" +HIGH_AGENT_ERROR="${TEMP_DIR}/agent-preamble-jdk17.err" CAPTURE_FILE="$HIGH_AGENT_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ - MOCK_JAVA_VERSION=11 MOCK_JAVA_PREAMBLE="$HIGH_AGENT_PREAMBLE" \ + MOCK_JAVA_VERSION=17 MOCK_JAVA_PREAMBLE="$HIGH_AGENT_PREAMBLE" \ STDOUT_MODE=true "$SERVER_SCRIPT" \ "${CONF}/gremlin-server.yaml" "${CONF}/rest-server.properties" true \ >/dev/null 2>"$HIGH_AGENT_ERROR" @@ -518,6 +645,15 @@ assert_argument \ "org.apache.hugegraph.bootstrap.HugeGraphServerBootstrap" "$HIGH_AGENT_CAPTURE" assert_no_argument '^-Djava\.security\.manager=' "$HIGH_AGENT_CAPTURE" +JDK11_ERROR="${TEMP_DIR}/jdk11.err" +if JAVA_HOME="$MOCK_JAVA_HOME" MOCK_JAVA_VERSION=11 STDOUT_MODE=true \ + "$SERVER_SCRIPT" "${CONF}/gremlin-server.yaml" \ + "${CONF}/rest-server.properties" false >/dev/null 2>"$JDK11_ERROR"; then + fail "launcher accepted a Java 11 runtime" +fi +grep -Fq "version >= 17, current is 11" "${SERVER_ROOT}/logs/hugegraph-server.log" || + fail "launcher did not report the Java 17 minimum" + JDK24_DISABLED_CAPTURE="${TEMP_DIR}/jdk24-disabled.args" CAPTURE_FILE="$JDK24_DISABLED_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ MOCK_JAVA_VERSION=24 STDOUT_MODE=true "$SERVER_SCRIPT" \ diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java17-upgrade-contracts.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java17-upgrade-contracts.sh new file mode 100755 index 0000000000..b371003713 --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-java17-upgrade-contracts.sh @@ -0,0 +1,220 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +SERVER_ROOT_INPUT="${1:?Usage: $0 PATH_TO_SERVER_DIST PATH_TO_SOURCE_ROOT}" +SOURCE_ROOT_INPUT="${2:?Usage: $0 PATH_TO_SERVER_DIST PATH_TO_SOURCE_ROOT}" +SERVER_ROOT=$(cd "$SERVER_ROOT_INPUT" && pwd) +SOURCE_ROOT=$(cd "$SOURCE_ROOT_INPUT" && pwd) + +fail() { + echo "FAIL: $1" >&2 + exit 1 +} + +assert_argument() { + local argument="$1" + local capture="$2" + grep -Fxq -- "$argument" "$capture" || \ + fail "missing JVM argument: $argument" +} + +assert_default_test_is_tolerant() { + local pom="$1" + + python3 - "$pom" <<'PY' +import sys +import xml.etree.ElementTree as ET + +pom = sys.argv[1] +namespace = {"m": "http://maven.apache.org/POM/4.0.0"} +root = ET.parse(pom).getroot() +value = root.find( + "m:properties/m:surefire.failIfNoSpecifiedTests", namespace +) +if value is None or (value.text or "").strip() != "false": + raise SystemExit( + "{}: default-test must tolerate -Dtest misses in reactor modules".format(pom) + ) +PY +} + +assert_supported_java_contract() { + local pom="$1" + + python3 - "$pom" <<'PY' +import sys +import xml.etree.ElementTree as ET + +pom = sys.argv[1] +namespace = {"m": "http://maven.apache.org/POM/4.0.0"} +root = ET.parse(pom).getroot() +properties = root.find("m:properties", namespace) +if properties is None: + raise SystemExit("{}: Maven properties are missing".format(pom)) + +release = properties.find("m:maven.compiler.release", namespace) +if release is None or (release.text or "").strip() != "17": + raise SystemExit("{}: compiler release must remain 17".format(pom)) + +supported_range = properties.find("m:java.supported.version.range", namespace) +if supported_range is None or (supported_range.text or "").strip() != "[17,18)": + raise SystemExit("{}: supported JDK range must be [17,18)".format(pom)) + +expected_reference = "${java.supported.version.range}" +actual_references = [] +for plugin in root.findall("m:build/m:plugins/m:plugin", namespace): + artifact_id = plugin.find("m:artifactId", namespace) + if artifact_id is None or artifact_id.text != "maven-enforcer-plugin": + continue + for rule in plugin.findall( + "m:executions/m:execution/m:configuration/m:rules/m:requireJavaVersion", + namespace, + ): + version = rule.find("m:version", namespace) + if version is not None: + actual_references.append((version.text or "").strip()) + +if actual_references != [expected_reference]: + raise SystemExit( + "{}: requireJavaVersion must consume {} exactly once; found {}".format( + pom, expected_reference, actual_references + ) + ) +PY +} + +assert_surefire_execution_scope() { + local pom="$1" + shift + + python3 - "$pom" "$@" <<'PY' +import sys +import xml.etree.ElementTree as ET + +pom = sys.argv[1] +expected_ids = sys.argv[2:] +namespace = {"m": "http://maven.apache.org/POM/4.0.0"} +root = ET.parse(pom).getroot() + +properties = root.find("m:properties", namespace) +if properties is not None: + module_value = properties.find("m:surefire.failIfNoSpecifiedTests", namespace) + if module_value is not None and (module_value.text or "").strip() == "true": + raise SystemExit( + "{}: leaf-wide failIfNoSpecifiedTests=true breaks default-test".format(pom) + ) + +surefire = None +for plugin in root.findall("m:build/m:plugins/m:plugin", namespace): + artifact_id = plugin.find("m:artifactId", namespace) + if artifact_id is not None and artifact_id.text == "maven-surefire-plugin": + surefire = plugin + break + +if surefire is None: + raise SystemExit("{}: maven-surefire-plugin is missing".format(pom)) + +strict_executions = set() +for execution in surefire.findall("m:executions/m:execution", namespace): + execution_id = execution.find("m:id", namespace) + strict = execution.find("m:configuration/m:failIfNoSpecifiedTests", namespace) + if execution_id is None or strict is None: + continue + if (strict.text or "").strip() == "true": + strict_executions.add(execution_id.text) + +missing = sorted(set(expected_ids) - strict_executions) +if missing: + raise SystemExit( + "{}: named Surefire executions are not strict: {}".format( + pom, ", ".join(missing) + ) + ) +PY +} + +assert_supported_java_contract "${SOURCE_ROOT}/pom.xml" +assert_default_test_is_tolerant "${SOURCE_ROOT}/pom.xml" +assert_surefire_execution_scope \ + "${SOURCE_ROOT}/hugegraph-server/hugegraph-test/pom.xml" \ + core-test unit-test api-test tinkerpop-structure-test tinkerpop-process-test +assert_surefire_execution_scope \ + "${SOURCE_ROOT}/hugegraph-pd/hg-pd-test/pom.xml" \ + pd-client-test pd-core-test pd-common-test pd-rest-test +assert_surefire_execution_scope \ + "${SOURCE_ROOT}/hugegraph-store/hg-store-test/pom.xml" \ + store-client-test store-core-test store-common-test store-rocksdb-test \ + store-server-test store-raftcore-test + +MODULE_OPTIONS="${SERVER_ROOT}/bin/jvm-module.options" +SERVER_SCRIPT="${SERVER_ROOT}/bin/hugegraph-server.sh" +INIT_STORE_SCRIPT="${SERVER_ROOT}/bin/init-store.sh" +UTIL_SCRIPT="${SERVER_ROOT}/bin/util.sh" +CONF_SOURCE="${SERVER_ROOT}/conf" + +for source_file in "$MODULE_OPTIONS" "$SERVER_SCRIPT" \ + "$INIT_STORE_SCRIPT" "$UTIL_SCRIPT"; do + [[ -f "$source_file" ]] || fail "runtime asset is missing: $source_file" +done +[[ -d "$CONF_SOURCE" ]] || fail "server conf is missing: $CONF_SOURCE" + +TEMP_DIR=$(mktemp -d) +trap 'rm -rf "$TEMP_DIR"' EXIT + +DIST_ROOT="${TEMP_DIR}/server" +MOCK_JAVA_HOME="${TEMP_DIR}/mock-java-home" +mkdir -p "${DIST_ROOT}"/{bin,conf,ext,lib,logs,plugins} \ + "${MOCK_JAVA_HOME}/bin" +cp "$MODULE_OPTIONS" "$SERVER_SCRIPT" "$INIT_STORE_SCRIPT" "$UTIL_SCRIPT" \ + "${DIST_ROOT}/bin/" +cp -R "${CONF_SOURCE}/." "${DIST_ROOT}/conf/" + +# Model a full pre-Phase-2 conf/ directory: it has no module argfile. Both +# launchers must get the immutable runtime copy from bin/ instead. +if [[ -e "${DIST_ROOT}/conf/jvm-module.options" ]]; then + fail "legacy conf unexpectedly contains jvm-module.options" +fi + +cat > "${MOCK_JAVA_HOME}/bin/java" <<'MOCK' +#!/bin/bash +for argument in "$@"; do + if [[ "$argument" == "-version" ]]; then + echo 'openjdk version "17.0.0"' >&2 + exit 0 + fi +done +printf '%s\n' "$@" > "${CAPTURE_FILE:?}" +MOCK +chmod +x "${MOCK_JAVA_HOME}/bin/java" "${DIST_ROOT}/bin/hugegraph-server.sh" \ + "${DIST_ROOT}/bin/init-store.sh" + +SERVER_CAPTURE="${TEMP_DIR}/server.args" +CAPTURE_FILE="$SERVER_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ + JAVA_OPTIONS="-Xms512m -Xmx512m" STDOUT_MODE=true \ + "${DIST_ROOT}/bin/hugegraph-server.sh" \ + "${DIST_ROOT}/conf/gremlin-server.yaml" \ + "${DIST_ROOT}/conf/rest-server.properties" false >/dev/null +assert_argument "@${DIST_ROOT}/bin/jvm-module.options" "$SERVER_CAPTURE" + +INIT_STORE_CAPTURE="${TEMP_DIR}/init-store.args" +CAPTURE_FILE="$INIT_STORE_CAPTURE" JAVA_HOME="$MOCK_JAVA_HOME" \ + "${DIST_ROOT}/bin/init-store.sh" >/dev/null +assert_argument "@${DIST_ROOT}/bin/jvm-module.options" "$INIT_STORE_CAPTURE" + +echo "PASS: Java 17 upgrade contracts" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh new file mode 100755 index 0000000000..aa3131543b --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-riscv64-java-runtime.sh @@ -0,0 +1,155 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail + +TRAVIS_DIR=$(cd "$(dirname "$0")" && pwd) +SMOKE_SCRIPT="$TRAVIS_DIR/run-rocksdb-jni-smoke-test.sh" +NATIVE_SMOKE_SCRIPT="$TRAVIS_DIR/run-native-runtime-smoke-test.sh" +WORK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/hugegraph-riscv64-java-test.XXXXXX") +MOCK_BIN="$WORK_DIR/bin" +MOCK_JAVA_HOME="$WORK_DIR/java-home" +SERVER_DIR="$WORK_DIR/server" + +cleanup() { + rm -rf "$WORK_DIR" +} +trap cleanup EXIT + +mkdir -p "$MOCK_BIN" "$MOCK_JAVA_HOME/bin" "$SERVER_DIR/bin" "$SERVER_DIR/lib" + +cat > "$MOCK_BIN/uname" <<'EOF' +#!/bin/bash +echo riscv64 +EOF + +cat > "$MOCK_JAVA_HOME/bin/java" <<'EOF' +#!/bin/bash +set -euo pipefail + +JAVA_VERSION=${MOCK_JAVA_VERSION:-17.0.20} +JAVA_VENDOR=${MOCK_JAVA_VENDOR:-Eclipse Adoptium} + +case "${1:-}" in + -version) + echo "openjdk version \"$JAVA_VERSION\"" >&2 + ;; + -XshowSettings:properties) + echo " java.vm.name = OpenJDK 64-Bit Server VM" >&2 + echo " java.vm.vendor = $JAVA_VENDOR" >&2 + echo " java.vm.version = $JAVA_VERSION+8" >&2 + echo " java.vm.info = mixed mode, sharing" >&2 + echo "openjdk version \"$JAVA_VERSION\"" >&2 + ;; + -cp) + echo "rocksdb-jni-smoke-ok" + ;; + *) + echo "Unexpected Java arguments: $*" >&2 + exit 1 + ;; +esac +EOF + +cat > "$SERVER_DIR/bin/util.sh" <<'EOF' +#!/bin/bash +configure_riscv64_libatomic() { + LD_PRELOAD=libatomic.so.1 +} +EOF + +cat > "$SERVER_DIR/bin/init-store.sh" <<'EOF' +#!/bin/bash +exit 42 +EOF + +chmod +x "$MOCK_BIN/uname" "$MOCK_JAVA_HOME/bin/java" \ + "$SERVER_DIR/bin/init-store.sh" + +run_smoke() { + env -u LD_PRELOAD \ + PATH="$MOCK_BIN:$PATH" \ + JAVA_HOME="$MOCK_JAVA_HOME" \ + "$@" "$SMOKE_SCRIPT" "$SERVER_DIR" +} + +run_native_smoke() { + env -u LD_PRELOAD \ + PATH="$MOCK_BIN:$PATH" \ + JAVA_HOME="$MOCK_JAVA_HOME" \ + "$@" "$NATIVE_SMOKE_SCRIPT" "$SERVER_DIR" +} + +if ! DEFAULT_OUTPUT=$(run_smoke 2>&1); then + echo "$DEFAULT_OUTPUT" >&2 + echo "RISC-V smoke rejected the Java 17 baseline" >&2 + exit 1 +fi +grep -q '^rocksdb-jni-smoke-ok$' <<< "$DEFAULT_OUTPUT" + +set +e +NATIVE_OUTPUT=$(run_native_smoke 2>&1) +NATIVE_STATUS=$? +set -e +if [[ $NATIVE_STATUS -ne 42 ]]; then + echo "$NATIVE_OUTPUT" >&2 + echo "Native smoke did not reach the controlled post-JNI boundary" >&2 + exit 1 +fi +grep -q '^rocksdb-jni-smoke-ok$' <<< "$NATIVE_OUTPUT" + +for JAVA_MAJOR_MISMATCH in 11.0.31 21.0.8; do + if MAJOR_OUTPUT=$(run_smoke \ + "MOCK_JAVA_VERSION=$JAVA_MAJOR_MISMATCH" 2>&1); then + echo "$MAJOR_OUTPUT" >&2 + echo "RISC-V smoke accepted Java $JAVA_MAJOR_MISMATCH" >&2 + exit 1 + fi + grep -Fq "Expected Java 17, got $JAVA_MAJOR_MISMATCH" <<< "$MAJOR_OUTPUT" +done + +EXPECTED_ARGS=( + EXPECTED_JAVA_MAJOR=17 + EXPECTED_RISCV64_JAVA_VERSION=17.0.20 + "EXPECTED_RISCV64_JAVA_VENDOR=Eclipse Adoptium" +) +if ! EXPECTED_OUTPUT=$(run_smoke "${EXPECTED_ARGS[@]}" 2>&1); then + echo "$EXPECTED_OUTPUT" >&2 + echo "RISC-V smoke rejected the configured Temurin 17 runtime" >&2 + exit 1 +fi +grep -q '^rocksdb-jni-smoke-ok$' <<< "$EXPECTED_OUTPUT" + +if VERSION_OUTPUT=$(run_smoke "${EXPECTED_ARGS[@]}" \ + MOCK_JAVA_VERSION=17.0.21 2>&1); then + echo "$VERSION_OUTPUT" >&2 + echo "RISC-V smoke accepted an unexpected Java version" >&2 + exit 1 +fi +grep -Fq 'Expected RISC-V Java 17.0.20, got 17.0.21' <<< "$VERSION_OUTPUT" + +if VENDOR_OUTPUT=$(run_smoke "${EXPECTED_ARGS[@]}" \ + "MOCK_JAVA_VENDOR=Unknown Vendor" 2>&1); then + echo "$VENDOR_OUTPUT" >&2 + echo "RISC-V smoke accepted an unexpected Java vendor" >&2 + exit 1 +fi +grep -Fq \ + 'Expected RISC-V Java vendor Eclipse Adoptium, got Unknown Vendor' \ + <<< "$VENDOR_OUTPUT" + +echo "PASS: RISC-V Java runtime contract" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-signal.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-signal.sh new file mode 100644 index 0000000000..cd3df0814c --- /dev/null +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph-signal.sh @@ -0,0 +1,107 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Verifies that foreground start-hugegraph.sh exits on SIGINT when its +# background server child ignores SIGINT, as a JVM may do in this launch mode. + +set -uo pipefail + +START_SCRIPT="${1:-}" +if [[ -z "$START_SCRIPT" ]]; then + echo "Usage: $0 " + exit 2 +fi + +if [[ ! -f "$START_SCRIPT" ]]; then + echo "ERROR: start script not found: $START_SCRIPT" + exit 2 +fi + +if ! command -v timeout >/dev/null 2>&1; then + echo "SKIP: required tool 'timeout' not found" + exit 77 +fi + +TEST_ROOT=$(mktemp -d) +PID_FILE="$TEST_ROOT/bin/pid" + +cleanup() { + if [[ -s "$PID_FILE" ]]; then + kill -TERM "$(cat "$PID_FILE")" 2>/dev/null || true + fi + rm -rf "$TEST_ROOT" +} +trap cleanup EXIT + +mkdir -p "$TEST_ROOT/bin" "$TEST_ROOT/conf" "$TEST_ROOT/logs" "$TEST_ROOT/scripts" +cp "$START_SCRIPT" "$TEST_ROOT/bin/start-hugegraph.sh" + +cat > "$TEST_ROOT/bin/util.sh" <<'EOF' +read_property() { + local file="$1" + local property="$2" + grep "^${property}=" "$file" | head -n 1 | cut -d '=' -f 2- +} + +check_port() { + : +} +EOF + +cat > "$TEST_ROOT/bin/hugegraph-server.sh" <<'EOF' +#!/bin/bash +trap 'exit 0' TERM +trap '' INT +while true; do + sleep 1 +done +EOF + +cat > "$TEST_ROOT/conf/rest-server.properties" <<'EOF' +gremlinserver.url=http://127.0.0.1:8182 +restserver.url=http://127.0.0.1:8080 +EOF + +chmod +x "$TEST_ROOT/bin/start-hugegraph.sh" "$TEST_ROOT/bin/hugegraph-server.sh" + +export PID_FILE +export START_SCRIPT="$TEST_ROOT/bin/start-hugegraph.sh" + +timeout --signal=TERM --kill-after=5s 10s bash -c ' + target_pid=$$ + ( + while [[ ! -s "$PID_FILE" ]]; do + sleep 0.05 + done + sleep 0.1 + kill -INT "$target_pid" + ) & + exec "$START_SCRIPT" -d false +' +ACTUAL_EXIT=$? + +if [[ "$ACTUAL_EXIT" -ne 130 ]]; then + echo "FAIL: expected exit 130 after SIGINT, got $ACTUAL_EXIT" + exit 1 +fi + +if [[ -s "$PID_FILE" ]] && kill -0 "$(cat "$PID_FILE")" 2>/dev/null; then + echo "FAIL: server child is still running after SIGINT" + exit 1 +fi + +echo "PASS: SIGINT terminates the foreground wrapper and its server child" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh index 9f0bcfaa63..81a7ddd9fa 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/test-start-hugegraph.sh @@ -30,6 +30,9 @@ set -uo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/ci-service-utils.sh" + HUGEGRAPH_ROOT="${1:-$(pwd)}" BIN="$HUGEGRAPH_ROOT/bin" START_SCRIPT="$BIN/start-hugegraph.sh" @@ -417,9 +420,9 @@ else wait_script_exit "$SCRIPT_PID" ACTUAL_EXIT=$? - # If the trap fired correctly, the wrapper's `wait $PID` already reaped Java. - # If wait_script_exit timed out (killer fired), Java may still be running — also a failure. - if ! ps -p "$FG_PID" >/dev/null 2>&1; then + # Allow a bounded shutdown window and treat a zombie as already terminated. + # If wait_script_exit timed out, a live Java process remains a failure. + if wait_for_process_exit "$FG_PID" "$SETTLE_WAIT"; then pass "Java process terminated after SIGTERM sent to wrapper" else fail "Java process still running after SIGTERM — signal not forwarded" diff --git a/hugegraph-server/hugegraph-test/conf/jvm-test-module.options b/hugegraph-server/hugegraph-test/conf/jvm-test-module.options new file mode 100644 index 0000000000..2525398325 --- /dev/null +++ b/hugegraph-server/hugegraph-test/conf/jvm-test-module.options @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Test-only access required by TinkerPop 3.8.1 and its Kryo dependencies on +# Java 17. Keep these permissions out of the Server runtime argument file. +--add-exports=java.base/sun.security.x509=ALL-UNNAMED +--add-exports=java.base/sun.security.action=ALL-UNNAMED +--add-opens=java.base/java.io=ALL-UNNAMED +--add-opens=java.base/java.nio=ALL-UNNAMED +--add-opens=java.base/sun.nio.cs=ALL-UNNAMED +--add-opens=java.base/java.lang=ALL-UNNAMED +--add-opens=java.base/java.lang.invoke=ALL-UNNAMED +--add-opens=java.base/java.lang.reflect=ALL-UNNAMED +--add-opens=java.base/java.util=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent=ALL-UNNAMED +--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED +--add-opens=java.base/java.net=ALL-UNNAMED diff --git a/hugegraph-server/hugegraph-test/pom.xml b/hugegraph-server/hugegraph-test/pom.xml index 259d5a9b9a..4bd8ff69ff 100644 --- a/hugegraph-server/hugegraph-test/pom.xml +++ b/hugegraph-server/hugegraph-test/pom.xml @@ -67,11 +67,11 @@ ${tinkerpop.version} - org.apache.tinkerpop - gremlin-groovy-test - 3.2.11 + com.google.inject + guice + 4.2.3 + provided - org.glassfish.jersey.test-framework.providers jersey-test-framework-provider-grizzly2 @@ -114,11 +114,18 @@ org.apache.maven.plugins maven-surefire-plugin - 2.20 + + + @{argLine} + @${project.basedir}/../hugegraph-dist/src/assembly/static/bin/jvm-module.options + @${project.basedir}/conf/jvm-test-module.options + + core-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -131,6 +138,7 @@ unit-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -143,6 +151,7 @@ api-test + true @@ -161,6 +170,7 @@ tinkerpop-structure-test + true ${basedir}/src/main/java/ ${basedir}/target/classes/ @@ -173,14 +183,22 @@ tinkerpop-process-test + true - -Dbuild.dir=${project.build.directory} + + @{argLine} + @${project.basedir}/../hugegraph-dist/src/assembly/static/bin/jvm-module.options + @${project.basedir}/conf/jvm-test-module.options + -Dbuild.dir=${project.build.directory} + ${basedir}/src/main/java/ ${basedir}/target/classes/ **/ProcessStandardTest.java + **/HugeGraphFeatureTest.java + **/HugeGraphProviderLifecycleTest.java @@ -210,7 +228,6 @@ org.jacoco jacoco-maven-plugin - 0.8.8 org/apache/hugegraph/traversal/algorithm/*.class diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/CypherApiTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/CypherApiTest.java index 3c3e3049f3..3c111bae3a 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/CypherApiTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/CypherApiTest.java @@ -19,8 +19,11 @@ import static org.apache.hugegraph.testutil.Assert.assertContains; +import java.util.List; import java.util.Map; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.util.JsonUtil; import org.junit.Before; import org.junit.Test; @@ -72,13 +75,153 @@ public void testRelationQuery() { this.testCypherQueryAndContains(cypher, "friend"); } - private void testCypherQueryAndContains(String cypher, String containsText) { + @Test + public void testReturnNodeIdAsPrimitiveValue() { + String cypher = "MATCH (n:person) WHERE n.name = 'marko' " + + "RETURN id(n) AS nodeId"; + + String content = this.testCypherQueryAndContains(cypher, "nodeId"); + List data = assertCypherSuccessData(content); + Map row = assertSingleMapRow(data); + Object nodeId = row.get("nodeId"); + + Assert.assertNotNull(nodeId); + assertPrimitiveValue(nodeId); + assertNoHugeGraphIdLeak(content); + } + + @Test + public void testReturnNodeDoesNotLeakInternalIdTypes() { + String cypher = "MATCH (n:person) WHERE n.name = 'marko' RETURN n"; + + String content = this.testCypherQueryAndContains(cypher, "marko"); + assertNoHugeGraphIdLeak(content); + } + + @Test + public void testReturnNestedIdDoesNotLeakInternalIdTypes() { + String cypher = "MATCH (n:person) WHERE n.name = 'marko' " + + "RETURN {nodeId: id(n), values: [id(n), n.name]} " + + "AS payload"; + + String content = this.testCypherQueryAndContains(cypher, "payload"); + List data = assertCypherSuccessData(content); + Map row = assertSingleMapRow(data); + Map payload = assertMapValue(row, "payload"); + List values = assertListValue(payload, "values"); + Object nodeId = payload.get("nodeId"); + + Assert.assertNotNull(nodeId); + assertPrimitiveValue(nodeId); + Assert.assertEquals(2, values.size()); + Assert.assertEquals(nodeId, values.get(0)); + Assert.assertEquals("marko", values.get(1)); + assertNoHugeGraphIdLeak(content); + } + + @Test + public void testReturnRelationIdDoesNotLeakInternalIdTypes() { + String cypher = "MATCH (n:person)-[r:knows]->(friend:person) " + + "WHERE n.name = 'marko' RETURN id(r) AS relationId"; + + String content = this.testCypherQueryAndContains(cypher, "relationId"); + List data = assertCypherSuccessData(content); + Map row = assertSingleMapRow(data); + Object relationId = row.get("relationId"); + + Assert.assertNotNull(relationId); + assertPrimitiveValue(relationId); + assertNoHugeGraphIdLeak(content); + } + + @Test + public void testReturnPathShape() { + String cypher = "MATCH p=(n:person)-[r:knows]->(friend:person) " + + "WHERE n.name = 'marko' RETURN p AS path"; + + String content = this.testCypherQueryAndContains(cypher, "path"); + List data = assertCypherSuccessData(content); + Map row = assertSingleMapRow(data); + List path = assertListValue(row, "path"); + + Assert.assertEquals(3, path.size()); + Map source = assertMapValue(path, 0); + Map relation = assertMapValue(path, 1); + Map target = assertMapValue(path, 2); + + Assert.assertEquals("node", source.get("_type")); + Assert.assertEquals("person", source.get("_label")); + Assert.assertEquals("marko", source.get("name")); + Assert.assertEquals("knows", relation.get("_label")); + Assert.assertEquals("node", target.get("_type")); + Assert.assertEquals("person", target.get("_label")); + Assert.assertEquals("peter", target.get("name")); + assertContains("marko", content); + assertContains("peter", content); + assertNoHugeGraphIdLeak(content); + } + + private String testCypherQueryAndContains(String cypher, + String containsText) { Response r = client().post(PATH, cypher); - this.validStatusAndTextContains(containsText, r); + return this.validStatusAndTextContains(containsText, r); } - private void validStatusAndTextContains(String value, Response r) { + private String validStatusAndTextContains(String value, Response r) { String content = assertResponseStatus(200, r); assertContains(value, content); + return content; + } + + private static void assertNoHugeGraphIdLeak(String content) { + Assert.assertFalse(content.contains("org.apache.hugegraph.backend.id")); + Assert.assertFalse(content.contains("StringId")); + Assert.assertFalse(content.contains("LongId")); + Assert.assertFalse(content.contains("UuidId")); + Assert.assertFalse(content.contains("EdgeId")); + } + + @SuppressWarnings("unchecked") + private static List assertCypherSuccessData(String content) { + Map response = JsonUtil.fromJson(content, Map.class); + Assert.assertTrue(response.containsKey("requestId")); + + Map status = assertMapValue(response, "status"); + Assert.assertEquals(200, ((Number) status.get("code")).intValue()); + Assert.assertEquals("", status.get("message")); + + Map result = assertMapValue(response, "result"); + Assert.assertInstanceOf(List.class, result.get("data")); + Assert.assertInstanceOf(Map.class, result.get("meta")); + return (List) result.get("data"); + } + + private static Map assertSingleMapRow(List data) { + Assert.assertEquals(1, data.size()); + Assert.assertInstanceOf(Map.class, data.get(0)); + return (Map) data.get(0); + } + + private static Map assertMapValue(Map map, String key) { + Assert.assertTrue(map.containsKey(key)); + Assert.assertInstanceOf(Map.class, map.get(key)); + return (Map) map.get(key); + } + + private static Map assertMapValue(List list, int index) { + Assert.assertTrue(list.size() > index); + Assert.assertInstanceOf(Map.class, list.get(index)); + return (Map) list.get(index); + } + + private static List assertListValue(Map map, String key) { + Assert.assertTrue(map.containsKey(key)); + Assert.assertInstanceOf(List.class, map.get(key)); + return (List) map.get(key); + } + + private static void assertPrimitiveValue(Object value) { + Assert.assertFalse(value instanceof Map); + Assert.assertFalse(value instanceof List); } } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/GremlinApiTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/GremlinApiTest.java index 0e537ec432..8f92122aaa 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/GremlinApiTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/GremlinApiTest.java @@ -17,11 +17,9 @@ package org.apache.hugegraph.api; -import java.util.List; import java.util.Map; import org.apache.hugegraph.testutil.Assert; -import org.junit.Assume; import org.junit.Test; import com.google.common.collect.ImmutableMap; @@ -37,22 +35,23 @@ public void testPost() { String body = "{" + "\"gremlin\":\"g.V()\"," + "\"bindings\":{}," + - "\"language\":\"gremlin-groovy\"," + + "\"language\":\"gremlin-lang\"," + "\"aliases\":{\"g\":\"__g_DEFAULT-hugegraph\"}}"; assertResponseStatus(200, client().post(path, body)); } @Test public void testGet() { - Map params = ImmutableMap.of("gremlin", - "this.binding.'DEFAULT-hugegraph'.traversal" + - "().V()"); + Map params = ImmutableMap.of( + "gremlin", "g.V()", + "language", "gremlin-lang", + "aliases.g", "__g_DEFAULT-hugegraph"); Response r = client().get(path, params); Assert.assertEquals(r.readEntity(String.class), 200, r.getStatus()); } @Test - public void testScript() { + public void testRemoteGroovyScriptIsRejected() { String bodyTemplate = "{" + "\"gremlin\":\"%s\"," + "\"bindings\":{}," + @@ -80,19 +79,11 @@ public void testScript() { "'city','235e1153928149578691cf79258e90eb');" + "marko.addEdge('knows',vadas,'date','20160110');"; String body = String.format(bodyTemplate, script); - assertResponseStatus(200, client().post(path, body)); - - String queryV = "g.V()"; - body = String.format(bodyTemplate, queryV); - assertResponseStatus(200, client().post(path, body)); - - String queryE = "g.E()"; - body = String.format(bodyTemplate, queryE); - assertResponseStatus(200, client().post(path, body)); + assertResponseStatus(400, client().post(path, body)); } @Test - public void testClearAndInit() { + public void testRemoteAdminGroovyIsRejected() { String body = "{" + "\"gremlin\":\"graph.backendStoreFeatures()" + " .supportsSharedStorage();\"," + @@ -100,48 +91,11 @@ public void testClearAndInit() { "\"language\":\"gremlin-groovy\"," + "\"aliases\":{\"graph\":\"DEFAULT-hugegraph\"," + "\"g\":\"__g_DEFAULT-hugegraph\"}}"; - String content = assertResponseStatus(200, client().post(path, body)); - Map result = assertJsonContains(content, "result"); - @SuppressWarnings({"unchecked"}) - Object data = ((List) assertMapContains(result, "data")).get(0); - boolean supportsSharedStorage = (boolean) data; - Assume.assumeTrue("Can't clear non-shared-storage backend", - supportsSharedStorage); - - body = "{" + - "\"gremlin\":\"" + - " if (!graph.backendStoreFeatures()" + - " .supportsSharedStorage())" + - " return;" + - " def auth = graph.hugegraph().authManager();" + - " def admin = auth.findUser('admin');" + - " graph.clearBackend();" + - " graph.initBackend();" + - " try {" + - " auth.createUser(admin);" + - " } catch(Exception e) {" + - " }" + - "\"," + - "\"bindings\":{}," + - "\"language\":\"gremlin-groovy\"," + - "\"aliases\":{\"graph\":\"DEFAULT-hugegraph\"," + - "\"g\":\"__g_DEFAULT-hugegraph\"}}"; - - assertResponseStatus(200, client().post(path, body)); - - body = "{" + - "\"gremlin\":\"graph.serverStarted(" + - " GlobalMasterInfo.master('server1'))\"," + - "\"bindings\":{}," + - "\"language\":\"gremlin-groovy\"," + - "\"aliases\":{\"graph\":\"DEFAULT-hugegraph\"," + - "\"g\":\"__g_DEFAULT-hugegraph\"}}"; - assertResponseStatus(200, client().post(path, body)); + assertResponseStatus(400, client().post(path, body)); } - //FIXME: non-pd will not delete admin, but pd mode will @Test - public void testTruncate() { + public void testRemoteTruncateGroovyIsRejected() { String body = "{" + "\"gremlin\":\"" + " def auth = graph.hugegraph().authManager();" @@ -158,7 +112,7 @@ public void testTruncate() { + "\"g\":\"__g_DEFAULT-hugegraph\"}" + "}"; - assertResponseStatus(200, client().post(path, body)); + assertResponseStatus(400, client().post(path, body)); } @Test @@ -194,7 +148,7 @@ public void testSetVertexProperty() { "\"gremlin\":\"g.addV('person').property(T.id, '1')" + ".property('foo', '123').property('bar', '123')\"," + "\"bindings\":{}," + - "\"language\":\"gremlin-groovy\"," + + "\"language\":\"gremlin-lang\"," + "\"aliases\":{\"g\":\"__g_DEFAULT-hugegraph\"}}"; assertResponseStatus(200, client().post(path, body)); @@ -203,7 +157,7 @@ public void testSetVertexProperty() { ".property(single, 'foo', '123')" + ".property(list, 'bar', '123')\"," + "\"bindings\":{}," + - "\"language\":\"gremlin-groovy\"," + + "\"language\":\"gremlin-lang\"," + "\"aliases\":{\"g\":\"__g_DEFAULT-hugegraph\"}}"; assertResponseStatus(200, client().post(path, body)); @@ -212,7 +166,7 @@ public void testSetVertexProperty() { ".property(list, 'foo', '123')" + ".property(list, 'bar', '123')\"," + "\"bindings\":{}," + - "\"language\":\"gremlin-groovy\"," + + "\"language\":\"gremlin-lang\"," + "\"aliases\":{\"g\":\"__g_DEFAULT-hugegraph\"}}"; assertResponseStatus(400, client().post(path, body)); @@ -221,25 +175,19 @@ public void testSetVertexProperty() { ".property(single, 'foo', '123')" + ".property(single, 'bar', '123')\"," + "\"bindings\":{}," + - "\"language\":\"gremlin-groovy\"," + + "\"language\":\"gremlin-lang\"," + "\"aliases\":{\"g\":\"__g_DEFAULT-hugegraph\"}}"; assertResponseStatus(200, client().post(path, body)); } @Test - public void testFileSerialize() { + public void testRemoteFileGroovyIsRejected() { String body = "{" + "\"gremlin\":\"File file = new File('test.text')\"," + "\"bindings\":{}," + "\"language\":\"gremlin-groovy\"," + "\"aliases\":{\"g\":\"__g_DEFAULT-hugegraph\"}}"; - Response r = client().post(path, body); - String content = r.readEntity(String.class); - Assert.assertEquals(content, 200, r.getStatus()); - Map result = assertJsonContains(content, "result"); - @SuppressWarnings({"unchecked", "rawtypes"}) - Map data = ((List) assertMapContains(result, "data")).get(0); - Assert.assertEquals("test.text", data.get("file")); + assertResponseStatus(400, client().post(path, body)); } @Test @@ -247,7 +195,7 @@ public void testVertexOrderByDesc() { String body = "{" + "\"gremlin\":\"g.V().order().by(desc)\"," + "\"bindings\":{}," + - "\"language\":\"gremlin-groovy\"," + + "\"language\":\"gremlin-lang\"," + "\"aliases\":{\"g\":\"__g_DEFAULT-hugegraph\"}}"; Response response = client().post(path, body); assertResponseStatus(200, response); @@ -258,7 +206,7 @@ public void testVertexOrderByAsc() { String body = "{" + "\"gremlin\":\"g.V().order().by(asc)\"," + "\"bindings\":{}," + - "\"language\":\"gremlin-groovy\"," + + "\"language\":\"gremlin-lang\"," + "\"aliases\":{\"g\":\"__g_DEFAULT-hugegraph\"}}"; Response response = client().post(path, body); assertResponseStatus(200, response); @@ -269,7 +217,7 @@ public void testEegeOrderByDesc() { String body = "{" + "\"gremlin\":\"g.E().order().by(desc)\"," + "\"bindings\":{}," + - "\"language\":\"gremlin-groovy\"," + + "\"language\":\"gremlin-lang\"," + "\"aliases\":{\"g\":\"__g_DEFAULT-hugegraph\"}}"; Response response = client().post(path, body); assertResponseStatus(200, response); @@ -280,7 +228,7 @@ public void testEdgeOrderByAsc() { String body = "{" + "\"gremlin\":\"g.E().order().by(asc)\"," + "\"bindings\":{}," + - "\"language\":\"gremlin-groovy\"," + + "\"language\":\"gremlin-lang\"," + "\"aliases\":{\"g\":\"__g_DEFAULT-hugegraph\"}}"; Response response = client().post(path, body); assertResponseStatus(200, response); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/MetricsApiTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/MetricsApiTest.java index d0a1775a16..282e423eb8 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/MetricsApiTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/MetricsApiTest.java @@ -29,6 +29,8 @@ public class MetricsApiTest extends BaseApiTest { private static final String PATH = "/metrics"; private static final String STATISTICS_PATH = PATH + "/statistics"; + private static final String GREMLIN_CHANNEL_METRICS_PREFIX = + "org_apache_tinkerpop_gremlin_server_GremlinServer_channels_"; @Test public void testBaseMetricsAll() { @@ -46,7 +48,13 @@ public void testBaseMetricsAll() { @Test public void testBaseMetricsPromAll() { Response r = client().get(PATH); - assertResponseStatus(200, r); + String result = assertResponseStatus(200, r); + Assert.assertContains(GREMLIN_CHANNEL_METRICS_PREFIX + "paused", + result); + Assert.assertContains(GREMLIN_CHANNEL_METRICS_PREFIX + "total", + result); + Assert.assertContains(GREMLIN_CHANNEL_METRICS_PREFIX + + "write_pauses", result); } @Test diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/cypher/CypherClientTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/cypher/CypherClientTest.java new file mode 100644 index 0000000000..d9d36638d5 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/api/cypher/CypherClientTest.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.api.cypher; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hugegraph.backend.id.IdGenerator; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.unit.BaseUnitTest; +import org.apache.tinkerpop.gremlin.process.traversal.Path; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.MutablePath; +import org.junit.Test; + +public class CypherClientTest extends BaseUnitTest { + + @Test + public void testNormalizeHandlesNullMapAndArrayValues() { + Map value = new LinkedHashMap<>(); + value.put(IdGenerator.of(1L), + new Object[]{IdGenerator.of("marko"), null}); + + Object normalized = CypherClient.normalize(value); + + Assert.assertInstanceOf(Map.class, normalized); + Map map = (Map) normalized; + Assert.assertTrue(map.containsKey(1L)); + Assert.assertInstanceOf(List.class, map.get(1L)); + + List values = (List) map.get(1L); + Assert.assertEquals("marko", values.get(0)); + Assert.assertNull(values.get(1)); + } + + @Test + public void testNormalizeHandlesCyclicReferences() { + Map value = new LinkedHashMap<>(); + value.put("private-value", value); + + Assert.assertThrows(IllegalArgumentException.class, + () -> CypherClient.normalize(value), e -> { + Assert.assertContains("cyclic Cypher result", e.getMessage()); + Assert.assertFalse(e.getMessage().contains("private-value")); + }); + } + + @Test + public void testNormalizePreservesThirtyTwoContainerLayers() { + Object value = "leaf"; + for (int i = 0; i < 32; i++) { + value = new Object[]{value}; + } + + Object normalized = CypherClient.normalize(value); + Object current = normalized; + for (int i = 0; i < 32; i++) { + Assert.assertInstanceOf(List.class, current); + List list = (List) current; + Assert.assertEquals(1, list.size()); + current = list.get(0); + } + + Assert.assertEquals("leaf", current); + } + + @Test + public void testNormalizeRejectsThirtyThirdContainerLayer() { + Object value = "leaf"; + for (int i = 0; i < 33; i++) { + value = new Object[]{value}; + } + Object deeplyNestedValue = value; + + Assert.assertThrows(IllegalArgumentException.class, + () -> CypherClient.normalize(deeplyNestedValue), + e -> Assert.assertContains( + "max normalization depth 32", + e.getMessage())); + } + + @Test + public void testNormalizePreservesPathLabelsAndObjects() { + Path path = MutablePath.make() + .extend(IdGenerator.of("marko"), + Set.of("a")) + .extend(IdGenerator.of("lop"), + Set.of("b", "software")); + + Object normalized = CypherClient.normalize(path); + + Assert.assertInstanceOf(Map.class, normalized); + Map map = (Map) normalized; + Assert.assertTrue(map.containsKey("labels")); + Assert.assertTrue(map.containsKey("objects")); + + Assert.assertInstanceOf(List.class, map.get("labels")); + Assert.assertInstanceOf(List.class, map.get("objects")); + + List labels = (List) map.get("labels"); + List objects = (List) map.get("objects"); + Assert.assertEquals(2, labels.size()); + Assert.assertEquals(2, objects.size()); + + Assert.assertEquals("marko", objects.get(0)); + Assert.assertEquals("lop", objects.get(1)); + List firstLabels = (List) labels.get(0); + List secondLabels = (List) labels.get(1); + Assert.assertTrue(firstLabels.contains("a")); + Assert.assertTrue(secondLabels.contains("b")); + Assert.assertTrue(secondLabels.contains("software")); + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/auth/GremlinLangRequestGuardTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/auth/GremlinLangRequestGuardTest.java new file mode 100644 index 0000000000..74cf0d2a0c --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/auth/GremlinLangRequestGuardTest.java @@ -0,0 +1,716 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.auth; + +import static com.codahale.metrics.MetricRegistry.name; +import static io.netty.handler.codec.http.HttpHeaderNames.ACCEPT; +import static io.netty.handler.codec.http.HttpHeaderNames.CONTENT_TYPE; +import static io.netty.handler.codec.http.HttpMethod.POST; +import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST; +import static io.netty.handler.codec.http.HttpVersion.HTTP_1_1; + +import java.nio.charset.StandardCharsets; +import java.util.Collections; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Function; + +import javax.script.Bindings; + +import org.apache.hugegraph.HugeException; +import org.apache.hugegraph.security.GremlinLangRestrictionStrategy; +import org.apache.hugegraph.security.GremlinLangVerificationStrategy; +import org.apache.hugegraph.testutil.Assert; +import org.apache.tinkerpop.gremlin.groovy.engine.GremlinExecutor; +import org.apache.tinkerpop.gremlin.process.traversal.Bytecode; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.LazyBarrierStrategy; +import org.apache.tinkerpop.gremlin.server.GraphManager; +import org.apache.tinkerpop.gremlin.server.GremlinServer; +import org.apache.tinkerpop.gremlin.server.Settings; +import org.apache.tinkerpop.gremlin.server.util.MetricManager; +import org.apache.tinkerpop.gremlin.util.MessageSerializer; +import org.apache.tinkerpop.gremlin.util.Tokens; +import org.apache.tinkerpop.gremlin.util.function.Lambda; +import org.apache.tinkerpop.gremlin.util.message.ResponseMessage; +import org.apache.tinkerpop.gremlin.util.message.ResponseStatusCode; +import org.apache.tinkerpop.gremlin.util.message.RequestMessage; +import org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1; +import org.apache.tinkerpop.gremlin.util.ser.GraphSONUntypedMessageSerializerV1; +import org.junit.Test; +import org.mockito.Mockito; + +import com.codahale.metrics.Meter; + +import io.netty.buffer.Unpooled; +import io.netty.buffer.UnpooledByteBufAllocator; +import io.netty.channel.embedded.EmbeddedChannel; +import io.netty.handler.codec.http.DefaultFullHttpRequest; +import io.netty.handler.codec.http.FullHttpResponse; + +public class GremlinLangRequestGuardTest { + + private static final String STANDARD_CHANNELIZER = + "org.apache.tinkerpop.gremlin.server.channel." + + "WsAndHttpChannelizer"; + + @Test + public void testRejectsUnprotectedServerChannelizer() { + Settings settings = new Settings(); + settings.channelizer = STANDARD_CHANNELIZER; + settings.gremlinPool = 1; + ExecutorService executor = null; + + try { + executor = ContextGremlinServer.newGremlinExecutorService( + settings); + Assert.fail("Expected an unprotected channelizer error"); + } catch (HugeException e) { + Assert.assertContains("channelizer", e.getMessage()); + } finally { + if (executor != null) { + executor.shutdownNow(); + } + } + } + + @Test + public void testServerCleanupWaitsForAsyncStopCompletion() { + CompletableFuture stop = new CompletableFuture<>(); + AtomicBoolean cleaned = new AtomicBoolean(false); + + CompletableFuture result = ContextGremlinServer.afterStop( + stop, () -> cleaned.set(true)); + + Assert.assertFalse(cleaned.get()); + stop.complete(null); + result.join(); + Assert.assertTrue(cleaned.get()); + } + + @Test + public void testAllowsStandardGremlinLangEval() { + RequestMessage request = eval("gremlin-lang"); + + Assert.assertNull(GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testDefaultsMissingLanguageToGremlinLang() { + RequestMessage request = RequestMessage.build(Tokens.OPS_EVAL) + .addArg(Tokens.ARGS_GREMLIN, + "g.V().count()") + .create(); + + Assert.assertNull(GremlinLangRequestGuard.rejection(request)); + RequestMessage normalized = GremlinLangRequestGuard.normalize(request); + Assert.assertEquals("hugegraph-gremlin-lang", + normalized.getArg(Tokens.ARGS_LANGUAGE)); + Assert.assertEquals(request.getRequestId(), normalized.getRequestId()); + } + + @Test + public void testRejectsExplicitNullLanguage() { + RequestMessage request = RequestMessage.build(Tokens.OPS_EVAL) + .addArg(Tokens.ARGS_GREMLIN, + "g.V().count()") + .addArg(Tokens.ARGS_LANGUAGE, + null) + .create(); + + Assert.assertContains("string", + GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testRejectsNonStringLanguage() { + RequestMessage request = RequestMessage.build(Tokens.OPS_EVAL) + .addArg(Tokens.ARGS_GREMLIN, + "g.V().count()") + .addArg(Tokens.ARGS_LANGUAGE, + 1) + .create(); + + Assert.assertContains("string", + GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testRejectsNonStringEvalPayload() { + RequestMessage request = RequestMessage.build(Tokens.OPS_EVAL) + .addArg(Tokens.ARGS_GREMLIN, + new Bytecode()) + .create(); + + Assert.assertContains("string", + GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testRejectsGroovy() { + RequestMessage request = eval("gremlin-groovy"); + + Assert.assertContains("gremlin-groovy", + GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testRejectsGroovyFromHttpRequest() { + RequestMessage request = RequestMessage.build("") + .addArg(Tokens.ARGS_GREMLIN, + "g.V().count()") + .addArg(Tokens.ARGS_LANGUAGE, + "gremlin-groovy") + .create(); + + Assert.assertContains("gremlin-groovy", + GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testAllowsSessionEval() { + RequestMessage request = RequestMessage.build(Tokens.OPS_EVAL) + .processor("session") + .addArg(Tokens.ARGS_GREMLIN, + "g.V().count()") + .addArg(Tokens.ARGS_SESSION, + "session-id") + .create(); + + Assert.assertNull(GremlinLangRequestGuard.rejection(request)); + RequestMessage normalized = GremlinLangRequestGuard.normalize(request); + Assert.assertEquals("hugegraph-gremlin-lang", + normalized.getArg(Tokens.ARGS_LANGUAGE)); + } + + @Test + public void testRejectsNonStringSessionForEval() { + RequestMessage request = RequestMessage.build(Tokens.OPS_EVAL) + .processor("session") + .addArg(Tokens.ARGS_GREMLIN, + "g.V().count()") + .addArg(Tokens.ARGS_SESSION, 1) + .create(); + + Assert.assertContains("string", + GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testAllowsTraversalBytecodeWithoutLambda() { + RequestMessage request = bytecode("traversal", new Bytecode()); + + Assert.assertNull(GremlinLangRequestGuard.rejection(request)); + Assert.assertSame(request, + GremlinLangRequestGuard.normalize(request)); + } + + @Test + public void testAllowsSessionBytecodeWithoutLambda() { + RequestMessage request = RequestMessage.from( + bytecode("session", new Bytecode())) + .addArg(Tokens.ARGS_SESSION, + "session-id") + .create(); + + Assert.assertNull(GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testRejectsNonStringSessionForBytecode() { + RequestMessage request = RequestMessage.from( + bytecode("session", new Bytecode())) + .addArg(Tokens.ARGS_SESSION, 1) + .create(); + + Assert.assertContains("string", + GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testRejectsBytecodeWithLambda() { + Bytecode bytecode = new Bytecode(); + bytecode.addStep("filter", Lambda.predicate("true")); + RequestMessage request = bytecode("traversal", bytecode); + + Assert.assertContains("Lambda", + GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testAllowsBytecodeThatRemovesQueryStrategy() { + Bytecode bytecode = new Bytecode(); + bytecode.addSource("withoutStrategies", LazyBarrierStrategy.class); + RequestMessage request = bytecode("traversal", bytecode); + + Assert.assertNull(GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testAllowsSessionBytecodeThatRemovesQueryStrategy() { + Bytecode bytecode = new Bytecode(); + bytecode.addSource("withoutStrategies", LazyBarrierStrategy.class); + RequestMessage request = RequestMessage.from( + bytecode("session", bytecode)) + .addArg(Tokens.ARGS_SESSION, + "session-id") + .create(); + + Assert.assertNull(GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testRejectsBytecodeThatRemovesRestrictionStrategy() { + Bytecode bytecode = new Bytecode(); + bytecode.addSource("withoutStrategies", + GremlinLangRestrictionStrategy.class); + RequestMessage request = bytecode("traversal", bytecode); + + Assert.assertContains("GremlinLangRestrictionStrategy", + GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testRejectsSessionBytecodeThatRemovesVerificationStrategy() { + Bytecode bytecode = new Bytecode(); + bytecode.addSource("withoutStrategies", + GremlinLangVerificationStrategy.class); + RequestMessage request = RequestMessage.from( + bytecode("session", bytecode)) + .addArg(Tokens.ARGS_SESSION, + "session-id") + .create(); + + Assert.assertContains("GremlinLangVerificationStrategy", + GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testAllowsLegacySessionClose() { + RequestMessage request = RequestMessage.build(Tokens.OPS_CLOSE) + .processor("session") + .addArg(Tokens.ARGS_SESSION, + "session-id") + .create(); + + Assert.assertNull(GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testRejectsNonStringSessionForClose() { + RequestMessage request = RequestMessage.build(Tokens.OPS_CLOSE) + .processor("session") + .addArg(Tokens.ARGS_SESSION, 1) + .create(); + + Assert.assertContains("string", + GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testWebSocketHandlerRejectsNonStringSession() { + EmbeddedChannel channel = new EmbeddedChannel( + new GremlinLangRequestHandler()); + RequestMessage request = RequestMessage.build(Tokens.OPS_EVAL) + .processor("session") + .addArg(Tokens.ARGS_GREMLIN, + "g.V().count()") + .addArg(Tokens.ARGS_SESSION, 1) + .create(); + + Assert.assertFalse(channel.writeInbound(request)); + ResponseMessage response = channel.readOutbound(); + Assert.assertEquals( + ResponseStatusCode.REQUEST_ERROR_INVALID_REQUEST_ARGUMENTS, + response.getStatus().getCode()); + Assert.assertContains("string", response.getStatus().getMessage()); + channel.finishAndReleaseAll(); + } + + @Test + public void testAllowsAuthenticationOperation() { + RequestMessage request = RequestMessage.build( + Tokens.OPS_AUTHENTICATION).create(); + + Assert.assertNull(GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testRejectsUnknownProcessorAndOperation() { + RequestMessage request = RequestMessage.build("future-operation") + .processor("future-processor") + .create(); + + Assert.assertContains("future-processor", + GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testAllowsCypherProcessor() { + RequestMessage request = RequestMessage.build(Tokens.OPS_EVAL) + .processor("cypher") + .addArg(Tokens.ARGS_GREMLIN, + "MATCH (n) RETURN n") + .create(); + + Assert.assertNull(GremlinLangRequestGuard.rejection(request)); + } + + @Test + public void testRejectsBytecodeWithCypherProcessor() { + RequestMessage request = bytecode("cypher", new Bytecode()); + + Assert.assertContains("text eval", + GremlinLangRequestGuard.rejection(request). + toLowerCase()); + } + + @Test + public void testWebSocketHandlerRejectsGroovyBeforeOpSelector() { + EmbeddedChannel channel = new EmbeddedChannel( + new GremlinLangRequestHandler()); + + Assert.assertFalse(channel.writeInbound(eval("gremlin-groovy"))); + ResponseMessage response = channel.readOutbound(); + Assert.assertEquals(ResponseStatusCode.REQUEST_ERROR_INVALID_REQUEST_ARGUMENTS, + response.getStatus().getCode()); + Assert.assertContains("gremlin-groovy", + response.getStatus().getMessage()); + channel.finishAndReleaseAll(); + } + + @Test + public void testWebSocketHandlerNormalizesGremlinLang() { + EmbeddedChannel channel = new EmbeddedChannel( + new GremlinLangRequestHandler()); + RequestMessage request = eval("gremlin-lang"); + + Assert.assertTrue(channel.writeInbound(request)); + RequestMessage normalized = channel.readInbound(); + Assert.assertEquals("hugegraph-gremlin-lang", + normalized.getArg(Tokens.ARGS_LANGUAGE)); + Assert.assertEquals(request.getRequestId(), normalized.getRequestId()); + channel.finishAndReleaseAll(); + } + + @Test + public void testWebSocketHandlerDefaultsMissingLanguage() { + EmbeddedChannel channel = new EmbeddedChannel( + new GremlinLangRequestHandler()); + RequestMessage request = RequestMessage.build(Tokens.OPS_EVAL) + .addArg(Tokens.ARGS_GREMLIN, + "g.V().count()") + .create(); + + Assert.assertTrue(channel.writeInbound(request)); + RequestMessage normalized = channel.readInbound(); + Assert.assertEquals("hugegraph-gremlin-lang", + normalized.getArg(Tokens.ARGS_LANGUAGE)); + channel.finishAndReleaseAll(); + } + + @Test + public void testHttpHandlerRejectsGroovyBeforeEvaluation() { + String json = "{\"gremlin\":\"g.V().count()\"," + + "\"language\":\"gremlin-groovy\"}"; + + assertHttpBadRequest(json, "gremlin-groovy"); + } + + @Test + public void testHttpHandlerRejectsExplicitNullLanguageBeforeEvaluation() { + assertHttpBadRequest("{\"gremlin\":\"g.V().count()\"," + + "\"language\":null}", + "language argument must be a string"); + } + + @Test + public void testHttpHandlerRejectsNonStringGremlinBeforeCoercion() { + String[] values = {"1", "true", "{}", "[]", "null"}; + + for (String value : values) { + assertHttpBadRequest("{\"gremlin\":" + value + "}", + "gremlin argument for a text eval request " + + "must be a string"); + } + } + + @Test + public void testHttpHandlerRejectsNonStringLanguageBeforeCoercion() { + String[] values = {"1", "true", "{}", "[]", "null"}; + + for (String value : values) { + assertHttpBadRequest("{\"gremlin\":\"g.V()\"," + + "\"language\":" + value + "}", + "language argument must be a string"); + } + } + + @Test + public void testHttpHandlerValidatesJsonContentTypeWithCharset() { + assertHttpBadRequest("{\"gremlin\":\"g.V()\",\"language\":1}", + "application/json; charset=UTF-8", + "language argument must be a string"); + } + + @Test + public void testHttpHandlerRejectionKeepsRequestId() { + UUID requestId = UUID.randomUUID(); + String response = assertHttpBadRequest( + "{\"requestId\":\"" + requestId + "\"," + + "\"gremlin\":\"g.V()\",\"language\":1}", + "language argument must be a string"); + + Assert.assertContains(requestId.toString(), response); + } + + @Test + public void testHttpHandlerParsedRejectionKeepsRequestId() { + UUID requestId = UUID.randomUUID(); + String response = assertHttpBadRequest( + "{\"requestId\":\"" + requestId + "\"," + + "\"gremlin\":\"g.V()\"," + + "\"language\":\"gremlin-groovy\"}", + "gremlin-groovy"); + + Assert.assertContains(requestId.toString(), response); + } + + @Test + public void testHttpHandlerRejectionMarksErrorMetric() { + Meter errorMeter = MetricManager.INSTANCE.getMeter( + name(GremlinServer.class, "errors")); + long count = errorMeter.getCount(); + + assertHttpBadRequest("{\"gremlin\":\"g.V()\"," + + "\"language\":\"gremlin-groovy\"}", + "gremlin-groovy"); + + Assert.assertEquals(count + 1L, errorMeter.getCount()); + } + + @Test + public void testHttpHandlerDefaultsMissingLanguageToGremlinLang() { + GremlinExecutor gremlinExecutor = Mockito.mock( + GremlinExecutor.class); + GraphManager graphManager = Mockito.mock(GraphManager.class); + ExecutorService executor = Executors.newSingleThreadExecutor(); + CompletableFuture pending = new CompletableFuture<>(); + Mockito.when(gremlinExecutor.getExecutorService()) + .thenReturn(executor); + Mockito.when(gremlinExecutor.eval( + Mockito.eq("g.V().count()"), Mockito.anyString(), + Mockito.any(Bindings.class), Mockito.isNull(), + Mockito.>any())) + .thenReturn(pending); + + GremlinLangHttpHandler handler = new GremlinLangHttpHandler( + Collections.singletonMap( + "application/json", + new GraphSONUntypedMessageSerializerV1()), + gremlinExecutor, graphManager, new Settings()); + EmbeddedChannel channel = new EmbeddedChannel(handler); + DefaultFullHttpRequest request = new DefaultFullHttpRequest( + HTTP_1_1, POST, "/gremlin", + Unpooled.copiedBuffer("{\"gremlin\":\"g.V().count()\"}", + StandardCharsets.UTF_8)); + request.headers().set(CONTENT_TYPE, "application/json"); + + try { + Assert.assertFalse(channel.writeInbound(request)); + Mockito.verify(gremlinExecutor).eval( + Mockito.eq("g.V().count()"), + Mockito.eq(GremlinLangRequestGuard.GREMLIN_LANG), + Mockito.any(Bindings.class), Mockito.isNull(), + Mockito.>any()); + } finally { + pending.cancel(true); + executor.shutdownNow(); + channel.finishAndReleaseAll(); + } + } + + @Test + public void testHttpHandlerDefaultsSerializedTextToGremlinLang() + throws Exception { + GraphBinaryMessageSerializerV1 graphBinary = + new GraphBinaryMessageSerializerV1(); + String mimeType = graphBinary.mimeTypesSupported()[0]; + Map> serializers = Map.of( + mimeType, graphBinary, + "application/json", + new GraphSONUntypedMessageSerializerV1()); + GremlinExecutor gremlinExecutor = Mockito.mock( + GremlinExecutor.class); + GraphManager graphManager = Mockito.mock(GraphManager.class); + ExecutorService executor = Executors.newSingleThreadExecutor(); + CompletableFuture pending = new CompletableFuture<>(); + Mockito.when(gremlinExecutor.getExecutorService()) + .thenReturn(executor); + Mockito.when(gremlinExecutor.eval( + Mockito.eq("g.V().count()"), Mockito.anyString(), + Mockito.any(Bindings.class), Mockito.isNull(), + Mockito.>any())) + .thenReturn(pending); + + GremlinLangHttpHandler handler = new GremlinLangHttpHandler( + serializers, gremlinExecutor, graphManager, new Settings()); + EmbeddedChannel channel = new EmbeddedChannel(handler); + RequestMessage gremlinRequest = RequestMessage.build(Tokens.OPS_EVAL) + .addArg( + Tokens.ARGS_GREMLIN, + "g.V().count()") + .create(); + DefaultFullHttpRequest request = new DefaultFullHttpRequest( + HTTP_1_1, POST, "/gremlin", + graphBinary.serializeRequestAsBinary( + gremlinRequest, UnpooledByteBufAllocator.DEFAULT)); + request.headers().set(CONTENT_TYPE, mimeType); + request.headers().set(ACCEPT, "application/json"); + + try { + Assert.assertFalse(channel.writeInbound(request)); + Mockito.verify(gremlinExecutor).eval( + Mockito.eq("g.V().count()"), + Mockito.eq("hugegraph-gremlin-lang"), + Mockito.any(Bindings.class), Mockito.isNull(), + Mockito.>any()); + } finally { + pending.cancel(true); + executor.shutdownNow(); + channel.finishAndReleaseAll(); + } + } + + @Test + public void testHttpHandlerRejectsSerializedBytecode() throws Exception { + GraphBinaryMessageSerializerV1 serializer = + new GraphBinaryMessageSerializerV1(); + String mimeType = serializer.mimeTypesSupported()[0]; + RequestMessage gremlinRequest = bytecode("traversal", + new Bytecode()); + DefaultFullHttpRequest request = new DefaultFullHttpRequest( + HTTP_1_1, POST, "/gremlin", + serializer.serializeRequestAsBinary( + gremlinRequest, UnpooledByteBufAllocator.DEFAULT)); + request.headers().set(CONTENT_TYPE, mimeType); + GremlinLangHttpHandler handler = new GremlinLangHttpHandler( + Collections.singletonMap(mimeType, serializer), + null, null, new Settings()); + EmbeddedChannel channel = new EmbeddedChannel(handler); + + Assert.assertFalse(channel.writeInbound(request)); + FullHttpResponse response = channel.readOutbound(); + Assert.assertEquals(BAD_REQUEST, response.status()); + Assert.assertContains( + "standard WebSocket traversal", + response.content().toString(StandardCharsets.UTF_8)); + response.release(); + channel.finishAndReleaseAll(); + } + + @Test + public void testHttpHandlerRejectsSerializedNonStringLanguage() + throws Exception { + GraphBinaryMessageSerializerV1 serializer = + new GraphBinaryMessageSerializerV1(); + String mimeType = serializer.mimeTypesSupported()[0]; + RequestMessage gremlinRequest = RequestMessage.build(Tokens.OPS_EVAL) + .addArg( + Tokens.ARGS_GREMLIN, + "g.V()") + .addArg( + Tokens.ARGS_LANGUAGE, + 1) + .create(); + DefaultFullHttpRequest request = new DefaultFullHttpRequest( + HTTP_1_1, POST, "/gremlin", + serializer.serializeRequestAsBinary( + gremlinRequest, UnpooledByteBufAllocator.DEFAULT)); + request.headers().set(CONTENT_TYPE, mimeType); + GremlinLangHttpHandler handler = new GremlinLangHttpHandler( + Collections.singletonMap(mimeType, serializer), + null, null, new Settings()); + EmbeddedChannel channel = new EmbeddedChannel(handler); + + Assert.assertFalse(channel.writeInbound(request)); + FullHttpResponse response = channel.readOutbound(); + Assert.assertEquals(BAD_REQUEST, response.status()); + Assert.assertContains( + "must be a string", + response.content().toString(StandardCharsets.UTF_8)); + response.release(); + channel.finishAndReleaseAll(); + } + + @Test + public void testHttpHandlerKeepsMalformedRequestResponse() { + assertHttpBadRequest("{\"gremlin\"", "body could not be parsed"); + } + + private static String assertHttpBadRequest(String json, + String expectedMessage) { + return assertHttpBadRequest(json, "application/json", + expectedMessage); + } + + private static String assertHttpBadRequest(String json, + String contentType, + String expectedMessage) { + GremlinLangHttpHandler handler = new GremlinLangHttpHandler( + Collections.singletonMap( + "application/json", + new GraphSONUntypedMessageSerializerV1()), + null, null, new Settings()); + EmbeddedChannel channel = new EmbeddedChannel(handler); + DefaultFullHttpRequest request = new DefaultFullHttpRequest( + HTTP_1_1, POST, "/gremlin", + Unpooled.copiedBuffer(json, StandardCharsets.UTF_8)); + request.headers().set(CONTENT_TYPE, contentType); + + Assert.assertFalse(channel.writeInbound(request)); + FullHttpResponse response = channel.readOutbound(); + Assert.assertEquals(BAD_REQUEST, response.status()); + String responseBody = response.content().toString( + StandardCharsets.UTF_8); + Assert.assertContains(expectedMessage, responseBody); + response.release(); + channel.finishAndReleaseAll(); + return responseBody; + } + + private static RequestMessage eval(String language) { + return RequestMessage.build(Tokens.OPS_EVAL) + .addArg(Tokens.ARGS_GREMLIN, "g.V().count()") + .addArg(Tokens.ARGS_LANGUAGE, language) + .create(); + } + + private static RequestMessage bytecode(String processor, + Bytecode bytecode) { + return RequestMessage.build(Tokens.OPS_BYTECODE) + .processor(processor) + .addArg(Tokens.ARGS_GREMLIN, bytecode) + .addArg(Tokens.ARGS_ALIASES, + Map.of("g", "__g_hugegraph")) + .create(); + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/CoreTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/CoreTestSuite.java index 230b8d2d06..f78d525018 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/CoreTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/CoreTestSuite.java @@ -41,6 +41,8 @@ VertexCoreTest.class, EdgeCoreTest.class, CountStrategyCoreTest.class, + GremlinLangTextContainsCoreTest.class, + TinkerPop37StepsCoreTest.class, ParentAndSubEdgeCoreTest.class, PropertyCoreTest.VertexPropertyCoreTest.class, PropertyCoreTest.EdgePropertyCoreTest.class, diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/CountStrategyCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/CountStrategyCoreTest.java index f009180508..cf29197cfc 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/CountStrategyCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/CountStrategyCoreTest.java @@ -17,12 +17,25 @@ package org.apache.hugegraph.core; +import java.util.HashSet; +import java.util.Iterator; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.hugegraph.backend.query.Aggregate; +import org.apache.hugegraph.backend.query.Aggregate.AggregateFunc; +import org.apache.hugegraph.backend.query.Query; +import org.apache.hugegraph.backend.tx.GraphTransaction; import org.apache.hugegraph.exception.NoIndexException; import org.apache.hugegraph.schema.SchemaManager; import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.traversal.optimize.HugeCountStep; +import org.apache.hugegraph.traversal.optimize.HugeCountStrategy; import org.apache.hugegraph.traversal.optimize.HugeGraphStep; +import org.apache.hugegraph.type.HugeType; import org.apache.tinkerpop.gremlin.process.traversal.P; import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.TextP; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; @@ -34,6 +47,7 @@ import org.apache.tinkerpop.gremlin.structure.Edge; import org.apache.tinkerpop.gremlin.structure.T; import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.util.CloseableIterator; import org.junit.Test; public class CountStrategyCoreTest extends BaseCoreTest { @@ -101,7 +115,8 @@ private static boolean hasRemainingHasStep(GraphTraversal traversal, if (!(step instanceof HasStep)) { continue; } - HasContainerHolder holder = (HasContainerHolder) step; + HasContainerHolder holder = + (HasContainerHolder) step; for (HasContainer has : holder.getHasContainers()) { if (key.equals(has.getKey())) { return true; @@ -111,6 +126,38 @@ private static boolean hasRemainingHasStep(GraphTraversal traversal, return false; } + private void assertNegatedBooleanPredicate(long expected, + P predicate) { + GraphTraversal traversal = graph().traversal().V() + .has("vp2", + P.not(predicate)) + .count(); + traversal.asAdmin().applyStrategies(); + + Assert.assertTrue(hasRemainingHasStep(traversal, "vp2")); + Assert.assertEquals(expected, traversal.next().longValue()); + } + + private static void assertUncommittedRangeUnsupported( + GraphTraversal traversal) { + Assert.assertThrows(IllegalArgumentException.class, traversal::next, + e -> { + Assert.assertContains("offset/limit", e.getMessage()); + Assert.assertContains("uncommitted records", e.getMessage()); + }); + } + + private static void assertNegatedCountHighRange(long expected, + P predicate) { + GraphTraversal traversal = __.count().is(P.not(predicate)); + HugeCountStrategy.instance().apply(traversal.asAdmin()); + + Step firstStep = traversal.asAdmin().getStartStep(); + Assert.assertInstanceOf(RangeGlobalStep.class, firstStep); + Assert.assertEquals(expected, + ((RangeGlobalStep) firstStep).getHighRange()); + } + private void initTextRangeSchema(boolean withEdge) { SchemaManager schema = graph().schema(); schema.propertyKey("vp4").asText().create(); @@ -134,6 +181,14 @@ private void initConnectiveRangeNoIndexSchema() { .nullableKeys("ep4").link("vl1", "vl1").create(); } + private void initNegatedDoubleSchema() { + SchemaManager schema = graph().schema(); + schema.propertyKey("score").asDouble().create(); + schema.vertexLabel("sample").properties("score").create(); + schema.indexLabel("sampleByScore").onV("sample") + .by("score").range().create(); + } + @Test public void testWhereCountLtNegativeIsAlwaysFalse() { this.initSchema(); @@ -246,6 +301,157 @@ public void testWhereCountNegatedNestedConnectivePredicate() { Assert.assertEquals(1L, count); } + @Test + public void testWhereCountNegatedScalarPredicatesKeepSemantics() { + this.initSchema(); + Vertex source = graph().addVertex(T.label, "person", "name", "source"); + Vertex first = graph().addVertex(T.label, "person", "name", "first"); + Vertex second = graph().addVertex(T.label, "person", "name", "second"); + source.addEdge("knows", first); + source.addEdge("knows", second); + commitTx(); + + long notEqZero = graph().traversal().V(source.id()) + .where(__.out("knows").count() + .is(P.not(P.eq(0L)))) + .count().next(); + long notNeqOne = graph().traversal().V(source.id()) + .where(__.out("knows").count() + .is(P.not(P.neq(1L)))) + .count().next(); + long notLtTwo = graph().traversal().V(source.id()) + .where(__.out("knows").count() + .is(P.not(P.lt(2L)))) + .count().next(); + long notLteOne = graph().traversal().V(source.id()) + .where(__.out("knows").count() + .is(P.not(P.lte(1L)))) + .count().next(); + long notGtOne = graph().traversal().V(source.id()) + .where(__.out("knows").count() + .is(P.not(P.gt(1L)))) + .count().next(); + long notGteThree = graph().traversal().V(source.id()) + .where(__.out("knows").count() + .is(P.not(P.gte(3L)))) + .count().next(); + + Assert.assertEquals(1L, notEqZero); + Assert.assertEquals(0L, notNeqOne); + Assert.assertEquals(1L, notLtTwo); + Assert.assertEquals(1L, notLteOne); + Assert.assertEquals(0L, notGtOne); + Assert.assertEquals(1L, notGteThree); + } + + @Test + public void testNegatedScalarPredicatesUseComplementedHighRange() { + assertNegatedCountHighRange(3L, P.eq(2L)); + assertNegatedCountHighRange(3L, P.neq(2L)); + assertNegatedCountHighRange(2L, P.lt(2L)); + assertNegatedCountHighRange(3L, P.lte(2L)); + assertNegatedCountHighRange(3L, P.gt(2L)); + assertNegatedCountHighRange(2L, P.gte(2L)); + } + + @Test + public void testNegatedTextPredicateStaysLocal() { + this.initTextRangeSchema(false); + graph().schema().indexLabel("vl1ByVp4").onV("vl1") + .by("vp4").secondary().create(); + graph().addVertex(T.label, "vl1", "vp4", "marko", "age", 29); + graph().addVertex(T.label, "vl1", "vp4", "josh", "age", 32); + commitTx(); + + GraphTraversal traversal = graph().traversal().V() + .hasLabel("vl1") + .has("vp4", + TextP.containing("ar") + .negate()) + .count(); + applyAndGetGraphStep(traversal); + + Assert.assertTrue(hasRemainingHasStep(traversal, "vp4")); + Assert.assertEquals(1L, traversal.next().longValue()); + } + + @Test + public void testNegatedNaNPredicatesKeepGremlinSemantics() { + this.initNegatedDoubleSchema(); + graph().addVertex(T.label, "sample", "score", 1.0D); + graph().addVertex(T.label, "sample", "score", Double.NaN); + commitTx(); + + long notLtNaN = graph().traversal().V() + .hasLabel("sample") + .has("score", P.not(P.lt(Double.NaN))) + .count().next(); + long notEqNaN = graph().traversal().V() + .hasLabel("sample") + .has("score", P.not(P.eq(Double.NaN))) + .count().next(); + + Assert.assertEquals(2L, notLtNaN); + Assert.assertEquals(2L, notEqNaN); + } + + @Test + public void testOptimizedGraphCountCanBeResetAndReused() { + this.initSchema(); + this.initGraph(); + + GraphTraversal traversal = graph().traversal().V().count(); + + Assert.assertEquals(3L, traversal.next()); + + traversal.asAdmin().reset(); + + Assert.assertEquals(3L, traversal.next()); + } + + @Test + public void testOptimizedGraphCountEqualityIgnoresExecutionState() { + this.initSchema(); + this.initGraph(); + + GraphTraversal first = graph().traversal().V().count(); + GraphTraversal second = graph().traversal().V().count(); + first.asAdmin().applyStrategies(); + second.asAdmin().applyStrategies(); + + Step firstStep = first.asAdmin().getEndStep(); + Step secondStep = second.asAdmin().getEndStep(); + Assert.assertInstanceOf(HugeCountStep.class, firstStep); + Assert.assertInstanceOf(HugeCountStep.class, secondStep); + Assert.assertEquals(firstStep, secondStep); + + int hashCode = firstStep.hashCode(); + Set> steps = new HashSet<>(); + steps.add(firstStep); + + Assert.assertEquals(3L, first.next()); + + Assert.assertEquals(hashCode, firstStep.hashCode()); + Assert.assertEquals(firstStep, secondStep); + Assert.assertTrue(steps.contains(firstStep)); + } + + @Test + public void testOptimizedGraphCountIncludesUncommittedRecords() { + this.initSchema(); + graph().schema().indexLabel("personByName") + .onV("person").by("name").create(); + + graph().addVertex(T.label, "person", "name", "marko"); + + long count = graph().traversal().V() + .hasLabel("person") + .has("name", "marko") + .count().next(); + + Assert.assertEquals(1L, count); + } + @Test public void testWhereCountFlatAndContradictionEmpty() { this.initSchema(); @@ -338,6 +544,184 @@ public void testWhereCountFlatConnectiveStillGetsRangeBound() { Assert.assertEquals(1L, count); } + @Test + public void testVertexLimitCountRejectsUncommittedAddition() { + this.initSchema(); + graph().addVertex(T.label, "person", "name", "marko"); + + assertUncommittedRangeUnsupported( + graph().traversal().V().limit(1L).count()); + } + + @Test + public void testVertexRangeCountRejectsUncommittedDeletion() { + this.initSchema(); + graph().schema().indexLabel("personByName") + .onV("person").by("name").create(); + this.initGraph(); + Vertex marko = graph().traversal().V() + .hasLabel("person") + .has("name", "marko") + .next(); + marko.remove(); + + assertUncommittedRangeUnsupported( + graph().traversal().V().range(1L, 3L).count()); + } + + @Test + public void testQueryNumberKeepsOriginalAggregate() { + this.initSchema(); + graph().addVertex(T.label, "person", "name", "marko"); + + Query query = new Query(HugeType.VERTEX); + Aggregate aggregate = new Aggregate(AggregateFunc.COUNT, null); + query.aggregate(aggregate); + + Assert.assertEquals(1L, graph().queryNumber(query).longValue()); + Assert.assertSame(aggregate, query.aggregate()); + } + + @Test + public void testUncommittedVertexCountClosesIteratorOnFailure() { + FailingCloseableIterator vertices = + new FailingCloseableIterator<>(); + AtomicBoolean dirty = new AtomicBoolean(true); + GraphTransaction transaction = + this.newFailingCountTransaction(vertices, null, dirty); + + try { + Query query = countQuery(HugeType.VERTEX); + Assert.assertThrows(IllegalStateException.class, + () -> transaction.queryNumber(query)); + Assert.assertTrue(vertices.closed()); + } finally { + dirty.set(false); + transaction.close(); + } + } + + @Test + public void testUncommittedEdgeCountClosesIteratorOnFailure() { + FailingCloseableIterator edges = + new FailingCloseableIterator<>(); + AtomicBoolean dirty = new AtomicBoolean(true); + GraphTransaction transaction = + this.newFailingCountTransaction(null, edges, dirty); + + try { + Query query = countQuery(HugeType.EDGE); + Assert.assertThrows(IllegalStateException.class, + () -> transaction.queryNumber(query)); + Assert.assertTrue(edges.closed()); + } finally { + dirty.set(false); + transaction.close(); + } + } + + @Test + public void testOptimizedEdgeCountIncludesUncommittedRecords() { + this.initSchema(); + graph().schema().indexLabel("personByName") + .onV("person").by("name").create(); + this.initGraph(); + + Vertex josh = graph().traversal().V() + .hasLabel("person").has("name", "josh").next(); + Vertex marko = graph().traversal().V() + .hasLabel("person").has("name", "marko").next(); + josh.addEdge("knows", marko); + + long count = graph().traversal().E().hasLabel("knows").count().next(); + + Assert.assertEquals(2L, count); + } + + private static Query countQuery(HugeType type) { + Query query = new Query(type); + query.aggregate(new Aggregate(AggregateFunc.COUNT, null)); + return query; + } + + private GraphTransaction newFailingCountTransaction( + Iterator vertices, Iterator edges, + AtomicBoolean dirty) { + return new GraphTransaction(params(), params().loadGraphStore()) { + + @Override + public boolean hasUpdate() { + return dirty.get(); + } + + @Override + public Iterator queryVertices(Query query) { + return vertices; + } + + @Override + public Iterator queryEdges(Query query) { + return edges; + } + }; + } + + private static final class FailingCloseableIterator + implements CloseableIterator { + + private boolean closed; + + @Override + public boolean hasNext() { + throw new IllegalStateException("Injected iterator failure"); + } + + @Override + public T next() { + throw new IllegalStateException("Injected iterator failure"); + } + + @Override + public void close() { + this.closed = true; + } + + public boolean closed() { + return this.closed; + } + } + + @Test + public void testEdgeRangeCountRejectsUncommittedAddition() { + this.initSchema(); + graph().schema().indexLabel("personByName") + .onV("person").by("name").create(); + this.initGraph(); + Vertex josh = graph().traversal().V() + .hasLabel("person") + .has("name", "josh") + .next(); + Vertex marko = graph().traversal().V() + .hasLabel("person") + .has("name", "marko") + .next(); + josh.addEdge("knows", marko); + + assertUncommittedRangeUnsupported( + graph().traversal().E().range(1L, 3L).count()); + } + + @Test + public void testEdgeLimitCountRejectsUncommittedDeletion() { + this.initSchema(); + this.initGraph(); + Edge edge = graph().traversal().E().hasLabel("knows").next(); + edge.remove(); + + assertUncommittedRangeUnsupported( + graph().traversal().E().limit(1L).count()); + } + @Test public void testRepeatAfterTextRangeFilterWithEmptyResult() { this.initTextRangeSchema(true); @@ -382,6 +766,33 @@ public void testTextRangeFilterKeepsMixedGraphHasStep() { Assert.assertEquals(direct, viaMatch); } + @Test + public void testTextRangeFilterExtractsIndexedGraphHasContainers() { + this.initTextRangeSchema(false); + graph().schema().indexLabel("vl1ByAge").onV("vl1") + .by("age").secondary().create(); + + graph().addVertex(T.label, "vl1", "vp4", "a", "age", 1); + graph().addVertex(T.label, "vl1", "vp4", "b", "age", 2); + commitTx(); + + GraphTraversal traversal = graph().traversal().V() + .hasLabel("vl1") + .has("vp4", P.lt("")) + .has("age", 1) + .count(); + HugeGraphStep graphStep = applyAndGetGraphStep(traversal); + + Assert.assertEquals(2, graphStep.getHasContainers().size()); + Assert.assertTrue(graphStep.getHasContainers().stream().anyMatch( + has -> T.label.getAccessor().equals(has.getKey()))); + Assert.assertTrue(graphStep.getHasContainers().stream().anyMatch( + has -> "age".equals(has.getKey()))); + Assert.assertTrue(hasRemainingHasStep(traversal, "vp4")); + Assert.assertFalse(hasRemainingHasStep(traversal, "age")); + Assert.assertEquals(0L, traversal.next().longValue()); + } + @Test public void testTextRangeFilterKeepsMixedVertexHasStep() { this.initTextRangeSchema(true); @@ -587,6 +998,53 @@ public void testMatchWithIndexedRangeConditionStillExtractsHas() { Assert.assertEquals(1L, traversal.next()); } + @Test + public void testMatchWithNegatedBooleanPredicateKeepsHas() { + this.initMatchNoIndexSchema(); + graph().schema().indexLabel("vl1ByVp2").onV("vl1") + .by("vp2").secondary().create(); + this.initMatchNoIndexGraph(); + + GraphTraversal traversal = graph().traversal().V() + .has("vp2", + P.not(P.eq(true))) + .match(__.as("s") + .has("vp2") + .as("m")) + .select("m") + .count(); + + HugeGraphStep graphStep = applyAndGetGraphStep(traversal); + Assert.assertEquals(0, graphStep.getHasContainers().size()); + Assert.assertTrue(hasRemainingHasStep(traversal, "vp2")); + Assert.assertEquals(1L, traversal.next()); + } + + @Test + public void testNegatedBooleanComparisonsKeepGremlinSemantics() { + this.initMatchNoIndexSchema(); + graph().schema().indexLabel("vl1ByVp2").onV("vl1") + .by("vp2").secondary().create(); + this.initMatchNoIndexGraph(); + + this.assertNegatedBooleanPredicate(1L, P.eq(true)); + this.assertNegatedBooleanPredicate(1L, P.eq(false)); + this.assertNegatedBooleanPredicate(1L, P.neq(true)); + this.assertNegatedBooleanPredicate(1L, P.neq(false)); + this.assertNegatedBooleanPredicate(1L, P.lt(true)); + this.assertNegatedBooleanPredicate(2L, P.lt(false)); + this.assertNegatedBooleanPredicate(0L, P.lte(true)); + this.assertNegatedBooleanPredicate(1L, P.lte(false)); + this.assertNegatedBooleanPredicate(2L, P.gt(true)); + this.assertNegatedBooleanPredicate(1L, P.gt(false)); + this.assertNegatedBooleanPredicate(1L, P.gte(true)); + this.assertNegatedBooleanPredicate(0L, P.gte(false)); + this.assertNegatedBooleanPredicate(1L, + P.eq(true).and(P.gte(false))); + this.assertNegatedBooleanPredicate(0L, + P.eq(true).or(P.lt(true))); + } + @Test public void testMatchWithNoIndexConditionKeepsExtractingNextHas() { this.initMatchNoIndexSchema(); @@ -721,6 +1179,32 @@ public void testMatchWithIndexedNumericNeqConditionKeepsHas() { Assert.assertEquals(0L, traversal.next()); } + @Test + public void testMatchWithNegatedNumericRangeConditionKeepsHas() { + this.initMatchNoIndexSchema(); + graph().schema().indexLabel("vl0ByVp3").onV("vl0") + .by("vp3").range().create(); + graph().schema().indexLabel("vl1ByVp2").onV("vl1") + .by("vp2").secondary().create(); + this.initMatchNoIndexGraph(); + + GraphTraversal traversal = graph().traversal().V() + .has("vp3", P.not(P.lte( + 4592737712018141718L))) + .has("vp2", true) + .match(__.as("s") + .has("vp2") + .as("m")) + .select("m") + .count(); + + HugeGraphStep graphStep = applyAndGetGraphStep(traversal); + Assert.assertEquals(0, graphStep.getHasContainers().size()); + Assert.assertTrue(hasRemainingHasStep(traversal, "vp3")); + Assert.assertTrue(hasRemainingHasStep(traversal, "vp2")); + Assert.assertEquals(0L, traversal.next()); + } + @Test public void testMatchWithSystemRangeConditionMatchesDirectTraversal() { this.initMatchNoIndexSchema(); diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/GremlinLangTextContainsCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/GremlinLangTextContainsCoreTest.java new file mode 100644 index 0000000000..59e93cd181 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/GremlinLangTextContainsCoreTest.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.core; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import javax.script.Bindings; +import javax.script.SimpleBindings; + +import org.apache.hugegraph.schema.SchemaManager; +import org.apache.hugegraph.security.HugeGraphGremlinLangScriptEngine; +import org.apache.hugegraph.security.HugeGraphGremlinLangScriptEngineFactory; +import org.apache.hugegraph.testutil.Assert; +import org.apache.tinkerpop.gremlin.jsr223.Customizer; +import org.apache.tinkerpop.gremlin.jsr223.GremlinLangPlugin; +import org.apache.tinkerpop.gremlin.jsr223.VariableResolverPlugin; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.T; +import org.junit.Test; + +public class GremlinLangTextContainsCoreTest extends BaseCoreTest { + + @Test + public void testTextContainsUsesHugeGraphSearchIndexSemantics() + throws Exception { + SchemaManager schema = graph().schema(); + schema.propertyKey("name").asText().create(); + schema.propertyKey("description").asText().create(); + schema.vertexLabel("dog") + .properties("name", "description") + .primaryKeys("name") + .create(); + schema.indexLabel("dogByDescription").onV("dog") + .search().by("description").create(); + + graph().addVertex(T.label, "dog", "name", "Bella", + "description", "black hair and eyes"); + graph().addVertex(T.label, "dog", "name", "Daisy", + "description", "yellow hair yellow tail"); + graph().addVertex(T.label, "dog", "name", "Coco", + "description", "yellow hair golden tail"); + this.commitTx(); + + try (GraphTraversalSource g = graph().traversal()) { + HugeGraphGremlinLangScriptEngine engine = engine(g); + Bindings bindings = new SimpleBindings(); + bindings.put("g", g); + try { + bindings.put("keyword", "yellow hair"); + Assert.assertEquals(3L, engine.eval( + "g.V().has('description', " + + "Text.contains(keyword)).count().next()", + bindings)); + + bindings.put("keyword", "black golden"); + Assert.assertEquals(2L, engine.eval( + "g.V().has('description', " + + "Text.contains(keyword)).count().next()", + bindings)); + + bindings.put("keyword", "(hair)"); + Assert.assertEquals(3L, engine.eval( + "g.V().has('description', " + + "Text.contains(keyword)).count().next()", + bindings)); + + bindings.put("keyword", "(black|golden)"); + Assert.assertEquals(2L, engine.eval( + "g.V().has('description', " + + "Text.contains(keyword)).count().next()", + bindings)); + } finally { + engine.clear(); + } + } + } + + private static HugeGraphGremlinLangScriptEngine engine( + GraphTraversalSource g) { + List customizers = new ArrayList<>(); + GremlinLangPlugin cache = GremlinLangPlugin.build() + .cacheEnabled(true) + .caffeine( + "maximumSize=16") + .create(); + VariableResolverPlugin variables = + VariableResolverPlugin.build() + .resolver("DefaultVariableResolver") + .create(); + customizers.addAll(Arrays.asList( + cache.getCustomizers("gremlin-lang").get())); + customizers.addAll(Arrays.asList( + variables.getCustomizers("gremlin-lang").get())); + HugeGraphGremlinLangScriptEngineFactory factory = + new HugeGraphGremlinLangScriptEngineFactory( + customizers.toArray(new Customizer[0])); + HugeGraphGremlinLangScriptEngine engine = factory.getScriptEngine(); + engine.add(g); + return engine; + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/TinkerPop37StepsCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/TinkerPop37StepsCoreTest.java new file mode 100644 index 0000000000..e70fbb9ff1 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/TinkerPop37StepsCoreTest.java @@ -0,0 +1,464 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.core; + +import java.time.OffsetDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.hugegraph.schema.SchemaManager; +import org.apache.hugegraph.testutil.Assert; +import org.apache.tinkerpop.gremlin.process.traversal.DT; +import org.apache.tinkerpop.gremlin.process.traversal.Merge; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.TextP; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.AddPropertyStepContract; +import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.FailStep; +import org.apache.tinkerpop.gremlin.structure.Direction; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.T; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.VertexProperty.Cardinality; +import org.junit.Test; + +public class TinkerPop37StepsCoreTest extends BaseCoreTest { + + @Test + public void testStringManipulationSteps() { + Assert.assertEquals("123", graph().traversal().inject(123) + .asString().next()); + Assert.assertEquals(5, graph().traversal().inject("marko") + .length().next()); + Assert.assertEquals("marko", graph().traversal().inject("MARKO") + .toLower().next()); + Assert.assertEquals("MARKO", graph().traversal().inject("marko") + .toUpper().next()); + Assert.assertEquals("marko", graph().traversal().inject(" marko ") + .trim().next()); + Assert.assertEquals("marko ", graph().traversal().inject(" marko ") + .lTrim().next()); + Assert.assertEquals(" marko", graph().traversal().inject(" marko ") + .rTrim().next()); + Assert.assertEquals("huge-graph", graph().traversal().inject("huge_graph") + .replace("_", "-").next()); + Assert.assertEquals("hugegraph", graph().traversal().inject("huge") + .concat("graph").next()); + Assert.assertEquals("eguh", graph().traversal().inject("huge") + .reverse().next()); + Assert.assertEquals(Arrays.asList("huge", "graph"), + graph().traversal().inject("huge-graph") + .split("-").next()); + Assert.assertEquals("graph", graph().traversal().inject("hugegraph") + .substring(4).next()); + Assert.assertEquals("huge", graph().traversal().inject("hugegraph") + .substring(0, 4).next()); + + Map values = new HashMap<>(); + values.put("name", "marko"); + values.put("age", 29); + Assert.assertEquals("marko is 29 years old", + graph().traversal().inject(values) + .format("%{name} is %{age} years old") + .next()); + } + + @Test + public void testListManipulationSteps() { + List values = Arrays.asList(1, 2); + List other = Arrays.asList(2, 3); + + Assert.assertEquals(Arrays.asList(1, 2, 2, 3), + graph().traversal().inject(values) + .combine(other).next()); + Assert.assertEquals(setOf(1, 2, 3), + asSet(graph().traversal().inject(values) + .merge(other).next())); + Assert.assertEquals(setOf(2), + asSet(graph().traversal().inject(values) + .intersect(other).next())); + Assert.assertEquals(setOf(1), + asSet(graph().traversal().inject(values) + .difference(other).next())); + Assert.assertEquals(setOf(1, 3), + asSet(graph().traversal().inject(values) + .disjunct(other).next())); + Assert.assertEquals(Arrays.asList(Arrays.asList(1, 2), + Arrays.asList(1, 3), + Arrays.asList(2, 2), + Arrays.asList(2, 3)), + graph().traversal().inject(values) + .product(other).next()); + Assert.assertEquals(Arrays.asList(3, 2, 1), + graph().traversal().inject(Arrays.asList(1, 2, 3)) + .reverse().next()); + Assert.assertEquals("huge-graph", + graph().traversal() + .inject(Arrays.asList("huge", "graph")) + .conjoin("-").next()); + Assert.assertEquals(Arrays.asList(1, 2, 3), + graph().traversal() + .inject(Arrays.asList(1, 2, 3)) + .all(P.gt(0)).next()); + Assert.assertEquals(Arrays.asList(1, 2, 3), + graph().traversal() + .inject(Arrays.asList(1, 2, 3)) + .any(P.eq(2)).next()); + } + + @Test + public void testDateManipulationSteps() { + OffsetDateTime start = OffsetDateTime.parse("2023-08-02T00:00:00Z"); + OffsetDateTime expected = OffsetDateTime.parse("2023-08-09T00:00:00Z"); + + OffsetDateTime actual = graph().traversal() + .inject("2023-08-02T00:00:00Z") + .asDate().dateAdd(DT.day, 7).next(); + long milliseconds = graph().traversal() + .inject("2023-08-02T00:00:00Z") + .asDate().dateAdd(DT.day, 7) + .dateDiff(start).next(); + + Assert.assertEquals(expected, actual); + Assert.assertEquals(604800000L, milliseconds); + } + + @Test + public void testAddVertexKeepsPropertiesFoldedByPrimaryKeyStrategy() { + this.initMutationSchema(); + + GraphTraversal traversal = graph().traversal() + .addV("person") + .property( + Cardinality.single, + "name", + __.constant("marko")) + .property( + Cardinality.single, + "status", "active"); + Assert.assertTrue(traversal.asAdmin().getSteps().stream().anyMatch( + step -> step instanceof AddPropertyStepContract)); + + Vertex vertex = traversal.next(); + commitTx(); + + Vertex stored = graph().traversal().V(vertex.id()).next(); + Assert.assertEquals("marko", stored.value("name")); + Assert.assertEquals("active", stored.value("status")); + } + + @Test + public void testMergeVertexWithHugeGraphIds() { + this.initMutationSchema(); + Map search = map(T.label, "person", + "name", "marko"); + + Vertex created = graph().traversal().mergeV(search) + .option(Merge.onCreate, + map("status", "created")) + .next(); + commitTx(); + Vertex matched = graph().traversal().mergeV(search) + .option(Merge.onMatch, + map("status", "matched")) + .next(); + commitTx(); + + Assert.assertEquals(created.id(), matched.id()); + Assert.assertEquals("matched", matched.value("status")); + Assert.assertEquals(1L, graph().traversal().V() + .hasLabel("person") + .has("name", "marko") + .count().next()); + } + + @Test + public void testMergeEdgeWithHugeGraphIds() { + this.initMutationSchema(); + Vertex marko = graph().addVertex(T.label, "person", + "name", "marko"); + Vertex vadas = graph().addVertex(T.label, "person", + "name", "vadas"); + commitTx(); + Map search = map(T.label, "knows", + Direction.OUT, marko.id(), + Direction.IN, vadas.id()); + + Edge created = graph().traversal().mergeE(search) + .option(Merge.onCreate, + map("status", "created", + "weight", 0.5D)) + .next(); + commitTx(); + Edge matched = graph().traversal().mergeE(search) + .option(Merge.onMatch, + map("status", "matched")) + .next(); + commitTx(); + + Assert.assertEquals(created.id(), matched.id()); + Assert.assertEquals("matched", matched.value("status")); + Assert.assertEquals(1L, graph().traversal().E() + .hasLabel("knows").count().next()); + } + + @Test + public void testMergeOnCreateValidation() { + this.initMutationSchema(); + Map search = map(T.label, "person", + "name", "marko"); + Map invalid = map(T.label, "person", + "name", "vadas"); + + Assert.assertThrows(IllegalArgumentException.class, () -> { + graph().traversal().mergeV(search) + .option(Merge.onCreate, invalid); + }); + } + + @Test + public void testPropertyMapAndMidTraversalE() { + this.initMutationSchema(); + Vertex marko = graph().addVertex(T.label, "person", + "name", "marko"); + Vertex vadas = graph().addVertex(T.label, "person", + "name", "vadas"); + Edge edge = marko.addEdge("knows", vadas); + commitTx(); + + graph().traversal().V(marko.id()) + .property(map("status", "active")) + .iterate(); + commitTx(); + + Assert.assertEquals("active", graph().traversal().V(marko.id()) + .values("status").next()); + Assert.assertEquals(edge.id(), graph().traversal().inject(1) + .E(edge.id()).next().id()); + } + + @Test + public void testUnproductiveByFiltersMissingGroupKey() { + this.initTextData(); + graph().traversal().V() + .hasLabel("person") + .has("name", "marko") + .property("status", "active") + .iterate(); + commitTx(); + + Map grouped = graph().traversal().V() + .group() + .by("status") + .by("name") + .next(); + + Assert.assertEquals(1, grouped.size()); + Assert.assertEquals(Collections.singletonList("marko"), + grouped.get("active")); + Assert.assertFalse(grouped.containsKey(null)); + } + + @Test + public void testUnproductiveByOmitsProjectKey() { + this.initTextData(); + + Map projected = graph().traversal().V() + .hasLabel("person") + .has("name", "vadas") + .project("name", "status") + .by("name") + .by("status") + .next(); + + Assert.assertEquals("vadas", projected.get("name")); + Assert.assertFalse(projected.containsKey("status")); + } + + @Test + public void testMissingByValueCanUseExplicitFallback() { + this.initTextData(); + graph().traversal().V() + .hasLabel("person") + .has("name", "marko") + .property("status", "active") + .iterate(); + commitTx(); + + Map grouped = graph().traversal().V() + .group() + .by(__.coalesce( + __.values("status"), + __.constant("missing"))) + .by("name") + .next(); + + Assert.assertEquals(Collections.singletonList("marko"), + grouped.get("active")); + Assert.assertEquals(setOf("lop", "vadas"), + asSet(grouped.get("missing"))); + } + + @Test + public void testFailStep() { + Assert.assertThrows(FailStep.FailException.class, () -> { + graph().traversal().inject(1).fail("expected failure").iterate(); + }); + } + + @Test + public void testTextPContaining() { + this.initTextData(); + Assert.assertEquals(Arrays.asList("marko"), + this.names(TextP.containing("ark"))); + } + + @Test + public void testTextPStartingWith() { + this.initTextData(); + Assert.assertEquals(Arrays.asList("marko"), + this.names(TextP.startingWith("mar"))); + } + + @Test + public void testTextPEndingWith() { + this.initTextData(); + Assert.assertEquals(Arrays.asList("vadas"), + this.names(TextP.endingWith("das"))); + } + + @Test + public void testTextPRegex() { + this.initTextData(); + Assert.assertEquals(Arrays.asList("marko"), + this.names(TextP.regex("^mar"))); + } + + @Test + public void testTextPNegations() { + this.initTextData(); + Assert.assertEquals(Arrays.asList("lop", "vadas"), + this.names(TextP.notContaining("ar"))); + Assert.assertEquals(Arrays.asList("lop", "vadas"), + this.names(TextP.notStartingWith("mar"))); + Assert.assertEquals(Arrays.asList("lop", "marko"), + this.names(TextP.notEndingWith("das"))); + Assert.assertEquals(Arrays.asList("lop", "vadas"), + this.names(TextP.notRegex("^mar"))); + } + + @Test + public void testTextPWithLocalFilter() { + this.initTextData(); + + Assert.assertEquals(Arrays.asList("marko"), + this.namesWithLocalFilter( + TextP.containing("ark"))); + Assert.assertEquals(Arrays.asList("marko"), + this.namesWithLocalFilter( + TextP.startingWith("mar"))); + Assert.assertEquals(Arrays.asList("vadas"), + this.namesWithLocalFilter( + TextP.endingWith("das"))); + Assert.assertEquals(Arrays.asList("marko"), + this.namesWithLocalFilter(TextP.regex("^mar"))); + Assert.assertEquals(Arrays.asList("lop", "vadas"), + this.namesWithLocalFilter( + TextP.notContaining("ar"))); + Assert.assertEquals(Arrays.asList("lop", "vadas"), + this.namesWithLocalFilter( + TextP.notStartingWith("mar"))); + Assert.assertEquals(Arrays.asList("lop", "marko"), + this.namesWithLocalFilter( + TextP.notEndingWith("das"))); + Assert.assertEquals(Arrays.asList("lop", "vadas"), + this.namesWithLocalFilter( + TextP.notRegex("^mar"))); + } + + private void initMutationSchema() { + SchemaManager schema = graph().schema(); + schema.propertyKey("name").asText().create(); + schema.propertyKey("status").asText().create(); + schema.propertyKey("weight").asDouble().create(); + schema.vertexLabel("person") + .properties("name", "status") + .primaryKeys("name") + .nullableKeys("status") + .create(); + schema.edgeLabel("knows") + .link("person", "person") + .properties("status", "weight") + .nullableKeys("status", "weight") + .create(); + } + + private void initTextData() { + this.initMutationSchema(); + graph().addVertex(T.label, "person", "name", "marko"); + graph().addVertex(T.label, "person", "name", "vadas"); + graph().addVertex(T.label, "person", "name", "lop"); + commitTx(); + } + + private List names(TextP predicate) { + return graph().traversal().V() + .hasLabel("person") + .has("name", predicate) + .values("name") + .order() + .toList(); + } + + private List namesWithLocalFilter(TextP predicate) { + return graph().traversal().V() + .hasLabel("person") + .filter(__.values("name").is(predicate)) + .values("name") + .order() + .toList(); + } + + private static Map map(Object... keyValues) { + Map result = new HashMap<>(); + for (int i = 0; i < keyValues.length; i += 2) { + result.put(keyValues[i], keyValues[i + 1]); + } + return result; + } + + private static Set setOf(Object... values) { + return new HashSet<>(Arrays.asList(values)); + } + + private static Set asSet(Object values) { + Assert.assertInstanceOf(Iterable.class, values); + List list = new ArrayList<>(); + for (Object value : (Iterable) values) { + list.add(value); + } + return new HashSet<>(list); + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java index 6118deee2b..78f1419387 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/core/VertexCoreTest.java @@ -3270,6 +3270,24 @@ public void testQueryByLabel() { SplicingIdGenerator.splicing(bookId, "java-5"))); } + @Test + public void testQueryByNullKeyAndLabel() { + HugeGraph graph = graph(); + init10Vertices(); + + Assert.assertFalse(graph.traversal().V() + .has((String) null, "test-null-key") + .hasNext()); + Assert.assertFalse(graph.traversal().V() + .hasLabel((String) null) + .hasNext()); + + List vertices = graph.traversal().V() + .hasLabel(null, "book") + .toList(); + Assert.assertEquals(5, vertices.size()); + } + @Test public void testQueryByLabelWithLimit() { HugeGraph graph = graph(); @@ -3994,6 +4012,35 @@ public void testQueryByIntProperty() { }); } + @Test + public void testQueryByNegatedNullPredicate() { + HugeGraph graph = graph(); + + graph.addVertex(T.label, "person", "name", "marko", + "city", "Beijing", "age", 29); + graph.addVertex(T.label, "person", "name", "vadas", + "city", "Beijing", "age", 27); + graph.addVertex(T.label, "person", "name", "lop", + "city", "Shanghai"); + this.commitTx(); + + List negatedNull = graph.traversal().V() + .hasLabel("person") + .has("age", P.not(P.eq(null))) + .values("name") + .toList(); + List notEqualNull = graph.traversal().V() + .hasLabel("person") + .has("age", P.neq(null)) + .values("name") + .toList(); + + Set expected = ImmutableSet.of("marko", "vadas"); + Assert.assertEquals(expected, ImmutableSet.copyOf(negatedNull)); + Assert.assertEquals(expected, ImmutableSet.copyOf(notEqualNull)); + Assert.assertEquals(notEqualNull.size(), negatedNull.size()); + } + @Test public void testQueryByLongProperty() { HugeGraph graph = graph(); @@ -4857,14 +4904,10 @@ public void testQueryWithMultiLayerConditions() { .and(P.lt(29).or(P.eq(35)).or(P.gt(45))) ).values("name").toList(); - // There is duplicate results with OR condition - Assert.assertEquals(5, vertices.size()); - Set names = ImmutableSet.of("Hebe", "James", "Tom Cat", "Lisa"); - for (Object name : vertices) { - Assert.assertTrue(names.contains(name)); - } + Assert.assertEquals(names.size(), vertices.size()); + Assert.assertEquals(names, ImmutableSet.copyOf(vertices)); } @Test diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphFeatureTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphFeatureTest.java new file mode 100644 index 0000000000..d3f96d1307 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphFeatureTest.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.tinkerpop; + +import org.apache.hugegraph.dist.RegisterUtil; +import org.apache.tinkerpop.gremlin.features.AbstractGuiceFactory; +import org.apache.tinkerpop.gremlin.features.World; +import org.junit.runner.RunWith; + +import com.google.inject.AbstractModule; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Stage; + +import io.cucumber.guice.CucumberModules; +import io.cucumber.junit.Cucumber; +import io.cucumber.junit.CucumberOptions; + +@RunWith(Cucumber.class) +@CucumberOptions( + tags = HugeGraphFeatureTest.TAGS, + name = HugeGraphFeatureTest.NAMES, + glue = {"org.apache.tinkerpop.gremlin.features"}, + objectFactory = HugeGraphFeatureTest.HugeGraphGuiceFactory.class, + features = { + "classpath:/org/apache/tinkerpop/gremlin/test/features" + }, + plugin = { + "progress", + "junit:target/cucumber-tp38.xml", + "org.apache.hugegraph.tinkerpop." + + "HugeGraphScenarioCountPlugin" + }) +public class HugeGraphFeatureTest { + + public static final String NAMES = + "^g_(?!mergeEXlabel_knows_out_marko_in_vadas_weight_05X_" + + "exists$)(?!V_hasXperson_name_marko_X_mergeEXlabel_knowsX_" + + "optionXonCreate_created_YX_optionXonMatch_created_NX_" + + "exists_updated$)" + + // HugeGraph rejects limit queries while graph initializer data + // is still uncommitted, before mergeE can validate its endpoints. + "(?!V_limitX1X_mergeEXnullvarX$).*"; + + public static final String TAGS = + "(@StepAsString or @StepConcat or @StepFormat or " + + "@StepLength or @StepSplit or @StepSubstring or " + + "@StepReplace or @StepReverse or @StepToLower or " + + "@StepToUpper or @StepTrim or @StepLTrim or @StepRTrim or " + + "@StepCombine or @StepMerge or @StepIntersect or " + + "@StepDifference or @StepDisjunct or @StepConjoin or " + + "@StepProduct or @StepAll or @StepAny or @StepAsDate or " + + "@StepDateAdd or @StepDateDiff or @StepMergeV or " + + "@StepMergeE or @StepFail) and " + + "not @RemoteOnly and not @GraphComputerOnly and " + + "not @AllowNullPropertyValues and not @MetaProperties and " + + "not @MultiProperties and " + + "not @UserSuppliedVertexIds and not @UserSuppliedEdgeIds and " + + "not @UserSuppliedVertexPropertyIds and " + + "not @InsertionOrderingRequired"; + + public static class HugeGraphGuiceFactory extends AbstractGuiceFactory { + + public HugeGraphGuiceFactory() { + super(createInjector()); + } + + private static Injector createInjector() { + RegisterUtil.registerBackends(); + return Guice.createInjector(Stage.PRODUCTION, + CucumberModules.createScenarioModule(), + new ServiceModule()); + } + } + + public static final class ServiceModule extends AbstractModule { + + @Override + protected void configure() { + bind(World.class).to(HugeGraphWorld.class); + } + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphProviderContext.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphProviderContext.java new file mode 100644 index 0000000000..794d7fc5e2 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphProviderContext.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.tinkerpop; + +import java.io.IOException; + +final class HugeGraphProviderContext { + + private ProcessTestGraphProvider provider; + + synchronized ProcessTestGraphProvider provider() { + if (this.provider == null) { + try { + this.provider = new ProcessTestGraphProvider(); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to create graph provider", e); + } + } + return this.provider; + } + + synchronized void clear() { + if (this.provider == null) { + return; + } + + ProcessTestGraphProvider provider = this.provider; + this.provider = null; + provider.clear(); + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphProviderLifecycleTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphProviderLifecycleTest.java new file mode 100644 index 0000000000..c5ed11e59f --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphProviderLifecycleTest.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.tinkerpop; + +import java.util.Map; + +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.configuration2.MapConfiguration; +import org.apache.hugegraph.dist.RegisterUtil; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.testutil.Utils; +import org.junit.Assume; +import org.junit.Test; + +public class HugeGraphProviderLifecycleTest { + + @Test + public void testProviderContextLifecycleWithMemoryBackend() + throws Exception { + Assume.assumeTrue("memory".equals( + Utils.getConf().getString("backend"))); + RegisterUtil.registerBackends(); + HugeGraphProviderContext context = new HugeGraphProviderContext(); + ProcessTestGraphProvider provider = context.provider(); + TestGraph graph = null; + try { + Assert.assertSame(provider, context.provider()); + + Map config = provider.getBaseConfiguration( + "provider_context", this.getClass(), + "testProviderContextLifecycleWithMemoryBackend", null); + Configuration configuration = new MapConfiguration(config); + graph = (TestGraph) provider.openTestGraph(configuration); + + Assert.assertEquals("memory", graph.hugegraph().backend()); + Assert.assertFalse(graph.closed()); + + provider.clear(graph, configuration); + Assert.assertFalse(graph.closed()); + + context.clear(); + Assert.assertTrue(graph.closed()); + + context.clear(); + Assert.assertNotSame(provider, context.provider()); + } finally { + context.clear(); + } + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphScenarioCountPlugin.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphScenarioCountPlugin.java new file mode 100644 index 0000000000..4503f60b31 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphScenarioCountPlugin.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.tinkerpop; + +import java.util.concurrent.atomic.AtomicInteger; + +import io.cucumber.plugin.ConcurrentEventListener; +import io.cucumber.plugin.event.EventPublisher; +import io.cucumber.plugin.event.TestCaseStarted; +import io.cucumber.plugin.event.TestRunFinished; + +public final class HugeGraphScenarioCountPlugin + implements ConcurrentEventListener { + + /* + * This is the exact number selected by TAGS and NAMES for TinkerPop 3.8.1. + * Update it together with an intentional filter or TinkerPop change. + */ + private static final int EXPECTED_SCENARIOS = 361; + + private final AtomicInteger scenarioCount = new AtomicInteger(); + + @Override + public void setEventPublisher(EventPublisher publisher) { + publisher.registerHandlerFor(TestCaseStarted.class, + event -> this.scenarioCount.incrementAndGet()); + publisher.registerHandlerFor(TestRunFinished.class, + event -> this.finishRun()); + } + + private void finishRun() { + try { + assertScenariosExecuted(this.scenarioCount.get()); + } finally { + HugeGraphWorld.clearProvider(); + } + } + + static void assertScenariosExecuted(int scenarioCount) { + if (scenarioCount != EXPECTED_SCENARIOS) { + throw new AssertionError( + scenarioCount + " TinkerPop Gherkin scenarios were " + + "executed, expected exactly " + EXPECTED_SCENARIOS + + " (check the TAGS/NAMES filters and update the expected " + + "count for intentional changes)"); + } + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphTestInfrastructureTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphTestInfrastructureTest.java new file mode 100644 index 0000000000..efe2d5fba2 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphTestInfrastructureTest.java @@ -0,0 +1,150 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.tinkerpop; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import org.apache.hugegraph.HugeGraph; +import org.apache.hugegraph.schema.PropertyKey; +import org.apache.hugegraph.schema.SchemaManager; +import org.apache.hugegraph.schema.VertexLabel; +import org.apache.hugegraph.testutil.Assert; +import org.junit.Test; +import org.mockito.Mockito; + +public class HugeGraphTestInfrastructureTest { + + @Test + public void testProviderContextLifecycle() { + HugeGraphProviderContext context = new HugeGraphProviderContext(); + ProcessTestGraphProvider provider = context.provider(); + try { + Assert.assertSame(provider, context.provider()); + + context.clear(); + context.clear(); + + Assert.assertNotSame(provider, context.provider()); + } finally { + context.clear(); + } + } + + @Test + public void testExactScenarioCount() { + HugeGraphScenarioCountPlugin.assertScenariosExecuted(361); + + Assert.assertThrows(AssertionError.class, () -> { + HugeGraphScenarioCountPlugin.assertScenariosExecuted(360); + }, e -> { + Assert.assertContains("expected exactly 361", e.getMessage()); + }); + Assert.assertThrows(AssertionError.class, () -> { + HugeGraphScenarioCountPlugin.assertScenariosExecuted(362); + }, e -> { + Assert.assertContains("expected exactly 361", e.getMessage()); + }); + } + + @Test + public void testScenarioNameFilterExcludesUnsupportedLimitMerge() { + Assert.assertFalse("g_V_limitX1X_mergeEXnullvarX" + .matches(HugeGraphFeatureTest.NAMES)); + Assert.assertTrue("g_mergeEXnullvarX" + .matches(HugeGraphFeatureTest.NAMES)); + } + + @Test + public void testHStoreCleanupTruncatesDataBeforeClearingSchema() { + HugeGraph graph = Mockito.mock(HugeGraph.class); + SchemaManager schema = Mockito.mock(SchemaManager.class); + PropertyKey propertyKey = Mockito.mock(PropertyKey.class); + Mockito.when(graph.schema()).thenReturn(schema); + Mockito.when(schema.getPropertyKeys()) + .thenReturn(Collections.singletonList(propertyKey)); + Mockito.when(graph.backend()).thenReturn("hstore"); + + CleanupTestGraph testGraph = new CleanupTestGraph(graph); + testGraph.clearAll(""); + + Assert.assertTrue(testGraph.backendTruncated); + Assert.assertTrue(testGraph.schemaCleared); + Assert.assertEquals(Arrays.asList("truncate", "schema"), + testGraph.cleanupSteps); + } + + @Test + public void testHStoreLoadCleanupDoesNotTruncateBackend() { + HugeGraph graph = Mockito.mock(HugeGraph.class); + Mockito.when(graph.backend()).thenReturn("hstore"); + + CleanupTestGraph testGraph = new CleanupTestGraph(graph); + testGraph.clearForLoad(); + + Assert.assertFalse(testGraph.backendTruncated); + Assert.assertTrue(testGraph.schemaCleared); + Assert.assertEquals(Collections.singletonList("schema"), + testGraph.cleanupSteps); + } + + @Test + public void testHStoreCleanupDoesNotSkipSchemaWithoutPropertyKeys() { + HugeGraph graph = Mockito.mock(HugeGraph.class); + SchemaManager schema = Mockito.mock(SchemaManager.class); + VertexLabel vertexLabel = Mockito.mock(VertexLabel.class); + Mockito.when(graph.schema()).thenReturn(schema); + Mockito.when(schema.getPropertyKeys()) + .thenReturn(Collections.emptyList()); + Mockito.when(schema.getVertexLabels()) + .thenReturn(Collections.singletonList(vertexLabel)); + Mockito.when(graph.backend()).thenReturn("hstore"); + + CleanupTestGraph testGraph = new CleanupTestGraph(graph); + testGraph.clearAll(""); + + Assert.assertTrue(testGraph.backendTruncated); + Assert.assertTrue(testGraph.schemaCleared); + } + + private static class CleanupTestGraph extends TestGraph { + + private boolean backendTruncated; + private boolean schemaCleared; + private final List cleanupSteps; + + private CleanupTestGraph(HugeGraph graph) { + super(graph); + this.cleanupSteps = new ArrayList<>(); + } + + @Override + protected void truncateBackend() { + this.backendTruncated = true; + this.cleanupSteps.add("truncate"); + } + + @Override + protected void clearSchema() { + this.schemaCleared = true; + this.cleanupSteps.add("schema"); + } + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphWorld.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphWorld.java new file mode 100644 index 0000000000..1b43187d71 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/HugeGraphWorld.java @@ -0,0 +1,205 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.tinkerpop; + +import java.lang.annotation.Annotation; +import java.util.Locale; +import java.util.Map; + +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.configuration2.MapConfiguration; +import org.apache.hugegraph.HugeGraph; +import org.apache.hugegraph.schema.SchemaManager; +import org.apache.hugegraph.type.define.DataType; +import org.apache.hugegraph.type.define.IdStrategy; +import org.apache.tinkerpop.gremlin.LoadGraphWith; +import org.apache.tinkerpop.gremlin.features.World; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.structure.Element; +import org.apache.tinkerpop.gremlin.structure.Graph; + +import io.cucumber.java.Scenario; + +public class HugeGraphWorld implements World { + + private static final HugeGraphProviderContext PROVIDER_CONTEXT = + new HugeGraphProviderContext(); + + private final ProcessTestGraphProvider provider; + private Scenario scenario; + private Graph graph; + private Configuration configuration; + + public HugeGraphWorld() { + this.provider = PROVIDER_CONTEXT.provider(); + } + + static void clearProvider() { + PROVIDER_CONTEXT.clear(); + } + + @Override + public void beforeEachScenario(Scenario scenario) { + this.scenario = scenario; + } + + @Override + public GraphTraversalSource getGraphTraversalSource( + LoadGraphWith.GraphData graphData) { + if (this.scenario == null) { + throw new IllegalStateException("Scenario has not been initialized"); + } + if (this.graph != null) { + this.clearGraph(); + } + + Map config = this.provider.getBaseConfiguration( + graphName(graphData), HugeGraphFeatureTest.class, + this.scenario.getName(), graphData); + this.configuration = new MapConfiguration(config); + this.graph = this.provider.openTestGraph(this.configuration); + this.prepareGraph(graphData); + return this.provider.traversal(this.graph); + } + + @Override + public void afterEachScenario() { + this.clearGraph(); + } + + @Override + public String convertIdToScript(Object id, + Class type) { + return this.provider.convertId(id, type); + } + + private void clearGraph() { + if (this.graph == null) { + return; + } + + try { + this.provider.clear(this.graph, this.configuration); + } catch (Exception e) { + throw new IllegalStateException("Failed to clear test graph", e); + } finally { + this.graph = null; + this.configuration = null; + } + } + + private void prepareGraph(LoadGraphWith.GraphData graphData) { + TestGraph testGraph = (TestGraph) this.graph; + if (graphData == null) { + testGraph.clearAll(""); + testGraph.initModernSchema(IdStrategy.AUTOMATIC); + this.prepareGherkinSchema(testGraph); + testGraph.tx().commit(); + testGraph.autoPerson(true); + return; + } + + this.provider.loadGraphData(testGraph, new GraphDataLoader(graphData), + HugeGraphFeatureTest.class, + this.scenario.getName()); + this.prepareGherkinSchema(testGraph); + testGraph.tx().commit(); + } + + private void prepareGherkinSchema(TestGraph testGraph) { + HugeGraph graph = testGraph.hugegraph(); + SchemaManager schema = graph.schema(); + schema.propertyKey("birthday").dataType(DataType.OBJECT) + .ifNotExist().create(); + schema.propertyKey("created").ifNotExist().create(); + schema.propertyKey("matched").ifNotExist().create(); + schema.vertexLabel("a").useAutomaticId().ifNotExist().create(); + schema.vertexLabel("b").useAutomaticId().ifNotExist().create(); + schema.vertexLabel("prefix_person").useAutomaticId() + .ifNotExist().create(); + this.prepareVertexLabel(graph, schema, "person"); + this.prepareVertexLabel(graph, schema, "software"); + this.prepareVertexLabel(graph, schema, TestGraph.DEFAULT_VL); + if (graph.existsVertexLabel("person")) { + schema.vertexLabel("person").properties("birthday") + .nullableKeys("birthday").append(); + } + this.prepareEdgeLabel(graph, schema, "knows"); + this.prepareEdgeLabel(graph, schema, "created"); + if (graph.existsVertexLabel("person")) { + schema.edgeLabel("self").link("person", "person") + .properties("weight", "created", "matched") + .nullableKeys("weight", "created", "matched") + .ifNotExist().create(); + this.prepareEdgeLabel(graph, schema, "self"); + } + } + + private void prepareVertexLabel(HugeGraph graph, SchemaManager schema, + String label) { + if (!graph.existsVertexLabel(label)) { + return; + } + schema.vertexLabel(label).properties("created", "matched") + .nullableKeys("created", "matched").append(); + schema.indexLabel(label + "ByCreated").onV(label).by("created") + .secondary().ifNotExist().create(); + schema.indexLabel(label + "ByMatched").onV(label).by("matched") + .secondary().ifNotExist().create(); + } + + private void prepareEdgeLabel(HugeGraph graph, SchemaManager schema, + String label) { + if (!graph.existsEdgeLabel(label)) { + return; + } + schema.edgeLabel(label).properties("created", "matched") + .nullableKeys("created", "matched").append(); + schema.indexLabel(label + "ByCreated").onE(label).by("created") + .secondary().ifNotExist().create(); + schema.indexLabel(label + "ByMatched").onE(label).by("matched") + .secondary().ifNotExist().create(); + } + + private static String graphName(LoadGraphWith.GraphData graphData) { + if (graphData == null) { + return "gherkin_empty_standard"; + } + return "gherkin_" + graphData.name().toLowerCase(Locale.ROOT) + + "_standard"; + } + + private static final class GraphDataLoader implements LoadGraphWith { + + private final GraphData graphData; + + private GraphDataLoader(GraphData graphData) { + this.graphData = graphData; + } + + @Override + public GraphData value() { + return this.graphData; + } + + @Override + public Class annotationType() { + return LoadGraphWith.class; + } + } +} diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/ProcessBasicSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/ProcessBasicSuite.java index e0fcba9832..b289912f00 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/ProcessBasicSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/ProcessBasicSuite.java @@ -25,7 +25,10 @@ import org.apache.tinkerpop.gremlin.process.traversal.CoreTraversalTest; import org.apache.tinkerpop.gremlin.process.traversal.TraversalEngine; import org.apache.tinkerpop.gremlin.process.traversal.TraversalInterruptionTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.ComparabilitySemanticsTest; import org.apache.tinkerpop.gremlin.process.traversal.step.ComplexTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.LambdaStepTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.OrderabilityTest; import org.apache.tinkerpop.gremlin.process.traversal.step.branch.BranchTest; import org.apache.tinkerpop.gremlin.process.traversal.step.branch.ChooseTest; import org.apache.tinkerpop.gremlin.process.traversal.step.branch.LocalTest; @@ -51,6 +54,7 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.CoalesceTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.ConstantTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.CountTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.ElementMapTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.FlatMapTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.FoldTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.GraphTest; @@ -61,6 +65,8 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.map.MathTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.MaxTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.MeanTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.MergeEdgeTest; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.MergeVertexTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.MinTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.OrderTest; import org.apache.tinkerpop.gremlin.process.traversal.step.map.PathTest; @@ -81,12 +87,12 @@ import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SackTest; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SideEffectCapTest; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SideEffectTest; -import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.StoreTest; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.SubgraphTest; import org.apache.tinkerpop.gremlin.process.traversal.step.sideEffect.TreeTest; import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.ElementIdStrategyProcessTest; import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.EventStrategyProcessTest; import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.PartitionStrategyProcessTest; +import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.SeedStrategyProcessTest; import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.SubgraphStrategyProcessTest; import org.apache.tinkerpop.gremlin.process.traversal.strategy.decoration.TranslationStrategyProcessTest; import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.EarlyLimitStrategyProcessTest; @@ -105,9 +111,13 @@ public class ProcessBasicSuite extends AbstractGremlinSuite { /** * This list of tests in the suite that will be executed - * as part of this suite. + * as part of this suite. It is synchronized with TinkerPop 3.8.1's + * official ProcessStandardSuite; HugeGraphWriteTest + * intentionally replaces WriteTest.Traversals. */ private static final Class[] ALL_TESTS = new Class[]{ + LambdaStepTest.Traversals.class, + // branch BranchTest.Traversals.class, ChooseTest.Traversals.class, @@ -138,6 +148,7 @@ public class ProcessBasicSuite extends AbstractGremlinSuite { CoalesceTest.Traversals.class, ConstantTest.Traversals.class, CountTest.Traversals.class, + ElementMapTest.Traversals.class, FlatMapTest.Traversals.class, FoldTest.Traversals.class, GraphTest.Traversals.class, @@ -149,6 +160,8 @@ public class ProcessBasicSuite extends AbstractGremlinSuite { MathTest.Traversals.class, MaxTest.Traversals.class, MeanTest.Traversals.class, + MergeEdgeTest.Traversals.class, + MergeVertexTest.Traversals.class, MinTest.Traversals.class, SumTest.Traversals.class, OrderTest.Traversals.class, @@ -161,7 +174,7 @@ public class ProcessBasicSuite extends AbstractGremlinSuite { VertexTest.Traversals.class, UnfoldTest.Traversals.class, ValueMapTest.Traversals.class, - // Override WriteTest.Traversals.class + // Intentionally replace WriteTest.Traversals.class HugeGraphWriteTest.class, // sideEffect @@ -173,7 +186,6 @@ public class ProcessBasicSuite extends AbstractGremlinSuite { SackTest.Traversals.class, SideEffectCapTest.Traversals.class, SideEffectTest.Traversals.class, - StoreTest.Traversals.class, SubgraphTest.Traversals.class, TreeTest.Traversals.class, @@ -190,11 +202,16 @@ public class ProcessBasicSuite extends AbstractGremlinSuite { EventStrategyProcessTest.class, ReadOnlyStrategyProcessTest.class, PartitionStrategyProcessTest.class, + SeedStrategyProcessTest.class, SubgraphStrategyProcessTest.class, // optimizations IncidentToAdjacentStrategyProcessTest.class, - EarlyLimitStrategyProcessTest.class + EarlyLimitStrategyProcessTest.class, + + // semantics + OrderabilityTest.Traversals.class, + ComparabilitySemanticsTest.class }; /** @@ -202,6 +219,8 @@ public class ProcessBasicSuite extends AbstractGremlinSuite { * Gremlin flavors should implement to be compliant with Gremlin. */ private static final Class[] TESTS_TO_ENFORCE = new Class[]{ + LambdaStepTest.class, + // branch BranchTest.class, ChooseTest.class, @@ -232,6 +251,7 @@ public class ProcessBasicSuite extends AbstractGremlinSuite { CoalesceTest.class, ConstantTest.class, CountTest.class, + ElementMapTest.class, FlatMapTest.class, FoldTest.class, LoopsTest.class, @@ -261,7 +281,6 @@ public class ProcessBasicSuite extends AbstractGremlinSuite { SackTest.class, SideEffectCapTest.class, SideEffectTest.class, - StoreTest.class, SubgraphTest.class, TreeTest.class }; diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/StructureBasicSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/StructureBasicSuite.java index 593e89359e..e6606bb10a 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/StructureBasicSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/StructureBasicSuite.java @@ -30,6 +30,7 @@ import org.apache.tinkerpop.gremlin.structure.GraphTest; import org.apache.tinkerpop.gremlin.structure.PropertyTest; import org.apache.tinkerpop.gremlin.structure.SerializationTest; +import org.apache.tinkerpop.gremlin.structure.TransactionMultiThreadedTest; import org.apache.tinkerpop.gremlin.structure.TransactionTest; import org.apache.tinkerpop.gremlin.structure.VariablesTest; import org.apache.tinkerpop.gremlin.structure.VertexPropertyTest; @@ -64,7 +65,8 @@ public class StructureBasicSuite extends AbstractGremlinSuite { /** * This list of tests in the suite that will be executed. * Gremlin developers should add to this list - * as needed to enforce tests upon implementations. + * as needed to enforce tests upon implementations. This list is synchronized + * with TinkerPop 3.8.1's official StructureStandardSuite. */ private static final Class[] ALL_TESTS = new Class[]{ CommunityGeneratorTest.class, @@ -94,6 +96,7 @@ public class StructureBasicSuite extends AbstractGremlinSuite { SerializationTest.class, StarGraphTest.class, TransactionTest.class, + TransactionMultiThreadedTest.class, VertexTest.class }; diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/TestGraph.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/TestGraph.java index 53dc1fe9ac..4d76cb72bf 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/TestGraph.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/TestGraph.java @@ -55,6 +55,8 @@ public class TestGraph implements Graph { public static final Set TRUNCATE_BACKENDS = ImmutableSet.of("rocksdb"); + private static final String HSTORE_BACKEND = "hstore"; + private static volatile int id = 666; private HugeGraph graph; @@ -95,24 +97,53 @@ protected void clearBackend() { @Watched protected void clearAll(String testClass) { - List pks = this.graph.schema().getPropertyKeys(); - if (pks.isEmpty()) { - // No need to clear if there is no PKs(that's no schema and data) + if (!this.hasSchema() && + !testClass.endsWith("VariableAsMapTest")) { + // No need to clear if there is no schema, data, or variables return; } - if (TRUNCATE_BACKENDS.contains(this.graph.backend())) { + String backend = this.graph.backend(); + if (HSTORE_BACKEND.equals(backend)) { + // HStore keeps schema in PD, outside the truncated data store + this.truncateBackend(); + this.clearSchemaAndVariables(testClass); + } else if (TRUNCATE_BACKENDS.contains(backend)) { // Delete all data by truncating tables this.truncateBackend(); } else { - // Clear schema (also include data) + this.clearSchemaAndVariables(testClass); + } + } + + @Watched + protected void clearForLoad() { + if (HSTORE_BACKEND.equals(this.graph.backend())) { + // An auxiliary graph can be loaded while its source remains open. + // Truncating it makes the source invisible to HStore scans. + // Only the bootstrap schema needs to be removed at this point. this.clearSchema(); + } else { + this.clearAll(""); + } + } - // Clear variables if needed (would not clear when clearing schema) - if (testClass.endsWith("VariableAsMapTest")) { - this.clearVariables(); - this.tx().commit(); - } + private boolean hasSchema() { + SchemaManager schema = this.graph.schema(); + return !schema.getPropertyKeys().isEmpty() || + !schema.getVertexLabels().isEmpty() || + !schema.getEdgeLabels().isEmpty() || + !schema.getIndexLabels().isEmpty(); + } + + private void clearSchemaAndVariables(String testClass) { + // Clear schema (also include data) + this.clearSchema(); + + // Clear variables if needed (would not clear when clearing schema) + if (testClass.endsWith("VariableAsMapTest")) { + this.clearVariables(); + this.tx().commit(); } } @@ -430,6 +461,7 @@ public void initModernSchema(IdStrategy idStrategy) { SchemaManager schema = this.graph.schema(); schema.propertyKey("weight").asDouble().ifNotExist().create(); + schema.propertyKey("a").asInt().ifNotExist().create(); schema.propertyKey("name").ifNotExist().create(); schema.propertyKey("lang").ifNotExist().create(); schema.propertyKey("age").asInt().ifNotExist().create(); @@ -499,12 +531,12 @@ public void initModernSchema(IdStrategy idStrategy) { } schema.edgeLabel("knows").link("person", "person") - .properties("weight", "year") - .nullableKeys("weight", "year") + .properties("weight", "year", "a") + .nullableKeys("weight", "year", "a") .ifNotExist().create(); schema.edgeLabel("created").link("person", "software") - .properties("weight") - .nullableKeys("weight") + .properties("weight", "a") + .nullableKeys("weight", "a") .ifNotExist().create(); schema.edgeLabel("codeveloper").link("person", "person") .properties("year") @@ -591,9 +623,15 @@ public void initClassicSchema(IdStrategy idStrategy) { @Watched public void initBasicSchema(IdStrategy idStrategy, String defaultVL) { + this.initBasicSchema(idStrategy, defaultVL, defaultVL); + } + + @Watched + public void initBasicSchema(IdStrategy idStrategy, String defaultVL, + String selfVL) { this.initBasicPropertyKey(); this.initBasicVertexLabelV(idStrategy, defaultVL); - this.initBasicVertexLabelAndEdgeLabelExceptV(defaultVL); + this.initBasicVertexLabelAndEdgeLabelExceptV(defaultVL, selfVL); } @Watched @@ -603,7 +641,9 @@ private void initBasicPropertyKey() { schema.propertyKey("__id").ifNotExist().create(); schema.propertyKey("oid").asInt().ifNotExist().create(); schema.propertyKey("communityIndex").asInt().ifNotExist().create(); - schema.propertyKey("test").ifNotExist().create(); + if (!this.graph.existsPropertyKey("test")) { + schema.propertyKey("test").ifNotExist().create(); + } schema.propertyKey("testing").ifNotExist().create(); schema.propertyKey("data").ifNotExist().create(); schema.propertyKey("name").ifNotExist().create(); @@ -748,7 +788,8 @@ private void initBasicVertexLabelV(IdStrategy idStrategy, String defaultVL) { } @Watched - private void initBasicVertexLabelAndEdgeLabelExceptV(String defaultVL) { + private void initBasicVertexLabelAndEdgeLabelExceptV(String defaultVL, + String selfVL) { SchemaManager schema = this.graph.schema(); if (!"person".equals(defaultVL)) { @@ -770,7 +811,7 @@ private void initBasicVertexLabelAndEdgeLabelExceptV(String defaultVL) { .nullableKeys("test") .ifNotExist().create(); - schema.edgeLabel("self").link(defaultVL, defaultVL) + schema.edgeLabel("self").link(selfVL, selfVL) .properties("__id", "test", "name", "some", "acl", "weight", "here", "to-change", "dropped", "not-dropped", "new", "to-drop", "short", "long") @@ -778,6 +819,8 @@ private void initBasicVertexLabelAndEdgeLabelExceptV(String defaultVL) { "here", "to-change", "dropped", "not-dropped", "new", "to-drop", "short", "long") .ifNotExist().create(); + schema.edgeLabel("self-but-different").link(selfVL, selfVL) + .ifNotExist().create(); schema.edgeLabel("aTOa").link(defaultVL, defaultVL) .properties("gremlin.partitionGraphStrategy.partition") .nullableKeys("gremlin.partitionGraphStrategy.partition") diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/TestGraphProvider.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/TestGraphProvider.java index c257e8bd1c..45c93c2774 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/TestGraphProvider.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/tinkerpop/TestGraphProvider.java @@ -30,6 +30,7 @@ import org.apache.commons.configuration2.Configuration; import org.apache.commons.configuration2.PropertiesConfiguration; +import org.apache.commons.text.StringEscapeUtils; import org.apache.hugegraph.HugeGraph; import org.apache.hugegraph.config.CoreOptions; import org.apache.hugegraph.perf.PerfUtil.Watched; @@ -47,11 +48,13 @@ import org.apache.tinkerpop.gremlin.FeatureRequirements; import org.apache.tinkerpop.gremlin.LoadGraphWith; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversalSource; +import org.apache.tinkerpop.gremlin.process.traversal.step.map.MergeEdgeTest; import org.apache.tinkerpop.gremlin.process.traversal.strategy.optimization.LazyBarrierStrategy; import org.apache.tinkerpop.gremlin.structure.Element; import org.apache.tinkerpop.gremlin.structure.Graph; import org.apache.tinkerpop.gremlin.structure.Graph.Features.VertexPropertyFeatures; import org.apache.tinkerpop.gremlin.structure.Transaction; +import org.apache.tinkerpop.gremlin.structure.TransactionMultiThreadedTest; import org.junit.Assert; import org.junit.Assume; import org.slf4j.Logger; @@ -85,10 +88,19 @@ public class TestGraphProvider extends AbstractGraphProvider { private static final String GREMLIN_GRAPH_KEY = "gremlin.graph"; private static final String GREMLIN_GRAPH_VALUE = "org.apache.hugegraph.tinkerpop.TestGraphFactory"; + private static final String BACKEND = "backend"; + private static final String BACKEND_ROCKSDB = "rocksdb"; + private static final String ROCKSDB_DATA_PATH = "rocksdb.data_path"; + private static final String ROCKSDB_WAL_PATH = "rocksdb.wal_path"; + private static final String ROCKSDB_DATA_DISKS = "rocksdb.data_disks"; + private static final String TEST_PATH_SEPARATOR = "/"; + private static final int MAX_ROCKSDB_PATH_SUFFIX_PREFIX_LENGTH = 80; private static final String AKEY_CLASS_PREFIX = "org.apache.tinkerpop.gremlin.structure." + "PropertyTest.PropertyFeatureSupportTest"; + private static final String SUPPORTS_PREFIX = "supports"; + private static final String FEATURE_VALUES_SUFFIX = "Values"; private static final String IO_CLASS_PREFIX = "org.apache.tinkerpop.gremlin.structure.io.IoGraphTest"; private static final String IO_TEST_PREFIX = @@ -182,8 +194,17 @@ public Map getBaseConfiguration( confMap.put(key, config.getProperty(key)); } String storePrefix = config.getString(CoreOptions.STORE.name()); - confMap.put(CoreOptions.STORE.name(), - storePrefix + "_" + this.suite + "_" + graphName); + String store = storePrefix + "_" + this.suite + "_" + graphName; + if (isTransactionMultiThreadedPropertyTest(testClass, testMethod)) { + store += "_txprop"; + } else if (isMergeEdgeSelfTest(testClass, testMethod)) { + store += "_meself"; + } + confMap.put(CoreOptions.STORE.name(), store); + if (isRocksDBBackend(config)) { + this.isolateRocksDBPaths(confMap, graphName, testClass, + testMethod); + } confMap.put(GREMLIN_GRAPH_KEY, GREMLIN_GRAPH_VALUE); confMap.put(TEST_CLASS, testClass); confMap.put(TEST_METHOD, testMethod); @@ -193,6 +214,90 @@ public Map getBaseConfiguration( return confMap; } + private void isolateRocksDBPaths(Map confMap, + String graphName, Class testClass, + String testMethod) { + String testClassName = testClass.getName(); + String rawSuffix = this.suite + "_" + graphName + "_" + + testClassName + "_" + testMethod; + String prefix = sanitizePathPart(this.suite + "_" + graphName + "_" + + testClass.getSimpleName() + "_" + + testMethod); + if (prefix.length() > MAX_ROCKSDB_PATH_SUFFIX_PREFIX_LENGTH) { + prefix = prefix.substring(0, + MAX_ROCKSDB_PATH_SUFFIX_PREFIX_LENGTH); + } + String pathSuffix = prefix + "_" + shortHash(rawSuffix); + isolatePath(confMap, ROCKSDB_DATA_PATH, pathSuffix); + isolatePath(confMap, ROCKSDB_WAL_PATH, pathSuffix); + + Object dataDisks = confMap.get(ROCKSDB_DATA_DISKS); + if (dataDisks != null) { + confMap.put(ROCKSDB_DATA_DISKS, + isolateDataDisks(dataDisks, pathSuffix)); + } + } + + private static void isolatePath(Map confMap, String key, + String pathSuffix) { + Object path = confMap.get(key); + if (path == null) { + return; + } + confMap.put(key, appendPath(path.toString(), pathSuffix)); + } + + private static String isolateDataDisks(Object dataDisks, + String pathSuffix) { + String value = dataDisks.toString().trim(); + if (value.isEmpty()) { + return value; + } + + boolean wrapped = value.startsWith("[") && value.endsWith("]"); + String body = wrapped ? value.substring(1, value.length() - 1) : value; + String[] entries = body.split(","); + StringBuilder builder = new StringBuilder(); + for (String entry : entries) { + String item = entry.trim(); + int index = item.indexOf(':'); + if (index < 0) { + return value; + } + String table = item.substring(0, index).trim(); + String path = item.substring(index + 1).trim(); + if (table.isEmpty() || path.isEmpty()) { + return value; + } + if (builder.length() > 0) { + builder.append(','); + } + builder.append(table).append(':') + .append(appendPath(path, pathSuffix)); + } + return wrapped ? "[" + builder + "]" : builder.toString(); + } + + private static String appendPath(String path, String suffix) { + if (path.endsWith("/") || path.endsWith("\\")) { + return path + suffix; + } + return path + TEST_PATH_SEPARATOR + suffix; + } + + private static String sanitizePathPart(String value) { + return value.replaceAll("[^A-Za-z0-9._-]", "_"); + } + + private static String shortHash(String value) { + return Integer.toHexString(value.hashCode()); + } + + private static boolean isRocksDBBackend(Configuration config) { + return config != null && + BACKEND_ROCKSDB.equals(config.getString(BACKEND, "")); + } + private static boolean customizedId(Class test, String testMethod) { Method method; try { @@ -215,10 +320,41 @@ private static boolean customizedId(Class test, String testMethod) { return false; } + private static boolean isTransactionMultiThreadedPropertyTest( + Class testClass, String testMethod) { + return testClass == TransactionMultiThreadedTest.class && + testMethod.equals("shouldChangeVertexProperty"); + } + + private static boolean isMergeEdgeSelfTest(Class testClass, + String testMethod) { + return testClass == MergeEdgeTest.Traversals.class && + testMethod.equals("g_V_mergeEXlabel_self_weight_05X"); + } + private static String getAKeyType(Class clazz, String method) { if (clazz.getCanonicalName().startsWith(AKEY_CLASS_PREFIX)) { - return method.substring(method.indexOf('[') + 9, - method.indexOf('(') - 6); + String feature = method; + int featureStart = method.indexOf('['); + int featureEnd = method.indexOf(']'); + if (featureStart >= 0 && featureEnd > featureStart) { + feature = method.substring(featureStart + 1, featureEnd); + } + + if (!feature.startsWith(SUPPORTS_PREFIX)) { + return null; + } + feature = feature.substring(SUPPORTS_PREFIX.length()); + + int valueStart = feature.indexOf('('); + if (valueStart >= 0) { + feature = feature.substring(0, valueStart); + } + if (!feature.endsWith(FEATURE_VALUES_SUFFIX)) { + return null; + } + return feature.substring(0, feature.length() - + FEATURE_VALUES_SUFFIX.length()); } return null; } @@ -292,8 +428,15 @@ public Graph openTestGraph(final Configuration config) { testGraph.initPropertyKey("long", "Long"); } + if (isTransactionMultiThreadedPropertyTest(testClass, testMethod)) { + testGraph.initPropertyKey("test", "Integer"); + } + // Basic schema is initiated by default once a graph is open - testGraph.initBasicSchema(idStrategy(config), TestGraph.DEFAULT_VL); + String selfVL = isMergeEdgeSelfTest(testClass, testMethod) ? + "person" : TestGraph.DEFAULT_VL; + testGraph.initBasicSchema(idStrategy(config), TestGraph.DEFAULT_VL, + selfVL); if (testClass.getName().equals( "org.apache.tinkerpop.gremlin.process.traversal.step.map.ReadTest$Traversals")) { testGraph.initEdgeLabelPersonKnowsPerson(); @@ -330,6 +473,10 @@ public void clear(Graph graph, Configuration config) throws Exception { String graphName = config.getString(CoreOptions.STORE.name()); if (!testGraph.initedBackend()) { testGraph.close(); + if (this.graphs.get(graphName) == testGraph) { + this.graphs.remove(graphName); + } + return; } if (testGraph.closed()) { if (this.graphs.get(graphName) == testGraph) { @@ -349,6 +496,13 @@ public void clear(Graph graph, Configuration config) throws Exception { Class testClass = (Class) config.getProperty(TEST_CLASS); testGraph.clearAll(testClass.getCanonicalName()); + if (isRocksDBBackend(config)) { + testGraph.close(); + if (this.graphs.get(graphName) == testGraph) { + this.graphs.remove(graphName); + } + } + LOG.debug("Clear graph '{}'", graphName); } @@ -389,7 +543,7 @@ public void loadGraphData(final Graph graph, TestGraph testGraph = (TestGraph) graph; // Clear basic schema initiated in openTestGraph - testGraph.clearAll(""); + testGraph.clearForLoad(); if (testGraph.loadedGraph() == null) { testGraph.loadedGraph(REGULAR_LOAD); @@ -441,6 +595,10 @@ public GraphTraversalSource traversal(Graph graph) { @Override public String convertId(Object id, Class c) { - return id.toString(); + if (id instanceof Number) { + return id.toString(); + } + return String.format("\"%s\"", StringEscapeUtils.escapeJava( + id.toString())); } } diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtilOptimizeTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtilOptimizeTest.java index c79db5056f..8367efe40a 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtilOptimizeTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/traversal/optimize/TraversalUtilOptimizeTest.java @@ -24,15 +24,21 @@ import org.apache.hugegraph.backend.id.Id; import org.apache.hugegraph.backend.id.IdGenerator; import org.apache.hugegraph.exception.NotFoundException; +import org.apache.hugegraph.schema.IndexLabel; import org.apache.hugegraph.schema.PropertyKey; +import org.apache.hugegraph.schema.VertexLabel; import org.apache.hugegraph.testutil.Assert; import org.apache.hugegraph.type.define.DataType; +import org.apache.hugegraph.type.define.IndexType; +import org.apache.hugegraph.type.define.SchemaStatus; import org.apache.tinkerpop.gremlin.process.traversal.P; import org.apache.tinkerpop.gremlin.process.traversal.Step; +import org.apache.tinkerpop.gremlin.process.traversal.TextP; import org.apache.tinkerpop.gremlin.process.traversal.Traversal; import org.apache.tinkerpop.gremlin.process.traversal.TraversalStrategy; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.__; +import org.apache.tinkerpop.gremlin.process.traversal.step.HasContainerHolder; import org.apache.tinkerpop.gremlin.process.traversal.step.TraversalParent; import org.apache.tinkerpop.gremlin.process.traversal.step.filter.AndStep; import org.apache.tinkerpop.gremlin.process.traversal.step.filter.HasStep; @@ -57,6 +63,46 @@ public void testCanExtractHasContainerWithoutGraph() { null, new HasContainer("~id", P.eq("1")))); Assert.assertFalse(TraversalUtil.canExtractHasContainer( null, new HasContainer("name", P.eq("marko")))); + Assert.assertFalse(TraversalUtil.canExtractHasContainer( + null, new HasContainer(null, P.eq("marko")))); + Assert.assertFalse(TraversalUtil.canExtractHasContainer( + null, new HasContainer(T.label.getAccessor(), P.eq(null)))); + Assert.assertFalse(TraversalUtil.canExtractHasContainer( + null, new HasContainer(T.label.getAccessor(), + P.within(null, "person")))); + } + + @Test + public void testCanExtractHasContainerWithNullPredicate() { + Assert.assertFalse(TraversalUtil.canExtractHasContainer( + null, new HasContainer("name", null))); + } + + @Test + public void testExtractHasContainerKeepsNullKeyLocal() { + Traversal.Admin traversal = __.V() + .has((String) null, + "test-null-key") + .asAdmin(); + HugeGraphStep newStep = replaceGraphStep(traversal); + + TraversalUtil.extractHasContainer(newStep, traversal); + + Assert.assertTrue(newStep.getHasContainers().isEmpty()); + Assert.assertTrue(hasStepExists(traversal)); + } + + @Test + public void testExtractHasContainerKeepsMixedNullLabelLocal() { + Traversal.Admin traversal = __.V() + .hasLabel(null, "person") + .asAdmin(); + HugeGraphStep newStep = replaceGraphStep(traversal); + + TraversalUtil.extractHasContainer(newStep, traversal); + + Assert.assertTrue(newStep.getHasContainers().isEmpty()); + Assert.assertTrue(hasStepExists(traversal, T.label.getAccessor())); } @Test @@ -89,6 +135,32 @@ public void testCanExtractHasContainerWithNonTextProperty() { graph, new HasContainer("age", P.eq(1)))); } + @Test + public void testCanExtractHasContainerKeepsNegatedComparePredicateLocal() { + HugeGraph graph = Mockito.mock(HugeGraph.class); + PropertyKey age = propertyKey(1L, "age", DataType.INT); + Mockito.when(graph.propertyKey("age")).thenReturn(age); + + Assert.assertFalse(TraversalUtil.canExtractHasContainer( + graph, new HasContainer("age", P.not(P.lte(10))))); + } + + @Test + public void testExtractHasContainerKeepsNestedNegatedPredicateLocal() { + HugeGraph graph = Mockito.mock(HugeGraph.class); + PropertyKey age = propertyKey(1L, "age", DataType.INT); + Mockito.when(graph.propertyKey("age")).thenReturn(age); + + Traversal.Admin traversal = traversal( + __.V().has("age", P.gt(18).and(P.not(P.lte(65)))), graph); + HugeGraphStep newStep = replaceGraphStep(traversal); + + TraversalUtil.extractHasContainer(newStep, traversal); + + Assert.assertTrue(newStep.getHasContainers().isEmpty()); + Assert.assertTrue(hasStepExists(traversal, "age")); + } + @Test public void testCanExtractHasContainerWithTextRangePredicate() { HugeGraph graph = Mockito.mock(HugeGraph.class); @@ -122,6 +194,68 @@ public void testExtractHasContainerKeepsTextRangeGraphHasStep() { Assert.assertTrue(hasStepExists(traversal)); } + @Test + public void testExtractHasContainerKeepsUnindexedGraphPropertyLocal() { + HugeGraph graph = Mockito.mock(HugeGraph.class); + PropertyKey age = propertyKey(1L, "age", DataType.INT); + PropertyKey name = propertyKey(2L, "name", DataType.TEXT); + Mockito.when(graph.propertyKey("age")).thenReturn(age); + Mockito.when(graph.propertyKey("name")).thenReturn(name); + + Traversal.Admin traversal = traversal( + __.V().has("person", "name", TextP.containing("ar")), + graph); + HasStep hasStep = (HasStep) traversal.getEndStep(); + hasStep.addHasContainer(new HasContainer("age", P.eq(29))); + HugeGraphStep newStep = replaceGraphStep(traversal); + + TraversalUtil.extractHasContainer(newStep, traversal); + + Assert.assertTrue(hasContainer(newStep, T.label.getAccessor())); + Assert.assertFalse(hasContainer(newStep, "age")); + Assert.assertFalse(hasContainer(newStep, "name")); + Assert.assertFalse(hasStepExists(traversal, T.label.getAccessor())); + Assert.assertTrue(hasStepExists(traversal, "age")); + Assert.assertTrue(hasStepExists(traversal, "name")); + } + + @Test + public void testExtractHasContainerKeepsRebuildingIndexPropertyLocal() { + HugeGraph graph = Mockito.mock(HugeGraph.class); + PropertyKey age = propertyKey(1L, "age", DataType.INT); + PropertyKey name = propertyKey(2L, "name", DataType.TEXT); + VertexLabel person = new VertexLabel(graph, IdGenerator.of(3L), + "person"); + person.properties(age.id(), name.id()); + IndexLabel ageIndex = new IndexLabel(graph, IdGenerator.of(4L), + "personByAge"); + ageIndex.indexField(age.id()); + ageIndex.indexType(IndexType.SECONDARY); + ageIndex.status(SchemaStatus.REBUILDING); + person.addIndexLabel(ageIndex.id()); + + Mockito.when(graph.propertyKey("age")).thenReturn(age); + Mockito.when(graph.propertyKey("name")).thenReturn(name); + Mockito.when(graph.vertexLabel("person")).thenReturn(person); + Mockito.when(graph.indexLabel(ageIndex.id())).thenReturn(ageIndex); + + Traversal.Admin traversal = traversal( + __.V().has("person", "name", TextP.containing("ar")), + graph); + HasStep hasStep = (HasStep) traversal.getEndStep(); + hasStep.addHasContainer(new HasContainer("age", P.eq(29))); + HugeGraphStep newStep = replaceGraphStep(traversal); + + TraversalUtil.extractHasContainer(newStep, traversal); + + Assert.assertTrue(hasContainer(newStep, T.label.getAccessor())); + Assert.assertFalse(hasContainer(newStep, "age")); + Assert.assertFalse(hasContainer(newStep, "name")); + Assert.assertFalse(hasStepExists(traversal, T.label.getAccessor())); + Assert.assertTrue(hasStepExists(traversal, "age")); + Assert.assertTrue(hasStepExists(traversal, "name")); + } + @Test public void testExtractHasContainerKeepsTextRangeWithoutGraph() { Traversal.Admin traversal = __.V() @@ -280,6 +414,31 @@ public void testExtractHasContainerKeepsTextRangeVertexHasStep() { Assert.assertTrue(hasStepExists(traversal)); } + @Test + public void testExtractHasContainerPartiallyExtractsVertexHasStep() { + HugeGraph graph = Mockito.mock(HugeGraph.class); + PropertyKey age = propertyKey(1L, "age", DataType.INT); + PropertyKey name = propertyKey(2L, "name", DataType.TEXT); + Mockito.when(graph.propertyKey("age")).thenReturn(age); + Mockito.when(graph.propertyKey("name")).thenReturn(name); + + Traversal.Admin traversal = traversal( + __.V().out().has("person", "name", TextP.containing("ar")), + graph); + HasStep hasStep = (HasStep) traversal.getEndStep(); + hasStep.addHasContainer(new HasContainer("age", P.eq(29))); + HugeVertexStep newStep = replaceVertexStep(traversal); + + TraversalUtil.extractHasContainer(newStep, traversal); + + Assert.assertTrue(hasContainer(newStep, T.label.getAccessor())); + Assert.assertTrue(hasContainer(newStep, "age")); + Assert.assertFalse(hasContainer(newStep, "name")); + Assert.assertFalse(hasStepExists(traversal, T.label.getAccessor())); + Assert.assertFalse(hasStepExists(traversal, "age")); + Assert.assertTrue(hasStepExists(traversal, "name")); + } + @Test public void testExtractHasContainerRemovesSafeVertexHasStep() { HugeGraph graph = Mockito.mock(HugeGraph.class); @@ -314,6 +473,11 @@ public void testIsPositiveLabelContainer() { Assert.assertFalse(TraversalUtil.isPositiveLabelContainer( new HasContainer(T.label.getAccessor(), P.within(Collections.emptyList())))); + Assert.assertFalse(TraversalUtil.isPositiveLabelContainer( + new HasContainer(T.label.getAccessor(), P.eq(null)))); + Assert.assertFalse(TraversalUtil.isPositiveLabelContainer( + new HasContainer(T.label.getAccessor(), + P.within(null, "person")))); } @Test @@ -419,7 +583,8 @@ private static void replaceStep(Step origin, Step newStep, TraversalHelper.replaceStep((Step) origin, (Step) newStep, traversal); } - private static boolean hasContainer(HugeGraphStep step, String key) { + private static boolean hasContainer(HasContainerHolder step, + String key) { for (HasContainer has : step.getHasContainers()) { if (key.equals(has.getKey())) { return true; diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java index 1733680e3f..21a75bb17a 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/UnitTestSuite.java @@ -19,12 +19,15 @@ import org.apache.hugegraph.api.auth.GraphSpaceAuthPayloadTest; import org.apache.hugegraph.api.auth.GraphSpaceGroupAPITest; +import org.apache.hugegraph.api.cypher.CypherClientTest; +import org.apache.hugegraph.auth.GremlinLangRequestGuardTest; import org.apache.hugegraph.auth.StandardAuthManagerV2Test; import org.apache.hugegraph.auth.WsAndHttpBasicAuthHandlerTest; import org.apache.hugegraph.core.RoleElectionStateMachineTest; import org.apache.hugegraph.meta.EtcdMetaDriverTest; import org.apache.hugegraph.meta.MetaManagerSchemaCacheClearEventTest; import org.apache.hugegraph.meta.managers.AuthMetaManagerTest; +import org.apache.hugegraph.tinkerpop.HugeGraphTestInfrastructureTest; import org.apache.hugegraph.traversal.optimize.TraversalUtilOptimizeTest; import org.apache.hugegraph.unit.api.auth.LoginAPITest; import org.apache.hugegraph.unit.api.filter.LoadDetectFilterTest; @@ -38,8 +41,10 @@ import org.apache.hugegraph.unit.cache.CachedSchemaTransactionTest; import org.apache.hugegraph.unit.cache.RamTableTest; import org.apache.hugegraph.unit.cmd.InitStoreConfigTest; +import org.apache.hugegraph.unit.config.GremlinConfigCompatibilityTest; import org.apache.hugegraph.unit.core.AnalyzerTest; import org.apache.hugegraph.unit.core.BackendMutationTest; +import org.apache.hugegraph.unit.core.BackendProviderFactoryTest; import org.apache.hugegraph.unit.core.BackendStoreInfoTest; import org.apache.hugegraph.unit.core.ConditionQueryFlattenTest; import org.apache.hugegraph.unit.core.ConditionTest; @@ -48,6 +53,8 @@ import org.apache.hugegraph.unit.core.ExceptionTest; import org.apache.hugegraph.unit.core.GraphManagerAdminInitTest; import org.apache.hugegraph.unit.core.GraphManagerConfigTest; +import org.apache.hugegraph.unit.core.GroovyScriptEngineCompatibilityTest; +import org.apache.hugegraph.unit.core.HugeFeaturesTest; import org.apache.hugegraph.unit.core.LocksTableTest; import org.apache.hugegraph.unit.core.PageStateTest; import org.apache.hugegraph.unit.core.QueryTest; @@ -73,11 +80,13 @@ import org.apache.hugegraph.unit.serializer.BinaryScatterSerializerTest; import org.apache.hugegraph.unit.serializer.BinarySerializerTest; import org.apache.hugegraph.unit.serializer.BytesBufferTest; +import org.apache.hugegraph.unit.serializer.HugeGraphSONModuleTest; import org.apache.hugegraph.unit.serializer.SerializerFactoryTest; import org.apache.hugegraph.unit.serializer.StoreSerializerTest; import org.apache.hugegraph.unit.serializer.TableBackendEntryTest; import org.apache.hugegraph.unit.serializer.TextBackendEntryTest; import org.apache.hugegraph.unit.serializer.TextSerializerTest; +import org.apache.hugegraph.unit.security.HugeGraphGremlinLangScriptEngineTest; import org.apache.hugegraph.unit.store.RamIntObjectMapTest; import org.apache.hugegraph.unit.util.CompressUtilTest; import org.apache.hugegraph.unit.util.JsonUtilTest; @@ -102,6 +111,8 @@ /* api gremlin */ GremlinQueryAPITest.class, + CypherClientTest.class, + GremlinLangRequestGuardTest.class, WsAndHttpBasicAuthHandlerTest.class, GraphSpaceGroupAPITest.class, GraphSpaceAuthPayloadTest.class, @@ -138,7 +149,10 @@ RowLockTest.class, AnalyzerTest.class, BackendMutationTest.class, + BackendProviderFactoryTest.class, ConditionTest.class, + GroovyScriptEngineCompatibilityTest.class, + HugeFeaturesTest.class, ConditionQueryFlattenTest.class, QueryTest.class, RangeTest.class, @@ -157,6 +171,7 @@ RoleElectionStateMachineTest.class, HugeGraphAuthProxyTest.class, SchemaElementTest.class, + HugeGraphTestInfrastructureTest.class, /* cmd */ InitStoreConfigTest.class, @@ -169,9 +184,16 @@ BinaryBackendEntryTest.class, BinarySerializerTest.class, BinaryScatterSerializerTest.class, + HugeGraphSONModuleTest.class, StoreSerializerTest.class, TextSerializerTest.class, + /* config */ + GremlinConfigCompatibilityTest.class, + + /* security */ + HugeGraphGremlinLangScriptEngineTest.class, + /* rocksdb */ RocksDBSessionsTest.class, RocksDBSessionTest.class, diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/auth/HugeGraphAuthProxyTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/auth/HugeGraphAuthProxyTest.java index 1b209c9139..c080668594 100644 --- a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/auth/HugeGraphAuthProxyTest.java +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/auth/HugeGraphAuthProxyTest.java @@ -21,6 +21,7 @@ import java.util.ArrayList; import java.util.List; +import org.apache.hugegraph.HugeException; import org.apache.hugegraph.HugeGraph; import org.apache.hugegraph.auth.AuthManager; import org.apache.hugegraph.auth.HugeAuthenticator; @@ -35,6 +36,7 @@ import org.apache.hugegraph.task.TaskScheduler; import org.apache.hugegraph.testutil.Assert; import org.apache.hugegraph.unit.BaseUnitTest; +import org.apache.hugegraph.util.Reflection; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.Filter; @@ -50,6 +52,43 @@ public class HugeGraphAuthProxyTest extends BaseUnitTest { + @Test + public void testJdk17ReflectionFilters() { + Reflection.registerFieldsToFilter(ReflectionFilterTarget.class, "field"); + Reflection.registerMethodsToFilter(ReflectionFilterTarget.class, "method"); + + Assert.assertThrows(NoSuchFieldException.class, + () -> ReflectionFilterTarget.class.getDeclaredField("field")); + Assert.assertThrows(NoSuchMethodException.class, + () -> ReflectionFilterTarget.class.getDeclaredMethod("method")); + Assert.assertThrows(IllegalArgumentException.class, + () -> Reflection.registerFieldsToFilter( + ReflectionFilterTarget.class, "field")); + } + + @Test + public void testJdk17ReflectionFilterFailureCause() { + Throwable exception = Assert.assertThrows( + HugeException.class, + () -> Reflection.registerFieldsToFilter( + ReflectionFailureTarget.class, (String) null)); + + Assert.assertInstanceOf(NullPointerException.class, exception.getCause()); + } + + private static class ReflectionFailureTarget { + } + + private static class ReflectionFilterTarget { + + @SuppressWarnings("unused") + private String field; + + @SuppressWarnings("unused") + private void method() { + } + } + private static HugeGraphAuthProxy.Context setContext( HugeGraphAuthProxy.Context context) { try { diff --git a/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/config/GremlinConfigCompatibilityTest.java b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/config/GremlinConfigCompatibilityTest.java new file mode 100644 index 0000000000..48804f6797 --- /dev/null +++ b/hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/config/GremlinConfigCompatibilityTest.java @@ -0,0 +1,1187 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hugegraph.unit.config; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.UUID; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Stream; + +import org.apache.hugegraph.backend.id.EdgeId; +import org.apache.hugegraph.backend.id.Id; +import org.apache.hugegraph.backend.id.IdGenerator; +import org.apache.hugegraph.structure.HugeEdge; +import org.apache.hugegraph.structure.HugeFeatures; +import org.apache.hugegraph.structure.HugeVertex; +import org.apache.hugegraph.testutil.Assert; +import org.apache.hugegraph.unit.BaseUnitTest; +import org.apache.hugegraph.unit.FakeObjects; +import org.apache.tinkerpop.gremlin.process.traversal.P; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.MutablePath; +import org.apache.tinkerpop.gremlin.process.traversal.step.util.Tree; +import org.apache.tinkerpop.gremlin.server.Settings; +import org.apache.tinkerpop.gremlin.structure.Edge; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedEdge; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedProperty; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertex; +import org.apache.tinkerpop.gremlin.structure.util.detached.DetachedVertexProperty; +import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceEdge; +import org.apache.tinkerpop.gremlin.structure.util.reference.ReferenceVertex; +import org.apache.tinkerpop.gremlin.util.MessageSerializer; +import org.apache.tinkerpop.gremlin.util.message.ResponseMessage; +import org.apache.tinkerpop.gremlin.util.message.ResponseStatusCode; +import org.apache.tinkerpop.gremlin.util.ser.GraphBinaryMessageSerializerV1; +import org.apache.tinkerpop.gremlin.util.ser.MessageTextSerializer; +import org.junit.Test; +import org.mockito.Mockito; +import org.yaml.snakeyaml.Yaml; + +import io.netty.buffer.ByteBuf; +import io.netty.buffer.ByteBufAllocator; + +public class GremlinConfigCompatibilityTest extends BaseUnitTest { + + private static final Pattern CLASS_NAME = + Pattern.compile("className:\\s*([^,}\\s]+)"); + private static final Pattern XML_COMMENT = + Pattern.compile("", Pattern.DOTALL); + private static final Pattern TINKERPOP_DEPENDENCY = Pattern.compile( + "\\s*