Skip to content

WIP - plan query with record type key #4028

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
hatyo wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

WIP - plan query with record type key #4028

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...rc/main/java/com/apple/foundationdb/record/query/expressions/RecordTypeKeyComparison.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ public static class RecordTypeComparison implements Comparisons.Comparison {
@Nonnull
private final String recordTypeName;

RecordTypeComparison(@Nonnull String recordTypeName) {
public RecordTypeComparison(@Nonnull String recordTypeName) {
this.recordTypeName = recordTypeName;
}

Expand Down
10 changes: 7 additions & 3 deletions ...ava/com/apple/foundationdb/record/query/plan/cascades/AggregateIndexExpansionVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Stream;

Expand Down Expand Up @@ -119,17 +120,18 @@ public boolean isPermuted() {
*
* @param baseQuantifierSupplier a quantifier supplier to create base data access
* @param ignored the primary key of the data object the caller wants to access, this parameter is ignored since
* an aggregate index does not possess primary key information, must be {@code null}.
* an aggregate index does not possess primary key information, must be {@code null}.
* @param isReverse an indicator whether the result set is expected to be returned in reverse order.
*
* @return A match candidate representing the aggregate index.
*/
@Nonnull
@Override
public MatchCandidate expand(@Nonnull final Supplier<Quantifier.ForEach> baseQuantifierSupplier,
public MatchCandidate expand(@Nonnull final Function<Optional<CorrelationIdentifier>, Quantifier.ForEach> baseQuantifierSupplier,
@Nullable final KeyExpression ignored,
final boolean isReverse) {
Verify.verify(ignored == null);
final var baseQuantifier = baseQuantifierSupplier.get();
final var baseQuantifier = baseQuantifierSupplier.apply(Optional.empty()); // todo

// 0. create a base expansion to resolve the key expression to columns with appropriate quantifiers
final var baseExpansion = constructBaseExpansion(baseQuantifier);
Expand All @@ -150,6 +152,8 @@ public MatchCandidate expand(@Nonnull final Supplier<Quantifier.ForEach> baseQua
// 3. construct SELECT-HAVING with SORT on top.
final var constructSelectHavingResult = constructSelectHaving(groupByQun, placeholders);
final var selectHaving = constructSelectHavingResult.getSelectExpression();

// todo, prepend the additional parameters.
final var placeHolderAliases = constructSelectHavingResult.getPlaceholderAliases();

// 4. add sort on top, if necessary, this will be absorbed later on as an ordering property of the match candidate.
Expand Down
11 changes: 7 additions & 4 deletions ...ore/src/main/java/com/apple/foundationdb/record/query/plan/cascades/ExpansionVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@

import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.util.function.Supplier;
import java.util.Optional;
import java.util.function.Function;

/**
* A sub interface of {@link KeyExpressionVisitor} that fixes the return type to be a {@link GraphExpansion} and
Expand All @@ -36,13 +37,15 @@
public interface ExpansionVisitor<S extends KeyExpressionVisitor.State> extends KeyExpressionVisitor<S, GraphExpansion> {
/**
* Method that expands a data structure into a data flow graph.
*
* @param baseQuantifierSupplier a quantifier supplier to create base data access
* @param primaryKey the primary key of the data object the caller wants to access
* @param isReverse an indicator whether the result set is expected to be returned in reverse order
*
* @return a new {@link MatchCandidate} that can be used for matching.
*/
@Nonnull
MatchCandidate expand(@Nonnull Supplier<Quantifier.ForEach> baseQuantifierSupplier,
@Nullable KeyExpression primaryKey,
boolean isReverse);
MatchCandidate expand(@Nonnull final Function<Optional<CorrelationIdentifier>, Quantifier.ForEach> baseQuantifierSupplier,
@Nullable final KeyExpression primaryKey,
final boolean isReverse);
}
22 changes: 22 additions & 0 deletions ...-core/src/main/java/com/apple/foundationdb/record/query/plan/cascades/GraphExpansion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import com.apple.foundationdb.record.query.plan.cascades.predicates.PredicateWithValueAndRanges;
import com.apple.foundationdb.record.query.plan.cascades.predicates.QueryPredicate;
import com.apple.foundationdb.record.query.plan.cascades.values.RecordConstructorValue;
import com.apple.foundationdb.record.query.plan.cascades.values.RecordTypeValue;
import com.apple.foundationdb.record.query.plan.cascades.values.Value;
import com.apple.foundationdb.record.util.pair.NonnullPair;
import com.google.common.base.Verify;
Expand All @@ -39,6 +40,7 @@
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import java.util.stream.IntStream;

Expand Down Expand Up @@ -512,6 +514,26 @@ public Builder addPredicate(@Nonnull final QueryPredicate predicate) {
return this;
}

@Nonnull
public Builder replacePlaceholder(@Nonnull final Function<Placeholder, Placeholder> replacementFunction) {
final ImmutableList<Placeholder> currentPlaceholders = placeholders.build();
placeholders = new ImmutableList.Builder<>();
for (final Placeholder placeholder : currentPlaceholders) {
placeholders.add(replacementFunction.apply(placeholder));
}
return this;
}

@Nonnull
public Builder replacePredicate(@Nonnull final Function<QueryPredicate, QueryPredicate> replacementFunction) {
final ImmutableList<QueryPredicate> currentPlaceholders = predicates.build();
predicates = new ImmutableList.Builder<>();
for (final QueryPredicate predicate : currentPlaceholders) {
predicates.add(replacementFunction.apply(predicate));
}
return this;
}

@Nonnull
public Builder addAllPredicates(@Nonnull final Iterable<? extends QueryPredicate> addPredicates) {
predicates.addAll(addPredicates);
Expand Down
27 changes: 17 additions & 10 deletions .../main/java/com/apple/foundationdb/record/query/plan/cascades/MatchCandidateExpansion.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,9 +99,9 @@ public static Optional<MatchCandidate> expandAggregateIndexMatchCandidate(@Nonnu
public static Optional<MatchCandidate> expandIndexMatchCandidate(@Nonnull IndexExpansionInfo info,
@Nullable KeyExpression commonPrimaryKey,
@Nonnull final ExpansionVisitor<?> expansionVisitor) {
final var baseRef = createBaseRef(info, new IndexAccessHint(info.getIndexName()));
try {
return Optional.of(expansionVisitor.expand(() -> Quantifier.forEach(baseRef), commonPrimaryKey, info.isReverse()));
return Optional.of(expansionVisitor.expand(aliasMaybe -> Quantifier.forEach(createBaseRef(info, aliasMaybe,
new IndexAccessHint(info.getIndexName()))), commonPrimaryKey, info.isReverse()));
} catch (final UnsupportedOperationException uOE) {
// just log and return empty
if (LOGGER.isDebugEnabled()) {
Expand All @@ -127,34 +127,41 @@ public static Optional<MatchCandidate> fromPrimaryDefinition(@Nonnull final Reco
.filter(recordType -> queriedRecordTypeNames.contains(recordType.getName()))
.collect(ImmutableList.toImmutableList());

final var baseRef = createBaseRef(metaData.getRecordTypes().keySet(), queriedRecordTypeNames, metaData.getPlannerType(queriedRecordTypeNames), new PrimaryAccessHint());
final var expansionVisitor = new PrimaryAccessExpansionVisitor(availableRecordTypes, queriedRecordTypes);
return Optional.of(expansionVisitor.expand(() -> Quantifier.forEach(baseRef), primaryKey, isReverse));
// I don't think we need to put the parameters here, but let's see if that's necessary.
return Optional.of(expansionVisitor.expand(aliasMaybe ->
Quantifier.forEach(createBaseRef(metaData.getRecordTypes().keySet(),
queriedRecordTypeNames, metaData.getPlannerType(queriedRecordTypeNames), aliasMaybe, new PrimaryAccessHint())), primaryKey, isReverse));
}

return Optional.empty();
}

@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
@Nonnull
private static Reference createBaseRef(@Nonnull IndexExpansionInfo info,
@Nonnull AccessHint accessHint) {
return createBaseRef(info.getAvailableRecordTypeNames(), info.getIndexedRecordTypeNames(), info.getBaseType(), accessHint);
@Nonnull final Optional<CorrelationIdentifier> recordTypeKeyAliasMaybe,
@Nonnull AccessHint accessHint) {
return createBaseRef(info.getAvailableRecordTypeNames(), info.getIndexedRecordTypeNames(), info.getBaseType(), recordTypeKeyAliasMaybe, accessHint);
}

@Nonnull
@SuppressWarnings("OptionalUsedAsFieldOrParameterType")
private static Reference createBaseRef(@Nonnull final Set<String> availableRecordTypeNames,
@Nonnull final Set<String> queriedRecordTypeNames,
@Nonnull final Type.Record baseType,
@Nonnull final Optional<CorrelationIdentifier> recordTypeKeyAliasMaybe,
@Nonnull AccessHint accessHint) {
final var quantifier =
Quantifier.forEach(
Reference.initialOf(
new FullUnorderedScanExpression(availableRecordTypeNames,
new Type.AnyRecord(false),
new AccessHints(accessHint))));
return Reference.initialOf(
new LogicalTypeFilterExpression(queriedRecordTypeNames,
quantifier,
baseType));
return Reference.initialOf(LogicalTypeFilterExpression.newInstanceForMatchCandidate(queriedRecordTypeNames,
availableRecordTypeNames,
quantifier,
recordTypeKeyAliasMaybe,
baseType));
}
}
39 changes: 33 additions & 6 deletions ...java/com/apple/foundationdb/record/query/plan/cascades/PrimaryAccessExpansionVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,15 @@

import com.apple.foundationdb.annotation.SpotBugsSuppressWarnings;
import com.apple.foundationdb.record.RecordCoreException;
import com.apple.foundationdb.record.metadata.Key;
import com.apple.foundationdb.record.metadata.RecordType;
import com.apple.foundationdb.record.metadata.expressions.KeyExpression;
import com.apple.foundationdb.record.query.plan.cascades.debug.Debugger;
import com.apple.foundationdb.record.query.plan.cascades.expressions.MatchableSortExpression;
import com.apple.foundationdb.record.query.plan.cascades.predicates.Placeholder;
import com.apple.foundationdb.record.query.plan.cascades.predicates.PredicateWithValueAndRanges;
import com.apple.foundationdb.record.query.plan.cascades.values.RecordTypeValue;
import com.apple.foundationdb.record.util.pair.NonnullPair;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;

Expand All @@ -35,11 +39,13 @@
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.function.Function;
import java.util.function.Supplier;

/**
* Class to expand primary data access into a candidate. The visitation methods are left unchanged from the super class
* {@link KeyExpressionExpansionVisitor}, this class merely provides a specific {@link #expand} method.
* {@link KeyExpressionExpansionVisitor}, this class merely provides a specific {@link ExpansionVisitor#expand} method.
*/
public class PrimaryAccessExpansionVisitor extends KeyExpressionExpansionVisitor implements ExpansionVisitor<KeyExpressionExpansionVisitor.VisitorState> {
@Nonnull
Expand All @@ -55,16 +61,22 @@ public PrimaryAccessExpansionVisitor(@Nonnull final Collection<RecordType> avail
@Nonnull
@Override
@SpotBugsSuppressWarnings("NP_PARAMETER_MUST_BE_NONNULL_BUT_MARKED_AS_NULLABLE")
public PrimaryScanMatchCandidate expand(@Nonnull final Supplier<Quantifier.ForEach> baseQuantifierSupplier,
public PrimaryScanMatchCandidate expand(@Nonnull final Function<Optional<CorrelationIdentifier>, Quantifier.ForEach> baseQuantifierSupplier,
@Nullable final KeyExpression primaryKey,
final boolean isReverse) {
Objects.requireNonNull(primaryKey);
Debugger.updateIndex(PredicateWithValueAndRanges.class, old -> 0);

final var baseQuantifier = baseQuantifierSupplier.get();
final Optional<CorrelationIdentifier> recordTypeQuantifierMaybe;
if (Key.Expressions.recordType().isPrefixKey(primaryKey)) {
recordTypeQuantifierMaybe = Optional.of(newParameterAlias());
} else {
recordTypeQuantifierMaybe = Optional.empty();
}
final Quantifier.ForEach baseQuantifier = baseQuantifierSupplier.apply(recordTypeQuantifierMaybe);

// expand
final var graphExpansion =
final var graphExpansionBuilder =
pop(primaryKey.expand(push(VisitorState.of(Lists.newArrayList(),
Lists.newArrayList(),
baseQuantifier,
Expand All @@ -74,8 +86,23 @@ public PrimaryScanMatchCandidate expand(@Nonnull final Supplier<Quantifier.ForEa
false,
true))))
.toBuilder()
.removeAllResultColumns()
.build();
.removeAllResultColumns();

recordTypeQuantifierMaybe.ifPresent(correlationIdentifier -> graphExpansionBuilder.replacePlaceholder(placeholder -> {
if (placeholder.getValue() instanceof RecordTypeValue) {
return placeholder.withAlias(correlationIdentifier);
}
return placeholder;
}));

recordTypeQuantifierMaybe.ifPresent(correlationIdentifier -> graphExpansionBuilder.replacePredicate(placeholder -> {
if (placeholder instanceof Placeholder && ((Placeholder)placeholder).getValue() instanceof RecordTypeValue) {
return ((Placeholder)placeholder).withAlias(correlationIdentifier);
}
return placeholder;
}));

final var graphExpansion = graphExpansionBuilder.build();

final var allExpansions =
GraphExpansion.ofOthers(GraphExpansion.ofQuantifier(baseQuantifier), graphExpansion);
Expand Down
9 changes: 5 additions & 4 deletions ...in/java/com/apple/foundationdb/record/query/plan/cascades/ValueIndexExpansionVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,14 +43,15 @@
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Supplier;
import java.util.function.Function;

import static com.apple.foundationdb.record.metadata.Key.Expressions.concat;

/**
* Class to expand value index access into a candidate graph. The visitation methods are left unchanged from the super
* class {@link KeyExpressionExpansionVisitor}, this class merely provides a specific {@link #expand} method.
* class {@link KeyExpressionExpansionVisitor}, this class merely provides a specific {@link ExpansionVisitor#expand} method.
*/
public class ValueIndexExpansionVisitor extends KeyExpressionExpansionVisitor implements ExpansionVisitor<KeyExpressionExpansionVisitor.VisitorState> {
// We may need to rethink this as it limits the set of indexes that can support a grouping key expression
Expand All @@ -76,12 +77,12 @@ public ValueIndexExpansionVisitor(@Nonnull Index index, @Nonnull Collection<Reco
@Nonnull
@Override
@SpotBugsSuppressWarnings("NP_PARAMETER_MUST_BE_NONNULL_BUT_MARKED_AS_NULLABLE")
public MatchCandidate expand(@Nonnull final Supplier<Quantifier.ForEach> baseQuantifierSupplier,
public MatchCandidate expand(@Nonnull final Function<Optional<CorrelationIdentifier>, Quantifier.ForEach> baseQuantifierSupplier,
@Nullable final KeyExpression primaryKey,
final boolean isReverse) {
Debugger.updateIndex(PredicateWithValueAndRanges.class, old -> 0);

final var baseQuantifier = baseQuantifierSupplier.get();
final var baseQuantifier = baseQuantifierSupplier.apply(Optional.empty()); // todo
final var allExpansionsBuilder = ImmutableList.<GraphExpansion>builder();

// add the value for the flow of records
Expand Down
9 changes: 6 additions & 3 deletions ...n/java/com/apple/foundationdb/record/query/plan/cascades/VectorIndexExpansionVisitor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@
import com.apple.foundationdb.record.query.plan.cascades.values.EuclideanDistanceRowNumberValue;
import com.apple.foundationdb.record.query.plan.cascades.values.EuclideanSquareDistanceRowNumberValue;
import com.apple.foundationdb.record.query.plan.cascades.values.Value;
import com.apple.foundationdb.record.util.pair.NonnullPair;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
Expand All @@ -50,12 +51,14 @@
import java.util.Collection;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
import java.util.function.Supplier;

/**
* Class to expand vector index access into a candidate graph. The visitation methods are left unchanged from the super
* class {@link KeyExpressionExpansionVisitor}, this class merely provides a specific {@link #expand} method.
* class {@link KeyExpressionExpansionVisitor}, this class merely provides a specific {@link ExpansionVisitor#expand} method.
*/
public class VectorIndexExpansionVisitor extends KeyExpressionExpansionVisitor implements ExpansionVisitor<KeyExpressionExpansionVisitor.VisitorState> {
@Nonnull
Expand All @@ -77,12 +80,12 @@ public VectorIndexExpansionVisitor(@Nonnull Index index, @Nonnull Collection<Rec
@Nonnull
@Override
@SpotBugsSuppressWarnings("NP_PARAMETER_MUST_BE_NONNULL_BUT_MARKED_AS_NULLABLE")
public MatchCandidate expand(@Nonnull final Supplier<Quantifier.ForEach> baseQuantifierSupplier,
public MatchCandidate expand(@Nonnull final Function<Optional<CorrelationIdentifier>, Quantifier.ForEach> baseQuantifierSupplier,
@Nullable final KeyExpression primaryKey,
final boolean isReverse) {
Debugger.updateIndex(PredicateWithValueAndRanges.class, old -> 0);

final var baseQuantifier = baseQuantifierSupplier.get();
final var baseQuantifier = baseQuantifierSupplier.apply(Optional.empty()); // todo
final var allExpansionsBuilder = ImmutableList.<GraphExpansion>builder();

allExpansionsBuilder.add(GraphExpansion.ofQuantifier(baseQuantifier));
Expand Down
Loading
Loading