Skip to content

NIFI-15829 - Support parameter value references to provided parameters from inherited parameter contexts #11130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pvillard31 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

NIFI-15829 - Support parameter value references to provided parameters from inherited parameter contexts #11130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 43 additions & 2 deletions .../nifi-framework-components/src/main/java/org/apache/nifi/groups/StandardProcessGroup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3320,13 +3320,54 @@ public void setParameterContext(final ParameterContext parameterContext) {
public void onParameterContextUpdated(final Map<String, ParameterUpdate> updatedParameters) {
readLock.lock();
try {
getProcessors().forEach(proc -> proc.onParametersModified(updatedParameters));
getControllerServices(false).forEach(cs -> cs.onParametersModified(updatedParameters));
final Map<String, ParameterUpdate> effectiveUpdates = augmentWithParameterValueReferences(updatedParameters);
getProcessors().forEach(proc -> proc.onParametersModified(effectiveUpdates));
getControllerServices(false).forEach(cs -> cs.onParametersModified(effectiveUpdates));
} finally {
readLock.unlock();
}
}

/**
* Augments the given parameter update map with entries for local parameters whose values are
* one-to-one references to changed parameters. For example, if this group's context defines
* parameter X with value {@code #{db_host}} and db_host is in the update map, then X is added
* to the augmented map with the same old/new values, allowing components referencing X to be
* properly notified of the change.
*/
private Map<String, ParameterUpdate> augmentWithParameterValueReferences(final Map<String, ParameterUpdate> updatedParameters) {
final ParameterContext context = getParameterContext();
if (context == null) {
return updatedParameters;
}

Map<String, ParameterUpdate> augmented = null;
for (final Map.Entry<ParameterDescriptor, Parameter> entry : context.getParameters().entrySet()) {
final Parameter localParam = entry.getValue();
final String referencedName = ParameterContext.extractOneToOneParameterReference(localParam.getValue());
if (referencedName == null) {
continue;
}

final Optional<Parameter> referencedParam = context.getParameter(referencedName);
if (referencedParam.isEmpty() || !referencedParam.get().isProvided()) {
continue;
}

final ParameterUpdate referencedUpdate = updatedParameters.get(referencedName);
if (referencedUpdate != null && localParam.getDescriptor().isSensitive() == referencedUpdate.isSensitive()) {
if (augmented == null) {
augmented = new HashMap<>(updatedParameters);
}
augmented.put(localParam.getDescriptor().getName(),
new StandardParameterUpdate(localParam.getDescriptor().getName(),
referencedUpdate.getPreviousValue(), referencedUpdate.getUpdatedValue(),
localParam.getDescriptor().isSensitive()));
}
}
return augmented != null ? augmented : updatedParameters;
}

private Map<String, ParameterUpdate> mapParameterUpdates(final ParameterContext previousParameterContext, final ParameterContext updatedParameterContext) {
if (previousParameterContext == null && updatedParameterContext == null) {
return Collections.emptyMap();
Expand Down
105 changes: 91 additions & 14 deletions ...ramework-components/src/main/java/org/apache/nifi/parameter/StandardParameterContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,9 @@ public Map<ParameterDescriptor, Parameter> getParameters() {
public Map<ParameterDescriptor, Parameter> getEffectiveParameters() {
readLock.lock();
try {
return this.getEffectiveParameters(inheritedParameterContexts);
final Map<ParameterDescriptor, Parameter> effective = getMergedEffectiveParameters(inheritedParameterContexts, this.parameters);
resolveParameterValueReferences(effective);
return effective;
} finally {
readLock.unlock();
}
Expand All @@ -358,7 +360,8 @@ public Map<String, Parameter> getEffectiveParameterUpdates(final Map<String, Par
Objects.requireNonNull(inheritedParameterContexts, "Inherited parameter contexts must be specified");

final Map<ParameterDescriptor, Parameter> currentEffectiveParameters = getEffectiveParameters();
final Map<ParameterDescriptor, Parameter> effectiveProposedParameters = getEffectiveParameters(inheritedParameterContexts, getProposedParameters(parameterUpdates));
final Map<ParameterDescriptor, Parameter> effectiveProposedParameters = getMergedEffectiveParameters(inheritedParameterContexts, getProposedParameters(parameterUpdates));
resolveParameterValueReferences(effectiveProposedParameters);

return getEffectiveParameterUpdates(currentEffectiveParameters, effectiveProposedParameters);
}
Expand All @@ -370,7 +373,9 @@ public Map<String, Parameter> getEffectiveParameterUpdates(final Map<String, Par
* @return The view of the parameters with all overriding applied
*/
private Map<ParameterDescriptor, Parameter> getEffectiveParameters(final Map<ParameterDescriptor, Parameter> proposedParameters) {
return getEffectiveParameters(this.inheritedParameterContexts, proposedParameters);
final Map<ParameterDescriptor, Parameter> effective = getMergedEffectiveParameters(this.inheritedParameterContexts, proposedParameters);
resolveParameterValueReferences(effective);
return effective;
}

/**
Expand All @@ -380,31 +385,103 @@ private Map<ParameterDescriptor, Parameter> getEffectiveParameters(final Map<Par
* @return The view of the parameters with all overriding applied
*/
private Map<ParameterDescriptor, Parameter> getEffectiveParameters(final List<ParameterContext> parameterContexts) {
return getEffectiveParameters(parameterContexts, this.parameters);
final Map<ParameterDescriptor, Parameter> effective = getMergedEffectiveParameters(parameterContexts, this.parameters);
resolveParameterValueReferences(effective);
return effective;
}

private Map<ParameterDescriptor, Parameter> getEffectiveParameters(final List<ParameterContext> parameterContexts,
final Map<ParameterDescriptor, Parameter> proposedParameters) {
return getEffectiveParameters(parameterContexts, proposedParameters, new HashMap<>());
private Map<ParameterDescriptor, Parameter> getMergedEffectiveParameters(final List<ParameterContext> parameterContexts,
final Map<ParameterDescriptor, Parameter> proposedParameters) {
return getMergedEffectiveParameters(parameterContexts, proposedParameters, new HashMap<>());
}

private Map<ParameterDescriptor, Parameter> getEffectiveParameters(final List<ParameterContext> parameterContexts,
final Map<ParameterDescriptor, Parameter> proposedParameters,
final Map<ParameterDescriptor, List<Parameter>> allOverrides) {
/**
* Merges parameters from inherited contexts with the proposed (local) parameters, applying
* override priority. Does NOT resolve parameter value references -- callers that need resolved
* values must call {@link #resolveParameterValueReferences} on the result.
*/
private Map<ParameterDescriptor, Parameter> getMergedEffectiveParameters(final List<ParameterContext> parameterContexts,
final Map<ParameterDescriptor, Parameter> proposedParameters,
final Map<ParameterDescriptor, List<Parameter>> allOverrides) {
final Map<ParameterDescriptor, Parameter> effectiveParameters = new LinkedHashMap<>();

// Loop backwards so that the first ParameterContext in the list will override any parameters later in the list
for (int i = parameterContexts.size() - 1; i >= 0; i--) {
ParameterContext parameterContext = parameterContexts.get(i);
combineOverrides(allOverrides, overrideParameters(effectiveParameters, parameterContext.getEffectiveParameters(), parameterContext));
final ParameterContext parameterContext = parameterContexts.get(i);
final Map<ParameterDescriptor, Parameter> inheritedParameters = getUnresolvedEffectiveParameters(parameterContext);
combineOverrides(allOverrides, overrideParameters(effectiveParameters, inheritedParameters, parameterContext));
}

// Finally, override all child parameters with our own
combineOverrides(allOverrides, overrideParameters(effectiveParameters, proposedParameters, this));

return effectiveParameters;
}

/**
* Returns the merged effective parameters from a context without applying parameter value
* reference resolution. For StandardParameterContext instances this avoids double-resolution
* when building a parent context's effective parameter set.
*/
private static Map<ParameterDescriptor, Parameter> getUnresolvedEffectiveParameters(final ParameterContext parameterContext) {
if (parameterContext instanceof StandardParameterContext standardContext) {
return standardContext.getMergedEffectiveParametersReadLocked();
}
return parameterContext.getEffectiveParameters();
}

private Map<ParameterDescriptor, Parameter> getMergedEffectiveParametersReadLocked() {
readLock.lock();
try {
return getMergedEffectiveParameters(inheritedParameterContexts, this.parameters);
} finally {
readLock.unlock();
}
}

/**
* Resolves one-to-one parameter value references within the effective parameter map.
* If a parameter's entire value is exactly {@code #{referencedName}}, and the referenced parameter
* exists in the effective map, is provided by a parameter provider, and has matching sensitivity,
* the value is replaced with the referenced parameter's value. Only a single level of resolution
* is performed (no chaining): the lookup uses a snapshot of the pre-resolution values so that
* transitive references are not followed.
*
* @param effectiveParameters the effective parameter map to resolve in place
*/
private void resolveParameterValueReferences(final Map<ParameterDescriptor, Parameter> effectiveParameters) {
final Map<String, Parameter> originalParametersByName = new HashMap<>();
for (final Map.Entry<ParameterDescriptor, Parameter> entry : effectiveParameters.entrySet()) {
originalParametersByName.put(entry.getKey().getName(), entry.getValue());
}

for (final Map.Entry<ParameterDescriptor, Parameter> entry : effectiveParameters.entrySet()) {
final ParameterDescriptor descriptor = entry.getKey();
final Parameter parameter = entry.getValue();
final String referencedName = ParameterContext.extractOneToOneParameterReference(parameter.getValue());
if (referencedName == null) {
continue;
}

final Parameter referencedParameter = originalParametersByName.get(referencedName);
if (referencedParameter == null) {
continue;
}

if (!referencedParameter.isProvided()) {
continue;
}

if (descriptor.isSensitive() != referencedParameter.getDescriptor().isSensitive()) {
continue;
}

final Parameter resolvedParameter = new Parameter.Builder()
.fromParameter(parameter)
.value(referencedParameter.getValue())
.build();
entry.setValue(resolvedParameter);
}
}

private void combineOverrides(final Map<ParameterDescriptor, List<Parameter>> existingOverrides, final Map<ParameterDescriptor, List<Parameter>> newOverrides) {
for (final Map.Entry<ParameterDescriptor, List<Parameter>> entry : newOverrides.entrySet()) {
final ParameterDescriptor key = entry.getKey();
Expand Down
Loading
Loading