diff --git a/core/src/main/java/org/apache/hop/core/Const.java b/core/src/main/java/org/apache/hop/core/Const.java index 7394cd4b334..fa25fec12dc 100644 --- a/core/src/main/java/org/apache/hop/core/Const.java +++ b/core/src/main/java/org/apache/hop/core/Const.java @@ -1291,6 +1291,237 @@ public static long toLong(String str, long def) { return retval; } + /** + * Pattern for scientific / power-of-ten integer forms: {@code 1e8}, {@code 1.5E+6}, {@code + * 1x10^8}, {@code 1×10^8}, {@code 10^8}. + */ + private static final Pattern EXPANDED_SCIENTIFIC_PATTERN = + Pattern.compile( + "^([+-]?)(?:([0-9][0-9_ .,]*[0-9]|[0-9])\\s*[x×]\\s*10\\s*\\^\\s*|([0-9][0-9_ .,]*)\\s*[eE]|10\\s*\\^\\s*)([+-]?\\d+)$"); + + /** Pattern for optional trailing magnitude suffix: k / m / g / b (case-insensitive). */ + private static final Pattern EXPANDED_SUFFIX_PATTERN = + Pattern.compile("^([+-]?)(.*?)\\s*([kKmMgGbB])$"); + + /** + * Expand a human-friendly integer string into a plain digit form (optional leading sign). + * + *

Supports: + * + *

+ * + * @param str the input string + * @return expanded digit string (with optional leading {@code -}), or {@code null} if the input + * cannot be expanded to an integer + */ + public static String expandIntegerString(String str) { + if (str == null) { + return null; + } + String trimmed = str.trim(); + if (trimmed.isEmpty()) { + return null; + } + + try { + // Scientific / power-of-ten first (e.g. 1e8, 1x10^8, 10^8) + Matcher sci = EXPANDED_SCIENTIFIC_PATTERN.matcher(trimmed); + if (sci.matches()) { + String sign = sci.group(1); + String coeffGroup = sci.group(2) != null ? sci.group(2) : sci.group(3); + String expGroup = sci.group(4); + BigDecimal coefficient; + if (coeffGroup == null || coeffGroup.isEmpty()) { + // Form: 10^N + coefficient = BigDecimal.ONE; + } else { + coefficient = parseExpandedCoefficient(coeffGroup, true); + } + int exp = Integer.parseInt(expGroup); + BigDecimal value = coefficient.multiply(BigDecimal.TEN.pow(exp)); + return formatExpandedLong(sign, value); + } + + // Trailing magnitude suffix: k / m / g / b + Matcher suffixMatcher = EXPANDED_SUFFIX_PATTERN.matcher(trimmed); + if (suffixMatcher.matches()) { + String sign = suffixMatcher.group(1); + String body = suffixMatcher.group(2).trim(); + char suffix = Character.toLowerCase(suffixMatcher.group(3).charAt(0)); + if (body.isEmpty()) { + return null; + } + BigDecimal coefficient = parseExpandedCoefficient(body, true); + long multiplier; + switch (suffix) { + case 'k': + multiplier = 1_000L; + break; + case 'm': + multiplier = 1_000_000L; + break; + case 'g': + case 'b': + multiplier = 1_000_000_000L; + break; + default: + return null; + } + BigDecimal value = coefficient.multiply(BigDecimal.valueOf(multiplier)); + return formatExpandedLong(sign, value); + } + + // Plain integer with optional grouping separators + String sign = ""; + String body = trimmed; + if (body.startsWith("+") || body.startsWith("-")) { + sign = body.startsWith("-") ? "-" : ""; + body = body.substring(1).trim(); + } + body = stripIntegerGrouping(body); + if (body.isEmpty() || !body.chars().allMatch(Character::isDigit)) { + return null; + } + // Normalize leading zeros via Long parse (overflows → BigInteger path not needed for plain) + long value = Long.parseLong(body); + return sign + Long.toString(value); + } catch (Exception e) { + return null; + } + } + + /** + * Parse a coefficient that may contain grouping separators and at most one decimal separator. + * + * @param body coefficient text without sign or magnitude suffix + * @param allowDecimal whether a single decimal point/comma is allowed + */ + private static BigDecimal parseExpandedCoefficient(String body, boolean allowDecimal) { + String s = body.trim(); + // Remove spaces and underscores always + s = s.replace(" ", "").replace("_", ""); + + int lastDot = s.lastIndexOf('.'); + int lastComma = s.lastIndexOf(','); + boolean hasDot = lastDot >= 0; + boolean hasComma = lastComma >= 0; + + if (allowDecimal && (hasDot || hasComma)) { + // Decide which separator is decimal: the last of '.' or ',' + int decimalPos = Math.max(lastDot, lastComma); + String intPart = s.substring(0, decimalPos).replace(",", "").replace(".", ""); + String fracPart = s.substring(decimalPos + 1).replace(",", "").replace(".", ""); + if (intPart.isEmpty()) { + intPart = "0"; + } + if (!intPart.chars().allMatch(Character::isDigit) + || !fracPart.chars().allMatch(Character::isDigit)) { + throw new NumberFormatException("Invalid coefficient: " + body); + } + return new BigDecimal(intPart + "." + fracPart); + } + + s = stripIntegerGrouping(s); + if (s.isEmpty() || !s.chars().allMatch(Character::isDigit)) { + throw new NumberFormatException("Invalid coefficient: " + body); + } + return new BigDecimal(s); + } + + /** + * Strip thousands grouping from a digit string. Removes commas always; removes dots only when + * they form thousands groups of three digits (e.g. {@code 100.000.000}). + */ + private static String stripIntegerGrouping(String body) { + String s = body.replace(" ", "").replace("_", ""); + if (s.indexOf(',') >= 0) { + s = s.replace(",", ""); + } + if (s.indexOf('.') >= 0) { + // Thousands pattern: digits with groups of 3 after each dot, no other characters + if (s.matches("\\d{1,3}(\\.\\d{3})+")) { + s = s.replace(".", ""); + } else if (s.chars().filter(c -> c == '.').count() > 1) { + // Multiple dots that are not a clean thousands pattern → invalid later + return s; + } else { + // Single dot without suffix path: treat as invalid for pure integers + // (leave the dot so digit check fails) + } + } + return s; + } + + private static String formatExpandedLong(String sign, BigDecimal value) { + // Truncate toward zero for fractional results (e.g. 1.5m is exact; 1e-1 would become 0) + BigDecimal truncated = value.setScale(0, RoundingMode.DOWN); + if (truncated.compareTo(BigDecimal.valueOf(Long.MAX_VALUE)) > 0 + || truncated.compareTo(BigDecimal.valueOf(Long.MIN_VALUE)) < 0) { + throw new NumberFormatException("Expanded value out of long range"); + } + long longValue = truncated.longValueExact(); + if ("-".equals(sign)) { + // Avoid Long.MIN_VALUE negation issues; value should already be positive magnitude + if (longValue == Long.MIN_VALUE) { + throw new NumberFormatException("Expanded value out of long range"); + } + return Long.toString(-longValue); + } + return Long.toString(longValue); + } + + /** + * Convert a human-friendly integer String into a {@code long}. If conversion fails, return the + * default. See {@link #expandIntegerString(String)} for accepted forms. + * + * @param str The String to convert + * @param def The default value + * @return The converted value or the default + */ + public static long toLongExpanded(String str, long def) { + try { + String expanded = expandIntegerString(str); + if (expanded == null) { + return def; + } + return Long.parseLong(expanded); + } catch (Exception e) { + return def; + } + } + + /** + * Convert a human-friendly integer String into an {@code int}. If conversion fails or the value + * is outside the {@code int} range, return the default. See {@link #expandIntegerString(String)} + * for accepted forms. + * + * @param str The String to convert + * @param def The default value + * @return The converted value or the default + */ + public static int toIntExpanded(String str, int def) { + try { + String expanded = expandIntegerString(str); + if (expanded == null) { + return def; + } + long value = Long.parseLong(expanded); + if (value < Integer.MIN_VALUE || value > Integer.MAX_VALUE) { + return def; + } + return (int) value; + } catch (Exception e) { + return def; + } + } + /** * Convert a String into a double. If the conversion fails, assign a default value. * diff --git a/core/src/test/java/org/apache/hop/core/ConstTest.java b/core/src/test/java/org/apache/hop/core/ConstTest.java index 787a3dd951c..f94a6bb1705 100644 --- a/core/src/test/java/org/apache/hop/core/ConstTest.java +++ b/core/src/test/java/org/apache/hop/core/ConstTest.java @@ -3639,6 +3639,64 @@ void testToLong() { assertEquals(-1447252914241L, Const.toLong("1447252914241L", -1447252914241L)); } + @Test + void testExpandIntegerString() { + assertEquals("123", Const.expandIntegerString("123")); + assertEquals("100000000", Const.expandIntegerString("100,000,000")); + assertEquals("100000000", Const.expandIntegerString("100.000.000")); + assertEquals("100000000", Const.expandIntegerString("100_000_000")); + assertEquals("100000000", Const.expandIntegerString("100 000 000")); + assertEquals("100000", Const.expandIntegerString("100k")); + assertEquals("100000", Const.expandIntegerString("100K")); + assertEquals("100000000", Const.expandIntegerString("100m")); + assertEquals("1000000000", Const.expandIntegerString("1g")); + assertEquals("1000000000", Const.expandIntegerString("1b")); + assertEquals("1500000", Const.expandIntegerString("1.5m")); + assertEquals("1500000", Const.expandIntegerString("1,5m")); + assertEquals("100000000", Const.expandIntegerString("1e8")); + assertEquals("100000000", Const.expandIntegerString("1E8")); + assertEquals("1500000", Const.expandIntegerString("1.5e6")); + assertEquals("100000000", Const.expandIntegerString("1x10^8")); + assertEquals("100000000", Const.expandIntegerString("1×10^8")); + assertEquals("100000000", Const.expandIntegerString("10^8")); + assertEquals("-1000000", Const.expandIntegerString("-1m")); + assertNull(Const.expandIntegerString(null)); + assertNull(Const.expandIntegerString("")); + assertNull(Const.expandIntegerString("abc")); + assertNull(Const.expandIntegerString("1z")); + } + + @Test + void testToLongExpanded() { + assertEquals(123L, Const.toLongExpanded("123", -12)); + assertEquals(100_000_000L, Const.toLongExpanded("100,000,000", -1)); + assertEquals(100_000_000L, Const.toLongExpanded("100.000.000", -1)); + assertEquals(100_000_000L, Const.toLongExpanded("100m", -1)); + assertEquals(1_500_000L, Const.toLongExpanded("1.5m", -1)); + assertEquals(100_000_000L, Const.toLongExpanded("1e8", -1)); + assertEquals(100_000_000L, Const.toLongExpanded("1x10^8", -1)); + assertEquals(100_000_000L, Const.toLongExpanded("10^8", -1)); + assertEquals(1_000_000_000L, Const.toLongExpanded("1b", -1)); + assertEquals(-1_000_000L, Const.toLongExpanded("-1m", -1)); + assertEquals(-12L, Const.toLongExpanded("123f", -12)); + assertEquals(-12L, Const.toLongExpanded(null, -12)); + assertEquals(-12L, Const.toLongExpanded("", -12)); + // Plain digit strings match toLong + assertEquals(1447252914241L, Const.toLongExpanded("1447252914241", -12)); + } + + @Test + void testToIntExpanded() { + assertEquals(123, Const.toIntExpanded("123", -12)); + assertEquals(100_000, Const.toIntExpanded("100k", -12)); + assertEquals(100_000_000, Const.toIntExpanded("100m", -12)); + assertEquals(-12, Const.toIntExpanded("123f", -12)); + assertEquals(-12, Const.toIntExpanded(null, -12)); + // Overflow past Integer.MAX_VALUE returns default + assertEquals(-12, Const.toIntExpanded("3g", -12)); + assertEquals(-12, Const.toIntExpanded("10^12", -12)); + } + @Test void testToDouble() { Assertions.assertEquals(123.45, Const.toDouble("123.45", -12.34), 1e-15); diff --git a/docker/Dockerfile b/docker/Dockerfile index 9a1f3d37eb5..a83a3c302e7 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -42,6 +42,10 @@ ENV HOP_PROJECT_DIRECTORY= ENV HOP_PROJECT_FOLDER= # name of the project config file including file extension ENV HOP_PROJECT_CONFIG_FILE_NAME=project-config.json +# optional one-level parent project (see issue #2596) +ENV HOP_PARENT_PROJECT_NAME= +ENV HOP_PARENT_PROJECT_FOLDER= +ENV HOP_PARENT_PROJECT_CONFIG_FILE_NAME=project-config.json # environment to use with hop run ENV HOP_ENVIRONMENT_NAME=environment1 # comma separated list of paths to environment config files (including filename and file extension). diff --git a/docker/resources/load-and-execute.sh b/docker/resources/load-and-execute.sh index fa754fa6dba..91fc3e23a89 100755 --- a/docker/resources/load-and-execute.sh +++ b/docker/resources/load-and-execute.sh @@ -188,6 +188,36 @@ if [ -n "${HOP_SYSTEM_PROPERTIES}" ]; then HOP_EXEC_OPTIONS+=("--system-properties=${HOP_SYSTEM_PROPERTIES}") fi +# Register a project in hop-config if it is not already present. +# Usage: register_hop_project NAME FOLDER CONFIG_FILE [PARENT_NAME] +# +register_hop_project() { + local name="$1" + local home="$2" + local cfg="$3" + local parent="${4:-}" + + if $("${DEPLOYMENT_PATH}"/hop-conf.sh -pl | grep -q -E "^ ${name} :"); then + log "project ${name} already exists" + return 0 + fi + + local conf_args=( + --project="${name}" + --project-create + --project-home="${home}" + --project-config-file="${cfg}" + --project-keep-config-file + ) + if [ -n "${parent}" ]; then + conf_args+=(--project-parent="${parent}") + fi + + log "Registering project ${name} in the Hop container configuration (home=${home})" + log "${DEPLOYMENT_PATH}/hop-conf.sh ${conf_args[*]}" + "${DEPLOYMENT_PATH}"/hop-conf.sh "${conf_args[@]}" +} + # If a project folder is defined we assume that we want to create it in the container # if [ -n "${HOP_PROJECT_FOLDER}" ]; then @@ -208,20 +238,33 @@ if [ -n "${HOP_PROJECT_FOLDER}" ]; then log "The specified project folder exists" fi - log "Registering project ${HOP_PROJECT_NAME} in the Hop container configuration" - log "${DEPLOYMENT_PATH}/hop-conf.sh --project=${HOP_PROJECT_NAME} --project-create --project-home='${HOP_PROJECT_FOLDER}' --project-config-file='${HOP_PROJECT_CONFIG_FILE_NAME}' --project-keep-config-file" - - if $("${DEPLOYMENT_PATH}"/hop-conf.sh -pl | grep -q -E "^ ${HOP_PROJECT_NAME} :"); then - log "project ${HOP_PROJECT_NAME} already exists" - else - "${DEPLOYMENT_PATH}"/hop-conf.sh \ - --project="${HOP_PROJECT_NAME}" \ - --project-create \ - --project-home="${HOP_PROJECT_FOLDER}" \ - --project-config-file="${HOP_PROJECT_CONFIG_FILE_NAME}" \ - --project-keep-config-file + # Optional one-level parent project (issue #2596). Register the parent first so + # metadata inheritance and PARENT_PROJECT_HOME resolve when the child is enabled. + # + HOP_PARENT_PROJECT_CONFIG_FILE_NAME="${HOP_PARENT_PROJECT_CONFIG_FILE_NAME:-project-config.json}" + if [ -n "${HOP_PARENT_PROJECT_FOLDER}" ] || [ -n "${HOP_PARENT_PROJECT_NAME}" ]; then + if [ -z "${HOP_PARENT_PROJECT_FOLDER}" ] || [ -z "${HOP_PARENT_PROJECT_NAME}" ]; then + log "Error: both HOP_PARENT_PROJECT_NAME and HOP_PARENT_PROJECT_FOLDER must be set to register a parent project" + exitWithCode 9 + fi + if [ ! -d "${HOP_PARENT_PROJECT_FOLDER}" ]; then + log "Warning: the folder specified in variable HOP_PARENT_PROJECT_FOLDER does not exist in the container: ${HOP_PARENT_PROJECT_FOLDER}" + else + log "The specified parent project folder exists" + fi + register_hop_project \ + "${HOP_PARENT_PROJECT_NAME}" \ + "${HOP_PARENT_PROJECT_FOLDER}" \ + "${HOP_PARENT_PROJECT_CONFIG_FILE_NAME}" fi + HOP_PROJECT_CONFIG_FILE_NAME="${HOP_PROJECT_CONFIG_FILE_NAME:-project-config.json}" + register_hop_project \ + "${HOP_PROJECT_NAME}" \ + "${HOP_PROJECT_FOLDER}" \ + "${HOP_PROJECT_CONFIG_FILE_NAME}" \ + "${HOP_PARENT_PROJECT_NAME:-}" + HOP_EXEC_OPTIONS+=("--project=${HOP_PROJECT_NAME}") # If we have environment files specified we want to create an environment as well: diff --git a/docker/resources/run-web.sh b/docker/resources/run-web.sh index be86ae7e612..0308838341a 100755 --- a/docker/resources/run-web.sh +++ b/docker/resources/run-web.sh @@ -80,6 +80,36 @@ install_jdbc_drivers() { # HOP_EXEC_OPTIONS="--level=${HOP_LOG_LEVEL}" +# Register a project in hop-config if it is not already present. +# Usage: register_hop_project NAME FOLDER CONFIG_FILE [PARENT_NAME] +# +register_hop_project() { + local name="$1" + local home="$2" + local cfg="$3" + local parent="${4:-}" + + if $("${DEPLOYMENT_PATH}"/hop-conf.sh -pl | grep -q -E "^ ${name} :"); then + log "project ${name} already exists" + return 0 + fi + + local conf_args=( + --project="${name}" + --project-create + --project-home="${home}" + --project-config-file="${cfg}" + --project-keep-config-file + ) + if [ -n "${parent}" ]; then + conf_args+=(--project-parent="${parent}") + fi + + log "Registering project ${name} in the Hop container configuration (home=${home})" + log "${DEPLOYMENT_PATH}/hop-conf.sh ${conf_args[*]}" + "${DEPLOYMENT_PATH}"/hop-conf.sh "${conf_args[@]}" +} + # If a project folder is defined we assume that we want to create it in the container # if [ -n "${HOP_PROJECT_FOLDER}" ]; then @@ -101,19 +131,34 @@ if [ -n "${HOP_PROJECT_FOLDER}" ]; then log "The specified project folder exists" fi - log "Registering project ${HOP_PROJECT_NAME} in the Hop container configuration" - log "${DEPLOYMENT_PATH}/hop-conf.sh --project=${HOP_PROJECT_NAME} --project-create --project-home='${HOP_PROJECT_FOLDER}' --project-config-file='${HOP_PROJECT_CONFIG_FILE_NAME}'" - - if $("${DEPLOYMENT_PATH}"/hop-conf.sh -pl | grep -q -E "^ ${HOP_PROJECT_NAME} :"); then - log "project ${HOP_PROJECT_NAME} already exists" - else - "${DEPLOYMENT_PATH}"/hop-conf.sh \ - --project="${HOP_PROJECT_NAME}" \ - --project-create \ - --project-home="${HOP_PROJECT_FOLDER}" \ - --project-config-file="${HOP_PROJECT_CONFIG_FILE_NAME}" + # Optional one-level parent project (issue #2596). Register the parent first so + # metadata inheritance and PARENT_PROJECT_HOME resolve when the child is enabled. + # + HOP_PARENT_PROJECT_CONFIG_FILE_NAME="${HOP_PARENT_PROJECT_CONFIG_FILE_NAME:-project-config.json}" + if [ -n "${HOP_PARENT_PROJECT_FOLDER}" ] || [ -n "${HOP_PARENT_PROJECT_NAME}" ]; then + if [ -z "${HOP_PARENT_PROJECT_FOLDER}" ] || [ -z "${HOP_PARENT_PROJECT_NAME}" ]; then + log "Error: both HOP_PARENT_PROJECT_NAME and HOP_PARENT_PROJECT_FOLDER must be set to register a parent project" + exitWithCode 9 + fi + if [ ! -d "${HOP_PARENT_PROJECT_FOLDER}" ]; then + log "Error: the folder specified in variable HOP_PARENT_PROJECT_FOLDER does not exist: ${HOP_PARENT_PROJECT_FOLDER}" + exitWithCode 9 + else + log "The specified parent project folder exists" + fi + register_hop_project \ + "${HOP_PARENT_PROJECT_NAME}" \ + "${HOP_PARENT_PROJECT_FOLDER}" \ + "${HOP_PARENT_PROJECT_CONFIG_FILE_NAME}" fi + HOP_PROJECT_CONFIG_FILE_NAME="${HOP_PROJECT_CONFIG_FILE_NAME:-project-config.json}" + register_hop_project \ + "${HOP_PROJECT_NAME}" \ + "${HOP_PROJECT_FOLDER}" \ + "${HOP_PROJECT_CONFIG_FILE_NAME}" \ + "${HOP_PARENT_PROJECT_NAME:-}" + HOP_EXEC_OPTIONS="${HOP_EXEC_OPTIONS} --project=${HOP_PROJECT_NAME}" # If we have environment files specified we want to create an environment as well: diff --git a/docker/unified.Dockerfile b/docker/unified.Dockerfile index e2390351010..4fdc6d5d018 100644 --- a/docker/unified.Dockerfile +++ b/docker/unified.Dockerfile @@ -258,6 +258,9 @@ ENV HOP_PROJECT_NAME= ENV HOP_PROJECT_DIRECTORY= ENV HOP_PROJECT_FOLDER= ENV HOP_PROJECT_CONFIG_FILE_NAME=project-config.json +ENV HOP_PARENT_PROJECT_NAME= +ENV HOP_PARENT_PROJECT_FOLDER= +ENV HOP_PARENT_PROJECT_CONFIG_FILE_NAME=project-config.json ENV HOP_ENVIRONMENT_NAME= ENV HOP_ENVIRONMENT_CONFIG_FILE_NAME_PATHS= ENV HOP_RUN_CONFIG= @@ -334,6 +337,9 @@ ENV HOP_DRIVERS_MAVEN_REPO= ENV HOP_GUI_ZOOM_FACTOR=1.0 ENV HOP_PROJECT_FOLDER= ENV HOP_PROJECT_CONFIG_FILE_NAME=project-config.json +ENV HOP_PARENT_PROJECT_NAME= +ENV HOP_PARENT_PROJECT_FOLDER= +ENV HOP_PARENT_PROJECT_CONFIG_FILE_NAME=project-config.json ENV HOP_ENVIRONMENT_NAME=environment1 ENV HOP_ENVIRONMENT_CONFIG_FILE_NAME_PATHS= diff --git a/docker/web.Dockerfile b/docker/web.Dockerfile index 7dfd19c72bf..70fed1a1d97 100644 --- a/docker/web.Dockerfile +++ b/docker/web.Dockerfile @@ -43,6 +43,10 @@ ENV HOP_GUI_ZOOM_FACTOR=1.0 ENV HOP_PROJECT_FOLDER= # name of the project config file including file extension ENV HOP_PROJECT_CONFIG_FILE_NAME=project-config.json +# optional one-level parent project (see issue #2596) +ENV HOP_PARENT_PROJECT_NAME= +ENV HOP_PARENT_PROJECT_FOLDER= +ENV HOP_PARENT_PROJECT_CONFIG_FILE_NAME=project-config.json # environment to use with hop run ENV HOP_ENVIRONMENT_NAME=environment1 # comma separated list of paths to environment config files (including filename and file extension). diff --git a/docs/hop-tech-manual/modules/ROOT/pages/docker-container.adoc b/docs/hop-tech-manual/modules/ROOT/pages/docker-container.adoc index 49bd5d46e6b..156f591e0a7 100644 --- a/docs/hop-tech-manual/modules/ROOT/pages/docker-container.adoc +++ b/docs/hop-tech-manual/modules/ROOT/pages/docker-container.adoc @@ -86,6 +86,22 @@ If you do not set this variable, no project or environment will be created. |`project-config.json` | Name of the project config file. +|```HOP_PARENT_PROJECT_NAME``` +| +| Optional name of a one-level parent project to register in the container before the main project. +Required together with `HOP_PARENT_PROJECT_FOLDER` when the main project inherits metadata from another project (`parentProjectName` in `project-config.json`). +Only a single parent level is supported in the image; deeper chains need a custom entrypoint. + +|```HOP_PARENT_PROJECT_FOLDER``` +| +| Path to the home of the parent Hop project inside the container. +Required together with `HOP_PARENT_PROJECT_NAME`. +Mount both parent and child project folders into the container. + +|```HOP_PARENT_PROJECT_CONFIG_FILE_NAME``` +|`project-config.json` +| Name of the parent project config file (relative to the parent home). + |```HOP_ENVIRONMENT_NAME``` | | The name of the Hop environment to create in the container. @@ -292,6 +308,34 @@ docker run -it --rm \ apache/hop: ---- +=== Projects with a parent + +When a project inherits metadata from another project (`parentProjectName` in `project-config.json`), both project homes must be mounted and the parent must be registered in the container as well. + +Set `HOP_PARENT_PROJECT_NAME` and `HOP_PARENT_PROJECT_FOLDER` (optionally `HOP_PARENT_PROJECT_CONFIG_FILE_NAME`). +The entrypoint registers the parent first, then the main project with `--project-parent`, so parent metadata is merged into `HOP_METADATA_FOLDER` and the built-in variables `PARENT_PROJECT_HOME` / `PARENT_PROJECT_NAME` resolve correctly. + +Only **one parent level** is supported through these environment variables. + +[source,bash] +---- +docker run -it --rm \ + --env HOP_LOG_LEVEL=Basic \ + --env HOP_FILE_PATH='jobs/load_projectB_data.hwf' \ + --env HOP_PROJECT_FOLDER=/files/projects/projectB \ + --env HOP_PROJECT_NAME=projectB \ + --env HOP_PARENT_PROJECT_FOLDER=/files/projects/projectA \ + --env HOP_PARENT_PROJECT_NAME=projectA \ + --env HOP_ENVIRONMENT_NAME=prod_docker_exec \ + --env HOP_ENVIRONMENT_CONFIG_FILE_NAME_PATHS=/files/projects/projectB/prod_docker_exec-config.json \ + --env HOP_RUN_CONFIG=docker_run_config \ + -v /path/to/projects:/files/projects \ + --name docker-exec-projectB \ + apache/hop: +---- + +The value of `HOP_PARENT_PROJECT_NAME` should match `parentProjectName` in the child project's `project-config.json`. + === Running the long-lived container (server) If you need a **long-lived container**, this option is also available. diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/database/generate-variables-proposals.png b/docs/hop-user-manual/modules/ROOT/assets/images/database/generate-variables-proposals.png new file mode 100644 index 00000000000..f2c6c95f284 Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/database/generate-variables-proposals.png differ diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/database/generate-variables-save-as-env-config-file-question.png b/docs/hop-user-manual/modules/ROOT/assets/images/database/generate-variables-save-as-env-config-file-question.png new file mode 100644 index 00000000000..6afae6cc977 Binary files /dev/null and b/docs/hop-user-manual/modules/ROOT/assets/images/database/generate-variables-save-as-env-config-file-question.png differ diff --git a/docs/hop-user-manual/modules/ROOT/pages/database/databases.adoc b/docs/hop-user-manual/modules/ROOT/pages/database/databases.adoc index 4d3f0dfc710..238df15b292 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/database/databases.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/database/databases.adoc @@ -18,7 +18,10 @@ under the License. :imagesdir: ../../assets/images :openvar: ${ :closevar: } -:description: Hop supports tens of databases out of the box. If your preferred database has no specific support, you can probably still connect through a generic database connection. +:description: Documentation about relational database connections + +:TOC: + = Database Plugins Apache Hop has optimized support for many database types. If you are using an unsupported database, you can always create a generic connection. @@ -28,6 +31,105 @@ To create a database connection go to the metadata perspective, right-click on ` The connection is saved in a central location and can then be used by all pipelines and workflows. If you have set your project to work with Hop, the database information will be in the `{openvar}PROJECT_HOME{closevar}/metadata/rdbms` folder. For each connection, a separate .json file will be generated in this folder. This json file will have the name of the connection and will contain all of the connection information. +TIP: Prefer variables for host, port, database name, username and password instead of hard-coded values. +The same connection metadata can then run unchanged in Development, Test, Acceptance and Production (DTAP) by loading different environment configuration files. +See xref:projects/index.adoc[Projects and environments] for how lifecycle environments work. + +== Generate environment variables + +Best practice is to parameterize relational database connections with environment variables. +The connection editor can generate those variables for you so you do not have to invent names or edit configuration files by hand in several places. + +=== How to use it + +. Open a relational database connection in the Metadata perspective and give it a name (for example `EDW`). +. Optionally fill in the current host, port, database, username, password and/or manual URL values. +. Click *Generate variables* in the button bar (next to Explore and Test). +. Review the proposed variables in the dialog. +You can change names, values and descriptions. +Leave values empty when you want a template without secrets or environment-specific data. ++ +image::database/generate-variables-proposals.png[Proposed environment variables for connection EDW,width="90%"] +. When asked, choose whether to create or update an environment configuration JSON file, and pick a filename (for example `{openvar}PROJECT_HOME{closevar}/EDW-config.json`). ++ +image::database/generate-variables-save-as-env-config-file-question.png[Prompt to create or update an environment configuration JSON file for DTAP,width="70%"] +. The editor replaces the connection fields with variable expressions such as `{openvar}EDW_HOSTNAME{closevar}`. + +For a connection named `EDW`, Hop typically proposes: + +[%header, width="90%", cols="2,3,3"] +|=== +|Variable|Purpose|When proposed +|`EDW_HOSTNAME`|Server host name|Always +|`EDW_PORT`|Port number|Always +|`EDW_DATABASE`|Database / catalog name|Always +|`EDW_USERNAME`|Username|When the username field is available for the connection type +|`EDW_PASSWORD`|Password|When the password field is available for the connection type +|`EDW_URL`|Manual JDBC URL|Only when a manual URL is filled in +|=== + +The connection name is turned into a variable prefix (upper case; characters other than letters, digits and underscore become `_`). +For example `My DB` becomes `MY_DB_HOSTNAME`, `MY_DB_PORT`, and so on. + +Each variable gets a short description (for example “Hostname for database connection EDW”). +In the review dialog you can encode sensitive values with the *Encode value* action if you store them encrypted in the configuration file. + +After the file is written, add it to the configuration files list of the relevant xref:projects/projects-environments.adoc[lifecycle environment] so Hop loads the variables when that environment is selected. +Paths in the file dialog may use variables such as `{openvar}PROJECT_HOME{closevar}`; Hop resolves them when reading or writing the file. + +=== DTAP and empty configuration files + +Development, Test, Acceptance and Production (DTAP) usually need the *same* variable names and the *same* connection metadata, but *different* host, credentials and sometimes database names. + +A practical pattern is: + +. *Generate once in the project* +Use *Generate variables* and create a configuration JSON that lists the variable names (and descriptions) for the connection. +Clear the values in the review dialog (or leave them empty) so the file contains structure only—no hostnames, passwords or other secrets. +Commit this empty or minimal template to version control with the project (or in a shared config location your team uses for templates). + +. *Fill values per environment outside the main project tree* +For each lifecycle stage (Development, Test, Acceptance, Production), maintain a copy of that configuration file with the real values for that stage. +Those files are typically *not* the empty template: they are deployed with the server, CI agent or runtime, or kept in a separate repository/secure store. +Point each Hop xref:projects/projects-environments.adoc[environment] at its own configuration file(s). + +. *Keep pipelines and connection metadata environment-agnostic* +Because the RDBMS connection stores only expressions like `{openvar}EDW_HOSTNAME{closevar}`, the same metadata under `{openvar}PROJECT_HOME{closevar}/metadata/rdbms` works in every stage. +You switch behavior by selecting a different environment (and therefore a different set of variable values), not by editing connections or pipelines. + +TIP: Store environment-specific configuration files *outside* the project home when they contain secrets or production endpoints. +See the tip in xref:projects/index.adoc#_environments[Environments]: keep them out of the project folder and consider a separate version control repository for environment configs. + +.Example empty / template environment config (structure only) +[source,json] +---- +{ + "variables" : [ { + "name" : "EDW_HOSTNAME", + "value" : "", + "description" : "Hostname for database connection EDW" + }, { + "name" : "EDW_PORT", + "value" : "", + "description" : "Port for database connection EDW" + }, { + "name" : "EDW_DATABASE", + "value" : "", + "description" : "Database name for database connection EDW" + }, { + "name" : "EDW_USERNAME", + "value" : "", + "description" : "Username for database connection EDW" + }, { + "name" : "EDW_PASSWORD", + "value" : "", + "description" : "Password for database connection EDW" + } ] +} +---- + +A Development environment file might set `EDW_HOSTNAME` to `localhost` and use local credentials; Production would point at the production host and credentials—same variable names, different values, same project metadata. + == Adding JDBC drivers Apache Hop ships the JDBC drivers it is allowed to redistribute - those under a license compatible with the Apache License - in the `lib/jdbc` folder of your installation. For databases whose driver has a restricted license, or that simply isn't bundled, Hop can download the driver for you on demand, or you can add it manually. diff --git a/docs/hop-user-manual/modules/ROOT/pages/hop-gui/hop-web.adoc b/docs/hop-user-manual/modules/ROOT/pages/hop-gui/hop-web.adoc index 1ceeb5d11be..2fedb3e159b 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/hop-gui/hop-web.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/hop-gui/hop-web.adoc @@ -65,6 +65,12 @@ If you do not set this variable, no project or environment will be created. |```HOP_PROJECT_FOLDER``` | Path to the home of the Hop project. +|```HOP_PARENT_PROJECT_NAME``` +| Optional name of a one-level parent project to register before the main project (required with `HOP_PARENT_PROJECT_FOLDER` when the project inherits metadata). + +|```HOP_PARENT_PROJECT_FOLDER``` +| Path to the home of the parent Hop project. + |```HOP_ENVIRONMENT_NAME``` | The name of the Hop environment to create in the container. If you do not set this variable, no environment will be created. diff --git a/docs/hop-user-manual/modules/ROOT/pages/projects/advanced.adoc b/docs/hop-user-manual/modules/ROOT/pages/projects/advanced.adoc index 1e6bad85011..963160a975a 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/projects/advanced.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/projects/advanced.adoc @@ -110,9 +110,26 @@ Let's take a look at it: You can define variables on a project level as well. This makes it handy to reference things like input and output folders which are not sensitive to being checked into version control. +Hop also sets a few built-in project variables when a project is activated: + +.Built-in project variables +[cols="30%,70%",options="header"] +|=== +|Variable|Description +|`PROJECT_HOME`|Home folder of the active project +|`HOP_PROJECT_NAME`|Name of the active project +|`PARENT_PROJECT_HOME`|Home folder of the immediate parent project (empty if none) +|`PARENT_PROJECT_NAME`|Name of the immediate parent project (empty if none) +|=== + +Use the parent variables to reference shared assets without fragile relative paths, for example: + +`{openvar}PARENT_PROJECT_HOME{closevar}/SHARED_PIPELINES/example.hpl` + ==== Parent projects As you can see from the project configuration file (`parentProjectName`), a project can have a parent from which it will inherit all the metadata objects as well as all the variables that are defined in it. +When a parent is configured, Hop also sets `{openvar}PARENT_PROJECT_HOME{closevar}` and `{openvar}PARENT_PROJECT_NAME{closevar}` so pipelines and environment config files can point into the parent project home (for example shared pipelines or secrets kept outside the child repository). === Environment configuration diff --git a/docs/hop-user-manual/modules/ROOT/pages/projects/index.adoc b/docs/hop-user-manual/modules/ROOT/pages/projects/index.adoc index fcbfc44752e..f1e5371cd7b 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/projects/index.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/projects/index.adoc @@ -72,3 +72,9 @@ Just like projects, environments are defined in one or a number of configuration TIP: store your environment configuration files outside of the project folder. You may even want to check them in to a separate version control repository. + +==== Import variables + +When you create or edit a lifecycle environment, the *Import variables* button scans the associated project for Unix-style variable expressions (`{openvar}VARIABLE_NAME{closevar}`) in pipelines, workflows and metadata objects (using the same project search infrastructure as the Hop Gui search perspective, with a regular expression for that pattern). + +Variables that are already defined in one of the environment's configuration files, as well as runtime/project managed names such as `{openvar}PROJECT_HOME{closevar}`, are omitted. The remaining names are shown in a dialog where you can set values and descriptions, then save them to an existing environment configuration file or create a new one that is added to the environment. diff --git a/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc b/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc index 5ff70d0d60e..1da0e5ea46c 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/projects/projects-environments.adoc @@ -89,6 +89,8 @@ The edit button opens the environment file and lets you add variable key-value p image::hop-gui/environment/environment-variables.png[Environment Variables, width="80%"] +When you browse for a file or directory in Hop Gui with a project active, paths under a matching path-like variable are rewritten automatically. For example, `{openvar}PROJECT_HOME{closevar}` is used for files under the project home, and an environment variable such as `SOURCE_FILES=/data/incoming` rewrites a selection under that folder to `{openvar}SOURCE_FILES{closevar}/…`. If more than one variable matches, the longest (most specific) path wins. The same applies to path variables defined on the project itself. + After creating an environment the user interface will switch to it. diff --git a/engine/src/main/java/org/apache/hop/execution/ExecutionStateBuilder.java b/engine/src/main/java/org/apache/hop/execution/ExecutionStateBuilder.java index 1f6ea6e36af..0b803d316aa 100644 --- a/engine/src/main/java/org/apache/hop/execution/ExecutionStateBuilder.java +++ b/engine/src/main/java/org/apache/hop/execution/ExecutionStateBuilder.java @@ -37,6 +37,13 @@ import org.apache.hop.workflow.engine.IWorkflowEngine; public final class ExecutionStateBuilder { + + /** Metric map key: component duration in milliseconds. */ + public static final String METRIC_HEADER_DURATION = "Duration"; + + /** Metric map key: component throughput in rows per second. */ + public static final String METRIC_HEADER_SPEED = "Speed"; + private ExecutionType executionType; private Date updateTime; private String statusDescription; @@ -122,6 +129,17 @@ public static ExecutionStateBuilder fromExecutor( addMetric(componentMetrics, engineMetrics, component, Pipeline.METRIC_DATA_VOLUME_IN); addMetric(componentMetrics, engineMetrics, component, Pipeline.METRIC_DATA_VOLUME_OUT); + // Duration (ms) and speed (rows/s) for the Metrics tab in the execution perspective. + // + long durationMs = resolveDurationMs(component); + if (durationMs > 0) { + componentMetrics.getMetrics().put(METRIC_HEADER_DURATION, durationMs); + } + Long speed = resolveSpeedRowsPerSecond(component, engineMetrics, durationMs); + if (speed != null) { + componentMetrics.getMetrics().put(METRIC_HEADER_SPEED, speed); + } + builder.addMetrics(componentMetrics); } } @@ -129,6 +147,80 @@ public static ExecutionStateBuilder fromExecutor( return builder; } + /** + * Resolves component duration in milliseconds. Prefers {@link + * IEngineComponent#getExecutionDuration()}; falls back to first/last row timestamps when duration + * is zero (same idea as the live pipeline metrics grid). + */ + static long resolveDurationMs(IEngineComponent component) { + long durationMs = component.getExecutionDuration(); + if (durationMs > 0) { + return durationMs; + } + Date first = component.getFirstRowReadDate(); + if (first == null) { + return 0; + } + Date last = component.getLastRowWrittenDate(); + if (last != null) { + return Math.max(0, last.getTime() - first.getTime()); + } + return Math.max(0, System.currentTimeMillis() - first.getTime()); + } + + /** + * Resolves throughput in rows per second. Prefers the engine speed map when a numeric value can + * be parsed; otherwise computes from row counts and duration (same formula as {@code + * TransformStatus}). + * + * @return rows/s as a long, or {@code null} when speed is not meaningful + */ + static Long resolveSpeedRowsPerSecond( + IEngineComponent component, EngineMetrics engineMetrics, long durationMs) { + if (engineMetrics != null) { + String speedText = engineMetrics.getComponentSpeedMap().get(component); + Long parsed = parseSpeedRowsPerSecond(speedText); + if (parsed != null) { + return parsed; + } + } + if (durationMs <= 0) { + return null; + } + long inProc = Math.max(component.getLinesInput(), component.getLinesRead()); + long outProc = + Math.max( + component.getLinesOutput() + component.getLinesUpdated(), + component.getLinesWritten() + component.getLinesRejected()); + double lapsedSec = durationMs / 1000.0; + if (lapsedSec <= 0) { + return null; + } + double inSpeed = inProc / lapsedSec; + double outSpeed = outProc / lapsedSec; + return Math.round(Math.max(inSpeed, outSpeed)); + } + + /** + * Parses a speed string as produced by the local pipeline engine / {@code TransformStatus} (e.g. + * {@code " 1,234"}). Returns {@code null} for empty, dash, or non-numeric values. + */ + static Long parseSpeedRowsPerSecond(String speedText) { + if (speedText == null) { + return null; + } + String cleaned = speedText.trim().replace(",", "").replace(" ", ""); + if (cleaned.isEmpty() || "-".equals(cleaned)) { + return null; + } + try { + // TransformStatus may use a decimal; store whole rows/s. + return Math.round(Double.parseDouble(cleaned)); + } catch (NumberFormatException e) { + return null; + } + } + public static ExecutionStateBuilder fromTransform( IPipelineEngine pipeline, IEngineComponent component) { return of().withExecutionType(ExecutionType.Transform) diff --git a/engine/src/test/java/org/apache/hop/execution/ExecutionStateBuilderMetricsTest.java b/engine/src/test/java/org/apache/hop/execution/ExecutionStateBuilderMetricsTest.java new file mode 100644 index 00000000000..18808fcd2a4 --- /dev/null +++ b/engine/src/test/java/org/apache/hop/execution/ExecutionStateBuilderMetricsTest.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.execution; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import org.apache.hop.pipeline.engine.EngineMetrics; +import org.apache.hop.pipeline.engine.IEngineComponent; +import org.junit.jupiter.api.Test; + +class ExecutionStateBuilderMetricsTest { + + @Test + void resolveDurationMsPrefersExecutionDuration() { + IEngineComponent component = mock(IEngineComponent.class); + when(component.getExecutionDuration()).thenReturn(2500L); + + assertEquals(2500L, ExecutionStateBuilder.resolveDurationMs(component)); + } + + @Test + void resolveDurationMsFallsBackToFirstLastRowDates() { + IEngineComponent component = mock(IEngineComponent.class); + when(component.getExecutionDuration()).thenReturn(0L); + Date first = new Date(1_000_000L); + Date last = new Date(1_005_000L); + when(component.getFirstRowReadDate()).thenReturn(first); + when(component.getLastRowWrittenDate()).thenReturn(last); + + assertEquals(5000L, ExecutionStateBuilder.resolveDurationMs(component)); + } + + @Test + void parseSpeedRowsPerSecondHandlesFormattedValues() { + assertEquals(1234L, ExecutionStateBuilder.parseSpeedRowsPerSecond(" 1,234")); + assertEquals(10L, ExecutionStateBuilder.parseSpeedRowsPerSecond("10.4")); + assertNull(ExecutionStateBuilder.parseSpeedRowsPerSecond("-")); + assertNull(ExecutionStateBuilder.parseSpeedRowsPerSecond("")); + assertNull(ExecutionStateBuilder.parseSpeedRowsPerSecond(null)); + } + + @Test + void resolveSpeedFromEngineMap() { + IEngineComponent component = mock(IEngineComponent.class); + EngineMetrics metrics = new EngineMetrics(); + Map speedMap = new HashMap<>(); + speedMap.put(component, " 500"); + metrics.setComponentSpeedMap(speedMap); + + assertEquals(500L, ExecutionStateBuilder.resolveSpeedRowsPerSecond(component, metrics, 1000L)); + } + + @Test + void resolveSpeedFromRowCountsAndDuration() { + IEngineComponent component = mock(IEngineComponent.class); + when(component.getLinesInput()).thenReturn(0L); + when(component.getLinesRead()).thenReturn(1000L); + when(component.getLinesOutput()).thenReturn(0L); + when(component.getLinesUpdated()).thenReturn(0L); + when(component.getLinesWritten()).thenReturn(1000L); + when(component.getLinesRejected()).thenReturn(0L); + + // 1000 rows over 2 seconds => 500 r/s + assertEquals( + 500L, + ExecutionStateBuilder.resolveSpeedRowsPerSecond(component, new EngineMetrics(), 2000L)); + } +} diff --git a/plugins/databases/cratedb/src/main/java/org/apache/hop/pipeline/transforms/cratedbbulkloader/CrateDBBulkLoader.java b/plugins/databases/cratedb/src/main/java/org/apache/hop/pipeline/transforms/cratedbbulkloader/CrateDBBulkLoader.java index 24880db16b1..3a5542f6596 100644 --- a/plugins/databases/cratedb/src/main/java/org/apache/hop/pipeline/transforms/cratedbbulkloader/CrateDBBulkLoader.java +++ b/plugins/databases/cratedb/src/main/java/org/apache/hop/pipeline/transforms/cratedbbulkloader/CrateDBBulkLoader.java @@ -364,7 +364,9 @@ private void defineSelectedFieldsMetadataList() throws HopTransformException { private void writeIfBatchSizeRecordsAreReached() throws HopException, CrateDBHopException, IOException { - int maxBatchSize = Integer.parseInt(meta.getBatchSize()); + String batchSize = meta.getBatchSize(); + String expandedBatchSize = Const.expandIntegerString(batchSize); + int maxBatchSize = Integer.parseInt(expandedBatchSize != null ? expandedBatchSize : batchSize); if (data.httpBulkArgs.size() >= maxBatchSize) { String[] columns = meta.getFields().stream() diff --git a/plugins/databases/cratedb/src/main/java/org/apache/hop/pipeline/transforms/cratedbbulkloader/CrateDBBulkLoaderDialog.java b/plugins/databases/cratedb/src/main/java/org/apache/hop/pipeline/transforms/cratedbbulkloader/CrateDBBulkLoaderDialog.java index bdb1a896507..c41ef878584 100644 --- a/plugins/databases/cratedb/src/main/java/org/apache/hop/pipeline/transforms/cratedbbulkloader/CrateDBBulkLoaderDialog.java +++ b/plugins/databases/cratedb/src/main/java/org/apache/hop/pipeline/transforms/cratedbbulkloader/CrateDBBulkLoaderDialog.java @@ -662,6 +662,7 @@ public void widgetSelected(SelectionEvent arg0) { wlBatchSize.setLayoutData(fdlBatchSize); wBatchSize = new TextVar(variables, wMainComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wBatchSize.enableExpandedInteger(); PropsUi.setLook(wBatchSize); wBatchSize.addModifyListener(lsMod); wBatchSize.addFocusListener(lsFocusLost); diff --git a/plugins/databases/monetdb/src/main/java/org/apache/hop/pipeline/transforms/monetdbbulkloader/MonetDbBulkLoader.java b/plugins/databases/monetdb/src/main/java/org/apache/hop/pipeline/transforms/monetdbbulkloader/MonetDbBulkLoader.java index fd54cb56cb9..cc908976599 100644 --- a/plugins/databases/monetdb/src/main/java/org/apache/hop/pipeline/transforms/monetdbbulkloader/MonetDbBulkLoader.java +++ b/plugins/databases/monetdb/src/main/java/org/apache/hop/pipeline/transforms/monetdbbulkloader/MonetDbBulkLoader.java @@ -584,7 +584,7 @@ public boolean init() { data.monetNumberMeta.setDecimalSymbol("."); data.monetNumberMeta.setStringEncoding(encoding); - data.bufferSize = Const.toInt(resolve(meta.getBufferSize()), 100000); + data.bufferSize = Const.toIntExpanded(resolve(meta.getBufferSize()), 100000); // Allocate the buffer // diff --git a/plugins/databases/monetdb/src/main/java/org/apache/hop/pipeline/transforms/monetdbbulkloader/MonetDbBulkLoaderDialog.java b/plugins/databases/monetdb/src/main/java/org/apache/hop/pipeline/transforms/monetdbbulkloader/MonetDbBulkLoaderDialog.java index c260755acc0..75feb73d7df 100644 --- a/plugins/databases/monetdb/src/main/java/org/apache/hop/pipeline/transforms/monetdbbulkloader/MonetDbBulkLoaderDialog.java +++ b/plugins/databases/monetdb/src/main/java/org/apache/hop/pipeline/transforms/monetdbbulkloader/MonetDbBulkLoaderDialog.java @@ -277,6 +277,7 @@ public void widgetSelected(SelectionEvent arg0) { PropsUi.setLook(wlBufferSize); wBufferSize = new TextVar(variables, wGeneralSettingsComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wBufferSize.enableExpandedInteger(); PropsUi.setLook(wBufferSize); wBufferSize.addModifyListener(lsMod); diff --git a/plugins/databases/mysql/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoader.java b/plugins/databases/mysql/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoader.java index 213568569d5..e378370bbba 100644 --- a/plugins/databases/mysql/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoader.java +++ b/plugins/databases/mysql/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoader.java @@ -577,7 +577,7 @@ public boolean init() { data.bulkNumberMeta.setDecimalSymbol("."); data.bulkNumberMeta.setStringEncoding(realEncoding); - data.bulkSize = Const.toLong(resolve(meta.getBulkSize()), -1L); + data.bulkSize = Const.toLongExpanded(resolve(meta.getBulkSize()), -1L); // Schema-table combination... DatabaseMeta databaseMeta = getPipelineMeta().findDatabase(meta.getConnection(), variables); diff --git a/plugins/databases/mysql/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderDialog.java b/plugins/databases/mysql/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderDialog.java index 3baf7cf6f8c..f289e639a7d 100644 --- a/plugins/databases/mysql/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderDialog.java +++ b/plugins/databases/mysql/src/main/java/org/apache/hop/pipeline/transforms/mysqlbulkloader/MySqlBulkLoaderDialog.java @@ -351,6 +351,7 @@ public void widgetSelected(SelectionEvent se) { fdlBulkSize.top = new FormAttachment(wCharSet, margin); wlBulkSize.setLayoutData(fdlBulkSize); wBulkSize = new TextVar(variables, wGeneralComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wBulkSize.enableExpandedInteger(); PropsUi.setLook(wBulkSize); wBulkSize.addModifyListener(lsMod); FormData fdBulkSize = new FormData(); diff --git a/plugins/databases/oracle/src/main/java/org/apache/hop/pipeline/transforms/orabulkloader/OraBulkLoaderDialog.java b/plugins/databases/oracle/src/main/java/org/apache/hop/pipeline/transforms/orabulkloader/OraBulkLoaderDialog.java index dd001020d54..ea59f540c10 100644 --- a/plugins/databases/oracle/src/main/java/org/apache/hop/pipeline/transforms/orabulkloader/OraBulkLoaderDialog.java +++ b/plugins/databases/oracle/src/main/java/org/apache/hop/pipeline/transforms/orabulkloader/OraBulkLoaderDialog.java @@ -360,6 +360,7 @@ public void widgetSelected(SelectionEvent e) { fdlMaxErrors.right = new FormAttachment(middle, -margin); wlMaxErrors.setLayoutData(fdlMaxErrors); wMaxErrors = new TextVar(variables, wBulkLoaderComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wMaxErrors.enableExpandedInteger(); PropsUi.setLook(wMaxErrors); wMaxErrors.addModifyListener(lsMod); FormData fdMaxErrors = new FormData(); @@ -378,6 +379,7 @@ public void widgetSelected(SelectionEvent e) { fdlCommit.right = new FormAttachment(middle, -margin); wlCommit.setLayoutData(fdlCommit); wCommit = new TextVar(variables, wBulkLoaderComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wCommit.enableExpandedInteger(); PropsUi.setLook(wCommit); wCommit.addModifyListener(lsMod); FormData fdCommit = new FormData(); @@ -396,6 +398,7 @@ public void widgetSelected(SelectionEvent e) { fdlBindSize.right = new FormAttachment(middle, -margin); wlBindSize.setLayoutData(fdlBindSize); wBindSize = new TextVar(variables, wBulkLoaderComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wBindSize.enableExpandedInteger(); PropsUi.setLook(wBindSize); wBindSize.addModifyListener(lsMod); FormData fdBindSize = new FormData(); @@ -414,6 +417,7 @@ public void widgetSelected(SelectionEvent e) { fdlReadSize.right = new FormAttachment(middle, -margin); wlReadSize.setLayoutData(fdlReadSize); wReadSize = new TextVar(variables, wBulkLoaderComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wReadSize.enableExpandedInteger(); PropsUi.setLook(wReadSize); wReadSize.addModifyListener(lsMod); FormData fdReadSize = new FormData(); diff --git a/plugins/databases/oracle/src/main/java/org/apache/hop/pipeline/transforms/orabulkloader/OraBulkLoaderMeta.java b/plugins/databases/oracle/src/main/java/org/apache/hop/pipeline/transforms/orabulkloader/OraBulkLoaderMeta.java index 31da7dd594e..98482e514ee 100644 --- a/plugins/databases/oracle/src/main/java/org/apache/hop/pipeline/transforms/orabulkloader/OraBulkLoaderMeta.java +++ b/plugins/databases/oracle/src/main/java/org/apache/hop/pipeline/transforms/orabulkloader/OraBulkLoaderMeta.java @@ -256,11 +256,7 @@ public OraBulkLoaderMeta() { } public int getCommitSizeAsInt(IVariables variables) { - try { - return Integer.parseInt(variables.resolve(getCommitSize())); - } catch (NumberFormatException ex) { - return DEFAULT_COMMIT_SIZE; - } + return Const.toIntExpanded(variables.resolve(getCommitSize()), DEFAULT_COMMIT_SIZE); } /** @@ -797,11 +793,7 @@ public void setEraseFiles(boolean eraseFiles) { } public int getBindSizeAsInt(IVariables variables) { - try { - return Integer.parseInt(variables.resolve(getBindSize())); - } catch (NumberFormatException ex) { - return DEFAULT_BIND_SIZE; - } + return Const.toIntExpanded(variables.resolve(getBindSize()), DEFAULT_BIND_SIZE); } public String getBindSize() { @@ -813,11 +805,7 @@ public void setBindSize(String bindSize) { } public int getMaxErrorsAsInt(IVariables variables) { - try { - return Integer.parseInt(variables.resolve(getMaxErrors())); - } catch (NumberFormatException ex) { - return DEFAULT_MAX_ERRORS; - } + return Const.toIntExpanded(variables.resolve(getMaxErrors()), DEFAULT_MAX_ERRORS); } public String getMaxErrors() { @@ -829,11 +817,7 @@ public void setMaxErrors(String maxErrors) { } public int getReadSizeAsInt(IVariables variables) { - try { - return Integer.parseInt(variables.resolve(getReadSize())); - } catch (NumberFormatException ex) { - return DEFAULT_READ_SIZE; - } + return Const.toIntExpanded(variables.resolve(getReadSize()), DEFAULT_READ_SIZE); } public String getReadSize() { diff --git a/plugins/databases/snowflake/src/main/java/org/apache/hop/pipeline/transforms/snowflake/bulkloader/SnowflakeBulkLoader.java b/plugins/databases/snowflake/src/main/java/org/apache/hop/pipeline/transforms/snowflake/bulkloader/SnowflakeBulkLoader.java index dd910ac7836..250676674b0 100644 --- a/plugins/databases/snowflake/src/main/java/org/apache/hop/pipeline/transforms/snowflake/bulkloader/SnowflakeBulkLoader.java +++ b/plugins/databases/snowflake/src/main/java/org/apache/hop/pipeline/transforms/snowflake/bulkloader/SnowflakeBulkLoader.java @@ -144,8 +144,8 @@ public synchronized boolean processRow() throws HopException { // Create a new split? if ((row != null && data.outputCount > 0 - && Const.toInt(resolve(meta.getSplitSize()), 0) > 0 - && (data.outputCount % Const.toInt(resolve(meta.getSplitSize()), 0)) == 0)) { + && Const.toIntExpanded(resolve(meta.getSplitSize()), 0) > 0 + && (data.outputCount % Const.toIntExpanded(resolve(meta.getSplitSize()), 0)) == 0)) { // Done with this part or with everything. closeFile(); diff --git a/plugins/databases/snowflake/src/main/java/org/apache/hop/pipeline/transforms/snowflake/bulkloader/SnowflakeBulkLoaderDialog.java b/plugins/databases/snowflake/src/main/java/org/apache/hop/pipeline/transforms/snowflake/bulkloader/SnowflakeBulkLoaderDialog.java index 97a24089250..b47e7d58b78 100644 --- a/plugins/databases/snowflake/src/main/java/org/apache/hop/pipeline/transforms/snowflake/bulkloader/SnowflakeBulkLoaderDialog.java +++ b/plugins/databases/snowflake/src/main/java/org/apache/hop/pipeline/transforms/snowflake/bulkloader/SnowflakeBulkLoaderDialog.java @@ -585,6 +585,7 @@ public void focusGained(FocusEvent focusEvent) { wlSplitSize.setLayoutData(fdlSplitSize); wSplitSize = new TextVar(variables, wLoaderComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wSplitSize.enableExpandedInteger(); PropsUi.setLook(wSplitSize); wSplitSize.addModifyListener(lsMod); FormData fdSplitSize = new FormData(); diff --git a/plugins/engines/beam/src/main/java/org/apache/hop/beam/engines/flink/BeamFlinkPipelineRunConfiguration.java b/plugins/engines/beam/src/main/java/org/apache/hop/beam/engines/flink/BeamFlinkPipelineRunConfiguration.java index 181ab191904..652cf1bc91d 100644 --- a/plugins/engines/beam/src/main/java/org/apache/hop/beam/engines/flink/BeamFlinkPipelineRunConfiguration.java +++ b/plugins/engines/beam/src/main/java/org/apache/hop/beam/engines/flink/BeamFlinkPipelineRunConfiguration.java @@ -367,7 +367,7 @@ public PipelineOptions getPipelineOptions() throws HopException { // The maximum number of elements in a bundle.") if (StringUtils.isNotEmpty(getFlinkMaxBundleSize())) { - long value = Const.toLong(resolve(getFlinkMaxBundleSize()), -1L); + long value = Const.toLongExpanded(resolve(getFlinkMaxBundleSize()), -1L); if (value > 0) { options.setMaxBundleSize(value); } diff --git a/plugins/engines/beam/src/main/java/org/apache/hop/beam/engines/spark/BeamSparkPipelineRunConfiguration.java b/plugins/engines/beam/src/main/java/org/apache/hop/beam/engines/spark/BeamSparkPipelineRunConfiguration.java index e8a2670e054..0f46671df02 100644 --- a/plugins/engines/beam/src/main/java/org/apache/hop/beam/engines/spark/BeamSparkPipelineRunConfiguration.java +++ b/plugins/engines/beam/src/main/java/org/apache/hop/beam/engines/spark/BeamSparkPipelineRunConfiguration.java @@ -231,7 +231,7 @@ public PipelineOptions getPipelineOptions() throws HopException { } } if (StringUtils.isNotEmpty(getSparkMaxRecordsPerBatch())) { - long records = Const.toLong(resolve(getSparkMaxRecordsPerBatch()), -1L); + long records = Const.toLongExpanded(resolve(getSparkMaxRecordsPerBatch()), -1L); if (records >= 0) { options.setMaxRecordsPerBatch(records); } @@ -249,7 +249,7 @@ public PipelineOptions getPipelineOptions() throws HopException { } } if (StringUtils.isNotEmpty(getSparkBundleSize())) { - long bundleSize = Const.toLong(resolve(getSparkBundleSize()), -1L); + long bundleSize = Const.toLongExpanded(resolve(getSparkBundleSize()), -1L); if (bundleSize >= 0) { options.setBundleSize(bundleSize); } diff --git a/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/handler/BeamRowGeneratorTransformHandler.java b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/handler/BeamRowGeneratorTransformHandler.java index 991d3bd39b4..1ad98fb7e5e 100644 --- a/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/handler/BeamRowGeneratorTransformHandler.java +++ b/plugins/engines/beam/src/main/java/org/apache/hop/beam/pipeline/handler/BeamRowGeneratorTransformHandler.java @@ -103,7 +103,7 @@ public void handleTransform( throw new HopException("Error encoding row as XML", e); } - long intervalMs = Const.toLong(variables.resolve(meta.getIntervalInMs()), -1L); + long intervalMs = Const.toLongExpanded(variables.resolve(meta.getIntervalInMs()), -1L); if (intervalMs < 0) { throw new HopException( "The interval in milliseconds is expected to be >= 0, not '" @@ -157,7 +157,7 @@ public void handleTransform( // A fixed number of records // - long numRecords = Const.toLong(variables.resolve(meta.getRowLimit()), -1L); + long numRecords = Const.toLongExpanded(variables.resolve(meta.getRowLimit()), -1L); if (numRecords < 0) { throw new HopException( "Please specify a valid number of records to generate, not '" diff --git a/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ActionDebugGuiPlugin.java b/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ActionDebugGuiPlugin.java index 97259f046ea..a90bc7fe80a 100644 --- a/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ActionDebugGuiPlugin.java +++ b/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ActionDebugGuiPlugin.java @@ -22,6 +22,7 @@ import org.apache.hop.core.action.GuiContextAction; import org.apache.hop.core.gui.plugin.GuiPlugin; import org.apache.hop.core.gui.plugin.action.GuiActionType; +import org.apache.hop.core.variables.IVariables; import org.apache.hop.debug.util.DebugLevelUtil; import org.apache.hop.debug.util.Defaults; import org.apache.hop.ui.core.dialog.ErrorDialog; @@ -67,6 +68,7 @@ public void applyCustomActionLogging(HopGuiWorkflowActionContext context) { try { WorkflowMeta workflowMeta = context.getWorkflowMeta(); ActionMeta action = context.getActionMeta(); + IVariables variables = context.getWorkflowGraph().getVariables(); Map> attributesMap = workflowMeta.getAttributesMap(); Map debugGroupAttributesMap = attributesMap.get(Defaults.DEBUG_GROUP); @@ -83,7 +85,7 @@ public void applyCustomActionLogging(HopGuiWorkflowActionContext context) { } ActionDebugLevelDialog dialog = - new ActionDebugLevelDialog(hopGui.getActiveShell(), debugLevel); + new ActionDebugLevelDialog(hopGui.getActiveShell(), debugLevel, variables); if (dialog.open()) { DebugLevelUtil.storeActionDebugLevel( debugGroupAttributesMap, action.toString(), debugLevel); diff --git a/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ActionDebugLevel.java b/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ActionDebugLevel.java index 32c0934e066..ee39bbee823 100644 --- a/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ActionDebugLevel.java +++ b/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ActionDebugLevel.java @@ -20,7 +20,7 @@ import org.apache.hop.core.logging.LogLevel; public class ActionDebugLevel implements Cloneable { - private LogLevel logLevel; + private String logLevel; private boolean loggingResult; private boolean loggingVariables; @@ -28,7 +28,7 @@ public class ActionDebugLevel implements Cloneable { private boolean loggingResultFiles; public ActionDebugLevel() { - logLevel = LogLevel.DEBUG; + logLevel = LogLevel.DEBUG.getCode(); loggingResult = false; loggingVariables = false; loggingResultRows = false; @@ -36,12 +36,17 @@ public ActionDebugLevel() { } public ActionDebugLevel(LogLevel logLevel) { + this(); + this.logLevel = logLevel != null ? logLevel.getCode() : LogLevel.DEBUG.getCode(); + } + + public ActionDebugLevel(String logLevel) { this(); this.logLevel = logLevel; } public ActionDebugLevel( - LogLevel logLevel, + String logLevel, boolean loggingResult, boolean loggingVariables, boolean loggingResultRows, @@ -53,6 +58,20 @@ public ActionDebugLevel( this.loggingResultFiles = loggingResultFiles; } + public ActionDebugLevel( + LogLevel logLevel, + boolean loggingResult, + boolean loggingVariables, + boolean loggingResultRows, + boolean loggingResultFiles) { + this( + logLevel != null ? logLevel.getCode() : LogLevel.DEBUG.getCode(), + loggingResult, + loggingVariables, + loggingResultRows, + loggingResultFiles); + } + @Override public ActionDebugLevel clone() { return new ActionDebugLevel( @@ -61,7 +80,7 @@ public ActionDebugLevel clone() { @Override public String toString() { - String s = logLevel.toString(); + String s = logLevel; if (loggingResult) { s += ", logging result"; } @@ -78,21 +97,28 @@ public String toString() { } /** - * Gets logLevel + * Gets logLevel (code or variable expression) * * @return value of logLevel */ - public LogLevel getLogLevel() { + public String getLogLevel() { return logLevel; } /** - * @param logLevel The logLevel to set + * @param logLevel The logLevel to set (code or variable expression) */ - public void setLogLevel(LogLevel logLevel) { + public void setLogLevel(String logLevel) { this.logLevel = logLevel; } + /** + * @param logLevel The logLevel enum to set (stores its code) + */ + public void setLogLevel(LogLevel logLevel) { + this.logLevel = logLevel != null ? logLevel.getCode() : null; + } + /** * Gets loggingResult * diff --git a/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ActionDebugLevelDialog.java b/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ActionDebugLevelDialog.java index 6db37f442e0..53ddb8ba6e4 100644 --- a/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ActionDebugLevelDialog.java +++ b/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ActionDebugLevelDialog.java @@ -17,20 +17,21 @@ package org.apache.hop.debug.action; -import org.apache.hop.core.Const; import org.apache.hop.core.logging.LogLevel; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.debug.util.DebugLevelUtil; import org.apache.hop.i18n.BaseMessages; import org.apache.hop.ui.core.PropsUi; import org.apache.hop.ui.core.dialog.BaseDialog; import org.apache.hop.ui.core.gui.GuiResource; import org.apache.hop.ui.core.gui.WindowProperty; +import org.apache.hop.ui.core.widget.ComboVar; import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; import org.eclipse.swt.SWT; import org.eclipse.swt.layout.FormAttachment; import org.eclipse.swt.layout.FormData; import org.eclipse.swt.layout.FormLayout; import org.eclipse.swt.widgets.Button; -import org.eclipse.swt.widgets.Combo; import org.eclipse.swt.widgets.Control; import org.eclipse.swt.widgets.Dialog; import org.eclipse.swt.widgets.Label; @@ -41,12 +42,13 @@ public class ActionDebugLevelDialog extends Dialog { private ActionDebugLevel input; private ActionDebugLevel debugLevel; + private IVariables variables; private Shell shell; // Connection properties // - private Combo wLogLevel; + private ComboVar wLogLevel; private Button wLoggingResult; private Button wLoggingVariables; private Button wLoggingRows; @@ -56,9 +58,10 @@ public class ActionDebugLevelDialog extends Dialog { private boolean ok; - public ActionDebugLevelDialog(Shell par, ActionDebugLevel debugLevel) { + public ActionDebugLevelDialog(Shell par, ActionDebugLevel debugLevel, IVariables variables) { super(par, SWT.NONE); this.input = debugLevel; + this.variables = variables; props = PropsUi.getInstance(); ok = false; @@ -90,7 +93,7 @@ public boolean open() { fdlName.left = new FormAttachment(0, 0); // First one in the left top corner fdlName.right = new FormAttachment(middle, -margin); wlName.setLayoutData(fdlName); - wLogLevel = new Combo(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wLogLevel = new ComboVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); wLogLevel.setItems(LogLevel.getLogLevelDescriptions()); PropsUi.setLook(wLogLevel); FormData fdName = new FormData(); @@ -196,7 +199,7 @@ public void dispose() { } public void getData() { - wLogLevel.setText(debugLevel.getLogLevel().getDescription()); + wLogLevel.setText(DebugLevelUtil.logLevelCodeToDisplay(debugLevel.getLogLevel())); wLoggingResult.setSelection(debugLevel.isLoggingResult()); wLoggingVariables.setSelection(debugLevel.isLoggingVariables()); wLoggingRows.setSelection(debugLevel.isLoggingResultRows()); @@ -218,8 +221,7 @@ public void ok() { // Get dialog info in securityService private void getInfo(ActionDebugLevel level) { - int index = Const.indexOfString(wLogLevel.getText(), LogLevel.getLogLevelDescriptions()); - level.setLogLevel(LogLevel.values()[index]); + level.setLogLevel(DebugLevelUtil.logLevelDisplayToCode(wLogLevel.getText())); level.setLoggingResult(wLoggingResult.getSelection()); level.setLoggingVariables(wLoggingVariables.getSelection()); level.setLoggingResultRows(wLoggingRows.getSelection()); diff --git a/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/EditActionDebugLevelExtensionPoint.java b/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/EditActionDebugLevelExtensionPoint.java index fe1a526bfff..ccf644d4513 100644 --- a/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/EditActionDebugLevelExtensionPoint.java +++ b/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/EditActionDebugLevelExtensionPoint.java @@ -48,7 +48,7 @@ public void callExtensionPoint( } ActionDebugLevel debugLevel = (ActionDebugLevel) ext.getAreaOwner().getOwner(); ActionDebugLevelDialog dialog = - new ActionDebugLevelDialog(HopGui.getInstance().getShell(), debugLevel); + new ActionDebugLevelDialog(HopGui.getInstance().getShell(), debugLevel, variables); if (dialog.open()) { WorkflowMeta workflowMeta = ext.getWorkflowGraph().getWorkflowMeta(); ActionMeta actionCopy = (ActionMeta) ext.getAreaOwner().getParent(); diff --git a/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ModifyActionLogLevelExtensionPoint.java b/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ModifyActionLogLevelExtensionPoint.java index c39a15659b5..483b57d4e64 100644 --- a/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ModifyActionLogLevelExtensionPoint.java +++ b/plugins/misc/debug/src/main/java/org/apache/hop/debug/action/ModifyActionLogLevelExtensionPoint.java @@ -144,10 +144,12 @@ public void beforeExecution( final ActionDebugLevel debugLevel = DebugLevelUtil.getActionDebugLevel(entryLevelMap, actionCopy.toString()); if (debugLevel != null) { - // Set the debug level for this one... + // Set the debug level for this one (resolve variables at runtime)... // - log.setLogLevel(debugLevel.getLogLevel()); - workflow.setLogLevel(debugLevel.getLogLevel()); + LogLevel resolvedLogLevel = + DebugLevelUtil.resolveLogLevel(workflow, debugLevel.getLogLevel()); + log.setLogLevel(resolvedLogLevel); + workflow.setLogLevel(resolvedLogLevel); } } } catch (Exception e) { diff --git a/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/EditTransformDebugLevelExtensionPoint.java b/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/EditTransformDebugLevelExtensionPoint.java index a82f80887ed..e45bab753af 100644 --- a/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/EditTransformDebugLevelExtensionPoint.java +++ b/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/EditTransformDebugLevelExtensionPoint.java @@ -53,7 +53,8 @@ public void callExtensionPoint( IRowMeta inputRowMeta = pipelineMeta.getPrevTransformFields(variables, transformMeta); TransformDebugLevelDialog dialog = - new TransformDebugLevelDialog(HopGui.getInstance().getShell(), debugLevel, inputRowMeta); + new TransformDebugLevelDialog( + HopGui.getInstance().getShell(), debugLevel, inputRowMeta, variables); if (dialog.open()) { DebugLevelUtil.storeTransformDebugLevel( pipelineMeta.getAttributesMap().get(Defaults.DEBUG_GROUP), diff --git a/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/SetTransformDebugLevelExtensionPoint.java b/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/SetTransformDebugLevelExtensionPoint.java index 58265e37ac9..50525e70051 100644 --- a/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/SetTransformDebugLevelExtensionPoint.java +++ b/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/SetTransformDebugLevelExtensionPoint.java @@ -81,6 +81,8 @@ public void callExtensionPoint( log.logDetailed("Found debug level info for transform " + transformName); List transformCopies = pipeline.getComponentCopies(transformName); + final LogLevel resolvedLogLevel = + DebugLevelUtil.resolveLogLevel(variables, debugLevel.getLogLevel()); if (debugLevel.getStartRow() < 0 && debugLevel.getEndRow() < 0 @@ -89,16 +91,15 @@ public void callExtensionPoint( "Set logging level for transform " + transformName + " to " - + debugLevel.getLogLevel().getDescription()); + + resolvedLogLevel.getDescription()); // Just a general log level on the transform // for (IEngineComponent transformCopy : transformCopies) { - LogLevel logLevel = debugLevel.getLogLevel(); - transformCopy.getLogChannel().setLogLevel(logLevel); + transformCopy.getLogChannel().setLogLevel(resolvedLogLevel); log.logDetailed( "Applied logging level " - + logLevel.getDescription() + + resolvedLogLevel.getDescription() + " on transform copy " + transformCopy.getName() + "." @@ -145,7 +146,7 @@ public void rowReadEvent(IRowMeta rowMeta, Object[] row) { } if (enabled) { - transformCopy.setLogLevel(debugLevel.getLogLevel()); + transformCopy.setLogLevel(resolvedLogLevel); } } diff --git a/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/TransformDebugGuiPlugin.java b/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/TransformDebugGuiPlugin.java index a86e3e1eaff..b2c8507b662 100644 --- a/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/TransformDebugGuiPlugin.java +++ b/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/TransformDebugGuiPlugin.java @@ -87,7 +87,8 @@ public void applyCustomTransformLogging(HopGuiPipelineTransformContext context) IRowMeta inputRowMeta = pipelineMeta.getPrevTransformFields(variables, transformMeta); TransformDebugLevelDialog dialog = - new TransformDebugLevelDialog(hopGui.getActiveShell(), debugLevel, inputRowMeta); + new TransformDebugLevelDialog( + hopGui.getActiveShell(), debugLevel, inputRowMeta, variables); if (dialog.open()) { DebugLevelUtil.storeTransformDebugLevel( debugGroupAttributesMap, transformMeta.getName(), debugLevel); diff --git a/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/TransformDebugLevel.java b/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/TransformDebugLevel.java index 4581bead224..e133fdd2bfc 100644 --- a/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/TransformDebugLevel.java +++ b/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/TransformDebugLevel.java @@ -21,33 +21,46 @@ import org.apache.hop.core.logging.LogLevel; public class TransformDebugLevel implements Cloneable { - private LogLevel logLevel; + private String logLevel; private int startRow; private int endRow; private Condition condition; public TransformDebugLevel() { condition = new Condition(); - logLevel = LogLevel.DEBUG; + logLevel = LogLevel.DEBUG.getCode(); startRow = -1; endRow = -1; } public TransformDebugLevel(LogLevel logLevel) { + this(); + this.logLevel = logLevel != null ? logLevel.getCode() : LogLevel.DEBUG.getCode(); + } + + public TransformDebugLevel(String logLevel) { this(); this.logLevel = logLevel; } - public TransformDebugLevel(LogLevel logLevel, int startRow, int endRow, Condition condition) { + public TransformDebugLevel(String logLevel, int startRow, int endRow, Condition condition) { this(logLevel); this.startRow = startRow; this.endRow = endRow; this.condition = condition; } + public TransformDebugLevel(LogLevel logLevel, int startRow, int endRow, Condition condition) { + this( + logLevel != null ? logLevel.getCode() : LogLevel.DEBUG.getCode(), + startRow, + endRow, + condition); + } + @Override public String toString() { - String s = logLevel.toString(); + String s = logLevel; if (startRow >= 0) { s += ", start row=" + startRow; } @@ -66,21 +79,28 @@ public TransformDebugLevel clone() { } /** - * Gets logLevel + * Gets logLevel (code or variable expression) * * @return value of logLevel */ - public LogLevel getLogLevel() { + public String getLogLevel() { return logLevel; } /** - * @param logLevel The logLevel to set + * @param logLevel The logLevel to set (code or variable expression) */ - public void setLogLevel(LogLevel logLevel) { + public void setLogLevel(String logLevel) { this.logLevel = logLevel; } + /** + * @param logLevel The logLevel enum to set (stores its code) + */ + public void setLogLevel(LogLevel logLevel) { + this.logLevel = logLevel != null ? logLevel.getCode() : null; + } + /** * Gets startRow * diff --git a/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/TransformDebugLevelDialog.java b/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/TransformDebugLevelDialog.java index 2e5b003ff40..ad1b397a6ab 100644 --- a/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/TransformDebugLevelDialog.java +++ b/plugins/misc/debug/src/main/java/org/apache/hop/debug/transform/TransformDebugLevelDialog.java @@ -20,11 +20,14 @@ import org.apache.hop.core.Const; import org.apache.hop.core.logging.LogLevel; import org.apache.hop.core.row.IRowMeta; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.debug.util.DebugLevelUtil; import org.apache.hop.i18n.BaseMessages; import org.apache.hop.ui.core.PropsUi; import org.apache.hop.ui.core.dialog.BaseDialog; import org.apache.hop.ui.core.gui.GuiResource; import org.apache.hop.ui.core.gui.WindowProperty; +import org.apache.hop.ui.core.widget.ComboVar; import org.apache.hop.ui.core.widget.ConditionEditor; import org.apache.hop.ui.pipeline.transform.BaseTransformDialog; import org.eclipse.swt.SWT; @@ -32,7 +35,6 @@ import org.eclipse.swt.layout.FormData; import org.eclipse.swt.layout.FormLayout; import org.eclipse.swt.widgets.Button; -import org.eclipse.swt.widgets.Combo; import org.eclipse.swt.widgets.Control; import org.eclipse.swt.widgets.Dialog; import org.eclipse.swt.widgets.Label; @@ -45,12 +47,13 @@ public class TransformDebugLevelDialog extends Dialog { private TransformDebugLevel input; private TransformDebugLevel debugLevel; private IRowMeta inputRowMeta; + private IVariables variables; private Shell shell; // Connection properties // - private Combo wLogLevel; + private ComboVar wLogLevel; private Text wStartRow; private Text wEndRow; @@ -59,10 +62,11 @@ public class TransformDebugLevelDialog extends Dialog { private boolean ok; public TransformDebugLevelDialog( - Shell par, TransformDebugLevel debugLevel, IRowMeta inputRowMeta) { + Shell par, TransformDebugLevel debugLevel, IRowMeta inputRowMeta, IVariables variables) { super(par, SWT.NONE); this.input = debugLevel; this.inputRowMeta = inputRowMeta; + this.variables = variables; props = PropsUi.getInstance(); ok = false; @@ -94,7 +98,7 @@ public boolean open() { fdlName.left = new FormAttachment(0, 0); // First one in the left top corner fdlName.right = new FormAttachment(middle, -margin); wlName.setLayoutData(fdlName); - wLogLevel = new Combo(shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wLogLevel = new ComboVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); wLogLevel.setItems(LogLevel.getLogLevelDescriptions()); PropsUi.setLook(wLogLevel); FormData fdName = new FormData(); @@ -184,7 +188,7 @@ public void dispose() { } public void getData() { - wLogLevel.setText(debugLevel.getLogLevel().getDescription()); + wLogLevel.setText(DebugLevelUtil.logLevelCodeToDisplay(debugLevel.getLogLevel())); wStartRow.setText( debugLevel.getStartRow() < 0 ? "" : Integer.toString(debugLevel.getStartRow())); wEndRow.setText(debugLevel.getEndRow() < 0 ? "" : Integer.toString(debugLevel.getEndRow())); @@ -205,8 +209,7 @@ public void ok() { // Get dialog info in securityService private void getInfo(TransformDebugLevel level) { - int index = Const.indexOfString(wLogLevel.getText(), LogLevel.getLogLevelDescriptions()); - level.setLogLevel(LogLevel.values()[index]); + level.setLogLevel(DebugLevelUtil.logLevelDisplayToCode(wLogLevel.getText())); level.setStartRow(Const.toInt(wStartRow.getText(), -1)); level.setEndRow(Const.toInt(wEndRow.getText(), -1)); level.setCondition(debugLevel.getCondition()); diff --git a/plugins/misc/debug/src/main/java/org/apache/hop/debug/util/DebugLevelUtil.java b/plugins/misc/debug/src/main/java/org/apache/hop/debug/util/DebugLevelUtil.java index e3b2d80ad30..b8ef3c53329 100644 --- a/plugins/misc/debug/src/main/java/org/apache/hop/debug/util/DebugLevelUtil.java +++ b/plugins/misc/debug/src/main/java/org/apache/hop/debug/util/DebugLevelUtil.java @@ -28,6 +28,7 @@ import org.apache.hop.core.exception.HopValueException; import org.apache.hop.core.exception.HopXmlException; import org.apache.hop.core.logging.LogLevel; +import org.apache.hop.core.variables.IVariables; import org.apache.hop.debug.action.ActionDebugLevel; import org.apache.hop.debug.transform.TransformDebugLevel; @@ -39,8 +40,7 @@ public static void storeTransformDebugLevel( TransformDebugLevel debugLevel) throws HopValueException, UnsupportedEncodingException { debugGroupAttributesMap.put( - transformName + " : " + Defaults.TRANSFORM_ATTR_LOGLEVEL, - debugLevel.getLogLevel().getCode()); + transformName + " : " + Defaults.TRANSFORM_ATTR_LOGLEVEL, debugLevel.getLogLevel()); debugGroupAttributesMap.put( transformName + " : " + Defaults.TRANSFORM_ATTR_START_ROW, Integer.toString(debugLevel.getStartRow())); @@ -75,7 +75,7 @@ public static TransformDebugLevel getTransformDebugLevel( } TransformDebugLevel debugLevel = new TransformDebugLevel(); - debugLevel.setLogLevel(LogLevel.lookupCode(logLevelCode)); + debugLevel.setLogLevel(logLevelCode); debugLevel.setStartRow(Const.toInt(startRowString, -1)); debugLevel.setEndRow(Const.toInt(endRowString, -1)); @@ -104,7 +104,7 @@ public static void clearDebugLevel( public static void storeActionDebugLevel( Map debugGroupAttributesMap, String entryName, ActionDebugLevel debugLevel) { debugGroupAttributesMap.put( - entryName + " : " + Defaults.ACTION_ATTR_LOGLEVEL, debugLevel.getLogLevel().getCode()); + entryName + " : " + Defaults.ACTION_ATTR_LOGLEVEL, debugLevel.getLogLevel()); debugGroupAttributesMap.put( entryName + " : " + Defaults.ACTION_ATTR_LOG_RESULT, debugLevel.isLoggingResult() ? "Y" : "N"); @@ -151,7 +151,7 @@ public static ActionDebugLevel getActionDebugLevel( } ActionDebugLevel debugLevel = new ActionDebugLevel(); - debugLevel.setLogLevel(LogLevel.lookupCode(logLevelCode)); + debugLevel.setLogLevel(logLevelCode); debugLevel.setLoggingResult(loggingResult); debugLevel.setLoggingVariables(loggingVariables); debugLevel.setLoggingResultRows(loggingResultRows); @@ -160,6 +160,82 @@ public static ActionDebugLevel getActionDebugLevel( return debugLevel; } + /** + * Resolve a log level specification (code, description, or variable expression) to a {@link + * LogLevel}. Variables are expanded first, then matched against codes (preferred) and + * descriptions. Falls back to {@link LogLevel#BASIC} when unresolved or unrecognized (same + * default as {@link LogLevel#lookupCode(String)}). + * + * @param variables variable space used to resolve expressions (may be null) + * @param logLevelSpec stored code, description, or variable expression + * @return resolved log level + */ + public static LogLevel resolveLogLevel(IVariables variables, String logLevelSpec) { + String resolved = + variables != null + ? variables.resolve(Const.NVL(logLevelSpec, "")) + : Const.NVL(logLevelSpec, ""); + if (StringUtils.isEmpty(resolved)) { + return LogLevel.BASIC; + } + // Prefer exact code match so unknown values do not silently become BASIC via lookupCode + for (LogLevel level : LogLevel.values()) { + if (level.getCode().equalsIgnoreCase(resolved)) { + return level; + } + } + for (LogLevel level : LogLevel.values()) { + if (level.getDescription().equalsIgnoreCase(resolved)) { + return level; + } + } + return LogLevel.BASIC; + } + + /** + * Map a stored log level code to its description for UI display. If the value is not a known code + * (e.g. a variable expression), return it unchanged. + * + * @param logLevelSpec stored code or variable expression + * @return description for a known code, otherwise the original string + */ + public static String logLevelCodeToDisplay(String logLevelSpec) { + if (StringUtils.isEmpty(logLevelSpec)) { + return Const.NVL(logLevelSpec, ""); + } + for (LogLevel level : LogLevel.values()) { + if (level.getCode().equalsIgnoreCase(logLevelSpec)) { + return level.getDescription(); + } + } + return logLevelSpec; + } + + /** + * Map dialog text to a value suitable for storage. If the text matches a log level description, + * store the stable code; otherwise store the text as-is (variable or free-form). + * + * @param displayText combo text from the dialog + * @return code for a known description, otherwise the original text + */ + public static String logLevelDisplayToCode(String displayText) { + if (StringUtils.isEmpty(displayText)) { + return displayText; + } + for (LogLevel level : LogLevel.values()) { + if (level.getDescription().equalsIgnoreCase(displayText)) { + return level.getCode(); + } + } + // Already a code? + for (LogLevel level : LogLevel.values()) { + if (level.getCode().equalsIgnoreCase(displayText)) { + return level.getCode(); + } + } + return displayText; + } + public static String getDurationHMS(double seconds) { int day = (int) TimeUnit.SECONDS.toDays((long) seconds); long hours = TimeUnit.SECONDS.toHours((long) seconds) - (day * 24); diff --git a/plugins/misc/debug/src/test/java/org/apache/hop/debug/util/DebugLevelUtilTest.java b/plugins/misc/debug/src/test/java/org/apache/hop/debug/util/DebugLevelUtilTest.java new file mode 100644 index 00000000000..733a15968b2 --- /dev/null +++ b/plugins/misc/debug/src/test/java/org/apache/hop/debug/util/DebugLevelUtilTest.java @@ -0,0 +1,126 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.debug.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.util.HashMap; +import java.util.Map; +import org.apache.hop.core.logging.LogLevel; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.debug.action.ActionDebugLevel; +import org.apache.hop.debug.transform.TransformDebugLevel; +import org.junit.jupiter.api.Test; + +class DebugLevelUtilTest { + + @Test + void storeAndLoadTransformDebugLevelWithCode() throws Exception { + Map map = new HashMap<>(); + TransformDebugLevel level = new TransformDebugLevel(LogLevel.DEBUG); + level.setStartRow(1); + level.setEndRow(10); + + DebugLevelUtil.storeTransformDebugLevel(map, "MyTransform", level); + TransformDebugLevel loaded = DebugLevelUtil.getTransformDebugLevel(map, "MyTransform"); + + assertNotNull(loaded); + assertEquals(LogLevel.DEBUG.getCode(), loaded.getLogLevel()); + assertEquals(1, loaded.getStartRow()); + assertEquals(10, loaded.getEndRow()); + } + + @Test + void storeAndLoadTransformDebugLevelWithVariable() throws Exception { + Map map = new HashMap<>(); + TransformDebugLevel level = new TransformDebugLevel("${MY_LOG_LEVEL}"); + + DebugLevelUtil.storeTransformDebugLevel(map, "MyTransform", level); + TransformDebugLevel loaded = DebugLevelUtil.getTransformDebugLevel(map, "MyTransform"); + + assertNotNull(loaded); + assertEquals("${MY_LOG_LEVEL}", loaded.getLogLevel()); + } + + @Test + void storeAndLoadActionDebugLevelWithVariable() { + Map map = new HashMap<>(); + ActionDebugLevel level = new ActionDebugLevel("${ACTION_LEVEL}"); + level.setLoggingResult(true); + + DebugLevelUtil.storeActionDebugLevel(map, "MyAction", level); + ActionDebugLevel loaded = DebugLevelUtil.getActionDebugLevel(map, "MyAction"); + + assertNotNull(loaded); + assertEquals("${ACTION_LEVEL}", loaded.getLogLevel()); + assertEquals(true, loaded.isLoggingResult()); + } + + @Test + void getDebugLevelReturnsNullWhenMissing() throws Exception { + Map map = new HashMap<>(); + assertNull(DebugLevelUtil.getTransformDebugLevel(map, "Missing")); + assertNull(DebugLevelUtil.getActionDebugLevel(map, "Missing")); + } + + @Test + void resolveLogLevelFromCode() { + Variables variables = new Variables(); + assertEquals(LogLevel.ROWLEVEL, DebugLevelUtil.resolveLogLevel(variables, "Rowlevel")); + assertEquals(LogLevel.DEBUG, DebugLevelUtil.resolveLogLevel(variables, "Debug")); + } + + @Test + void resolveLogLevelFromVariable() { + Variables variables = new Variables(); + variables.setVariable("MY_LEVEL", "Detailed"); + assertEquals(LogLevel.DETAILED, DebugLevelUtil.resolveLogLevel(variables, "${MY_LEVEL}")); + } + + @Test + void resolveLogLevelFromDescription() { + Variables variables = new Variables(); + String description = LogLevel.MINIMAL.getDescription(); + assertEquals(LogLevel.MINIMAL, DebugLevelUtil.resolveLogLevel(variables, description)); + } + + @Test + void resolveLogLevelUnknownFallsBackToBasic() { + Variables variables = new Variables(); + assertEquals(LogLevel.BASIC, DebugLevelUtil.resolveLogLevel(variables, "not-a-level")); + assertEquals(LogLevel.BASIC, DebugLevelUtil.resolveLogLevel(variables, "")); + assertEquals(LogLevel.BASIC, DebugLevelUtil.resolveLogLevel(null, null)); + } + + @Test + void logLevelCodeToDisplayMapsCodeToDescription() { + assertEquals(LogLevel.DEBUG.getDescription(), DebugLevelUtil.logLevelCodeToDisplay("Debug")); + assertEquals("${MY_LEVEL}", DebugLevelUtil.logLevelCodeToDisplay("${MY_LEVEL}")); + } + + @Test + void logLevelDisplayToCodeMapsDescriptionToCode() { + assertEquals( + LogLevel.DEBUG.getCode(), + DebugLevelUtil.logLevelDisplayToCode(LogLevel.DEBUG.getDescription())); + assertEquals("${MY_LEVEL}", DebugLevelUtil.logLevelDisplayToCode("${MY_LEVEL}")); + assertEquals("Debug", DebugLevelUtil.logLevelDisplayToCode("Debug")); + } +} diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EnvironmentVariablesImportHelper.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EnvironmentVariablesImportHelper.java new file mode 100644 index 00000000000..5254ae366bf --- /dev/null +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/EnvironmentVariablesImportHelper.java @@ -0,0 +1,398 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.environment; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.Const; +import org.apache.hop.core.config.DescribedVariablesConfigFile; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.search.ISearchQuery; +import org.apache.hop.core.search.ISearchResult; +import org.apache.hop.core.search.ISearchable; +import org.apache.hop.core.search.ISearchableAnalyser; +import org.apache.hop.core.search.SearchQuery; +import org.apache.hop.core.variables.DescribedVariable; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.metadata.util.HopMetadataUtil; +import org.apache.hop.projects.project.Project; +import org.apache.hop.projects.project.ProjectConfig; +import org.apache.hop.projects.search.ProjectsSearchablesLocation; +import org.apache.hop.projects.util.Defaults; +import org.apache.hop.projects.util.ProjectsUtil; +import org.apache.hop.ui.hopgui.search.HopGuiSearchHelper; + +/** + * Finds {@code ${VARIABLE}} expressions used in a project's pipelines, workflows and metadata via + * the project search infrastructure, and prepares described variables that are not yet defined in + * environment configuration files. + */ +public final class EnvironmentVariablesImportHelper { + + /** + * Regex used with {@link SearchQuery} (regex mode) to find property values that contain a Unix + * variable expression. Partial match via {@link java.util.regex.Matcher#find()}. + */ + public static final String VARIABLE_EXPRESSION_REGEX = "\\$\\{[^}]+\\}"; + + /** Extracts the variable name from each {@code ${…}} occurrence in a string. */ + public static final Pattern VARIABLE_NAME_PATTERN = Pattern.compile("\\$\\{([^}]+)\\}"); + + /** JVM temporary directory system property often referenced as {@code ${java.io.tmpdir}}. */ + public static final String JAVA_IO_TMPDIR = "java.io.tmpdir"; + + /** Max distinct search locations kept in a variable description. */ + private static final int MAX_DESCRIPTION_LOCATIONS = 10; + + private static final Set MANAGED_VARIABLE_NAMES; + + static { + Set managed = new HashSet<>(); + managed.add(ProjectsUtil.VARIABLE_PROJECT_HOME); + managed.add(ProjectsUtil.VARIABLE_PARENT_PROJECT_HOME); + managed.add(ProjectsUtil.VARIABLE_PARENT_PROJECT_NAME); + managed.add(ProjectsUtil.VARIABLE_HOP_DATASETS_FOLDER); + managed.add(ProjectsUtil.VARIABLE_HOP_UNIT_TESTS_FOLDER); + managed.add(Defaults.VARIABLE_HOP_PROJECT_NAME); + managed.add(Defaults.VARIABLE_HOP_ENVIRONMENT_NAME); + managed.add(Const.HOP_METADATA_FOLDER); + managed.add(JAVA_IO_TMPDIR); + MANAGED_VARIABLE_NAMES = Collections.unmodifiableSet(managed); + } + + private EnvironmentVariablesImportHelper() { + // Utility class + } + + /** + * Extract distinct variable names from {@code ${NAME}} expressions in the given text. Only Unix + * style is considered; {@code %%NAME%%} and {@code #{…}} are ignored. + * + * @param text text that may contain variable expressions (may be null) + * @return ordered set of variable names (insertion order of first occurrence) + */ + public static Set extractVariableNames(String text) { + Set names = new LinkedHashSet<>(); + if (StringUtils.isEmpty(text)) { + return names; + } + Matcher matcher = VARIABLE_NAME_PATTERN.matcher(text); + while (matcher.find()) { + String name = matcher.group(1); + if (StringUtils.isNotBlank(name)) { + names.add(name.trim()); + } + } + return names; + } + + /** + * Whether the variable is managed by Hop / the project runtime and should not be proposed for an + * environment configuration file. + * + * @param name variable name + * @return true if the name should be skipped + */ + public static boolean isManagedVariable(String name) { + if (StringUtils.isBlank(name)) { + return true; + } + if (name.startsWith(Const.INTERNAL_VARIABLE_PREFIX)) { + return true; + } + return MANAGED_VARIABLE_NAMES.contains(name); + } + + /** + * Load variable names already defined in the given environment configuration files. + * + * @param configurationFiles paths (may contain variables) listed on the environment + * @param variables variables used to resolve paths + * @return set of configured variable names + * @throws HopException if a config file exists but cannot be read + */ + public static Set loadConfiguredVariableNames( + List configurationFiles, IVariables variables) throws HopException { + Set names = new HashSet<>(); + if (configurationFiles == null) { + return names; + } + for (String configurationFile : configurationFiles) { + if (StringUtils.isBlank(configurationFile)) { + continue; + } + String realFilename = variables.resolve(configurationFile); + try { + if (!HopVfs.fileExists(realFilename)) { + continue; + } + DescribedVariablesConfigFile configFile = new DescribedVariablesConfigFile(realFilename); + configFile.readFromFile(); + for (DescribedVariable describedVariable : configFile.getDescribedVariables()) { + if (describedVariable != null && StringUtils.isNotBlank(describedVariable.getName())) { + names.add(describedVariable.getName()); + } + } + } catch (Exception e) { + throw new HopException( + "Error reading variables from environment configuration file '" + realFilename + "'", + e); + } + } + return names; + } + + /** + * Crawl the project associated with the environment using project search and a regex for {@code + * ${…}}, then return described variables not already present in the environment configuration + * files. + * + * @param projectConfig project linked to the lifecycle environment + * @param variables parent variable space (typically HopGui variables) + * @param configurationFiles environment configuration file paths (as currently listed in the + * dialog) + * @param environmentName optional environment name for project variable setup + * @return sorted list of proposed described variables (empty values when unset) + * @throws HopException if the project cannot be loaded or search fails + */ + public static List findMissingVariables( + ProjectConfig projectConfig, + IVariables variables, + List configurationFiles, + String environmentName) + throws HopException { + if (projectConfig == null) { + throw new HopException( + "Please select a project for this environment before importing variables."); + } + + IVariables scanVariables = new Variables(); + scanVariables.initializeFrom(variables); + + Project project = projectConfig.loadProject(scanVariables); + List configFiles = + configurationFiles != null ? configurationFiles : Collections.emptyList(); + project.modifyVariables(scanVariables, projectConfig, configFiles, environmentName); + + IHopMetadataProvider metadataProvider = + HopMetadataUtil.getStandardHopMetadataProvider(scanVariables); + + ProjectsSearchablesLocation location = new ProjectsSearchablesLocation(projectConfig); + ISearchQuery query = new SearchQuery(VARIABLE_EXPRESSION_REGEX, true, true); + + @SuppressWarnings("rawtypes") + Map, ISearchableAnalyser> analysers = + HopGuiSearchHelper.loadSearchableAnalysers(); + List results = + HopGuiSearchHelper.searchLocation( + location, query, analysers, metadataProvider, scanVariables); + + // Preserve a stable order of names while collecting where each was found. + Map> locationsByName = new LinkedHashMap<>(); + Map canonicalNameByKey = new LinkedHashMap<>(); + for (ISearchResult result : results) { + if (result == null) { + continue; + } + // Prefer the full property value so multiple ${…} in one field are all captured. + String text = Const.NVL(result.getValue(), result.getMatchingString()); + Set namesInResult = extractVariableNames(text); + if (namesInResult.isEmpty()) { + continue; + } + String foundAt = formatFoundLocation(result); + for (String name : namesInResult) { + if (isManagedVariable(name)) { + continue; + } + String key = name.toLowerCase(Locale.ROOT); + canonicalNameByKey.putIfAbsent(key, name); + if (StringUtils.isNotEmpty(foundAt)) { + locationsByName.computeIfAbsent(key, k -> new LinkedHashSet<>()).add(foundAt); + } else { + locationsByName.computeIfAbsent(key, k -> new LinkedHashSet<>()); + } + } + } + + Set configured = loadConfiguredVariableNames(configFiles, scanVariables); + + // Sorted presentation order + Set sortedKeys = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + sortedKeys.addAll(canonicalNameByKey.keySet()); + + List proposed = new ArrayList<>(); + for (String key : sortedKeys) { + String name = canonicalNameByKey.get(key); + if (containsIgnoreCase(configured, name)) { + continue; + } + String value = Const.NVL(scanVariables.getVariable(name), ""); + String description = formatLocationsDescription(locationsByName.get(key)); + proposed.add(new DescribedVariable(name, value, description)); + } + return proposed; + } + + /** + * Build a short human-readable location for a search hit (type, object name, field/component). + * + * @param result a project-search match + * @return e.g. {@code Pipeline 'load-data' → Table input → filename}, or empty when unknown + */ + public static String formatFoundLocation(ISearchResult result) { + if (result == null || result.getMatchingSearchable() == null) { + return ""; + } + ISearchable searchable = result.getMatchingSearchable(); + StringBuilder sb = new StringBuilder(); + String type = Const.NVL(searchable.getType(), ""); + String name = Const.NVL(searchable.getName(), ""); + if (StringUtils.isNotEmpty(type) && StringUtils.isNotEmpty(name)) { + sb.append(type).append(" '").append(name).append("'"); + } else if (StringUtils.isNotEmpty(name)) { + sb.append(name); + } else if (StringUtils.isNotEmpty(type)) { + sb.append(type); + } + + String where = HopGuiSearchHelper.matchWhere(result); + if (StringUtils.isNotEmpty(where)) { + if (sb.length() > 0) { + sb.append(" → "); + } + sb.append(where); + } + return sb.toString(); + } + + /** + * Join unique found-at locations into a description string. + * + * @param locations distinct location strings (may be null/empty) + * @return description text, never null + */ + public static String formatLocationsDescription(Set locations) { + if (locations == null || locations.isEmpty()) { + return ""; + } + List list = new ArrayList<>(); + for (String location : locations) { + if (StringUtils.isNotEmpty(location)) { + list.add(location); + } + } + if (list.isEmpty()) { + return ""; + } + if (list.size() <= MAX_DESCRIPTION_LOCATIONS) { + return String.join("; ", list); + } + List head = list.subList(0, MAX_DESCRIPTION_LOCATIONS); + int remaining = list.size() - MAX_DESCRIPTION_LOCATIONS; + return String.join("; ", head) + "; (+" + remaining + " more)"; + } + + /** + * Merge the given variables into a described-variables JSON configuration file (create or + * update). + * + * @param configFilename resolved filesystem path + * @param variables variables to add or update + * @throws HopException on I/O errors + */ + public static void saveVariablesToConfigFile( + String configFilename, List variables) throws HopException { + if (StringUtils.isBlank(configFilename)) { + throw new HopException("Configuration filename is empty"); + } + try { + DescribedVariablesConfigFile configFile = new DescribedVariablesConfigFile(configFilename); + if (HopVfs.fileExists(configFilename)) { + configFile.readFromFile(); + } + if (variables != null) { + for (DescribedVariable variable : variables) { + if (variable != null && StringUtils.isNotBlank(variable.getName())) { + configFile.setDescribedVariable(variable); + } + } + } + configFile.saveToFile(); + } catch (Exception e) { + throw new HopException( + "Error saving variables to configuration file '" + configFilename + "'", e); + } + } + + private static boolean containsIgnoreCase(Set names, String name) { + if (names.contains(name)) { + return true; + } + for (String existing : names) { + if (existing != null && existing.equalsIgnoreCase(name)) { + return true; + } + } + return false; + } + + /** + * @return unmodifiable set of runtime/project managed variable names (for tests) + */ + static Set managedVariableNames() { + return MANAGED_VARIABLE_NAMES; + } + + /** + * Suggest a default config filename for an environment. + * + * @param environmentName environment name + * @return e.g. {@code dev-config.json} + */ + public static String defaultConfigFilename(String environmentName) { + if (StringUtils.isBlank(environmentName)) { + return "environment-config.json"; + } + String sanitized = + environmentName + .trim() + .toLowerCase(Locale.ROOT) + .replaceAll("[^a-z0-9._-]+", "-") + .replaceAll("-+", "-"); + sanitized = StringUtils.strip(sanitized, "-"); + if (sanitized.isEmpty()) { + return "environment-config.json"; + } + return sanitized + "-config.json"; + } +} diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironment.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironment.java index 9dbf5ed793f..bb4b68e5c03 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironment.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironment.java @@ -30,6 +30,8 @@ public class LifecycleEnvironment { private String projectName; + private String canvasText; + private List configurationFiles; public LifecycleEnvironment() { @@ -48,6 +50,7 @@ public LifecycleEnvironment(LifecycleEnvironment env) { this.name = env.name; this.purpose = env.purpose; this.projectName = env.projectName; + this.canvasText = env.canvasText; this.configurationFiles = new ArrayList<>(env.configurationFiles); } @@ -116,6 +119,22 @@ public void setProjectName(String projectName) { this.projectName = projectName; } + /** + * Gets canvasText — optional large watermark drawn top-right on pipeline/workflow canvases. + * + * @return value of canvasText + */ + public String getCanvasText() { + return canvasText; + } + + /** + * @param canvasText The canvas text to set (empty/null means do not draw) + */ + public void setCanvasText(String canvasText) { + this.canvasText = canvasText; + } + /** * Gets configurationFiles * diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironmentDialog.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironmentDialog.java index 3a19b4c9171..d70f99078d5 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironmentDialog.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironmentDialog.java @@ -17,11 +17,15 @@ package org.apache.hop.projects.environment; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import org.apache.commons.lang3.StringUtils; import org.apache.commons.vfs2.FileObject; import org.apache.hop.core.Const; import org.apache.hop.core.config.DescribedVariablesConfigFile; import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.variables.DescribedVariable; import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.vfs.HopVfs; import org.apache.hop.i18n.BaseMessages; @@ -31,7 +35,9 @@ import org.apache.hop.ui.core.ConstUi; import org.apache.hop.ui.core.PropsUi; import org.apache.hop.ui.core.dialog.BaseDialog; +import org.apache.hop.ui.core.dialog.EnterSelectionDialog; import org.apache.hop.ui.core.dialog.ErrorDialog; +import org.apache.hop.ui.core.dialog.HopDescribedVariablesDialog; import org.apache.hop.ui.core.dialog.MessageBox; import org.apache.hop.ui.core.gui.GuiResource; import org.apache.hop.ui.core.gui.WindowProperty; @@ -67,15 +73,28 @@ public class LifecycleEnvironmentDialog extends Dialog { private Text wName; private Combo wPurpose; private Combo wProject; + private Text wCanvasText; private TableView wConfigFiles; private IVariables variables; private Button wbEdit; + private Button wbImportVariables; private String originalName; private boolean needingEnvironmentRefresh; + /** Last name we auto-suggested (for detecting when the user edits away from it). */ + private String lastSuggestedName; + + /** When true, project/purpose changes rewrite the name field (new environments only). */ + private boolean nameAutoManaged; + + private boolean updatingSuggestedName; + + /** Localized purpose label → fixed English suffix (for new-environment name suggestion). */ + private Map knownPurposeSuffixes; + public LifecycleEnvironmentDialog( Shell parent, LifecycleEnvironment environment, IVariables variables) { super(parent, SWT.DIALOG_TRIM | SWT.APPLICATION_MODAL | SWT.RESIZE); @@ -88,6 +107,9 @@ public LifecycleEnvironmentDialog( props = PropsUi.getInstance(); needingEnvironmentRefresh = false; + lastSuggestedName = null; + nameAutoManaged = StringUtils.isEmpty(originalName); + updatingSuggestedName = false; } public String open() { @@ -139,6 +161,7 @@ public String open() { fdName.right = new FormAttachment(100, 0); fdName.top = new FormAttachment(wlName, 0, SWT.CENTER); wName.setLayoutData(fdName); + wName.addListener(SWT.Modify, e -> onNameModified()); Control lastControl = wName; Label wlPurpose = new Label(shell, SWT.RIGHT); @@ -157,7 +180,12 @@ public String open() { fdPurpose.right = new FormAttachment(100, 0); fdPurpose.top = new FormAttachment(wlPurpose, 0, SWT.CENTER); wPurpose.setLayoutData(fdPurpose); - wPurpose.addListener(SWT.Modify, e -> needingEnvironmentRefresh = true); + wPurpose.addListener( + SWT.Modify, + e -> { + needingEnvironmentRefresh = true; + updateSuggestedName(); + }); lastControl = wPurpose; Label wlProject = new Label(shell, SWT.RIGHT); @@ -176,9 +204,34 @@ public String open() { fdProject.right = new FormAttachment(100, 0); fdProject.top = new FormAttachment(wlProject, 0, SWT.CENTER); wProject.setLayoutData(fdProject); - wProject.addListener(SWT.Modify, e -> needingEnvironmentRefresh = true); + wProject.addListener( + SWT.Modify, + e -> { + needingEnvironmentRefresh = true; + updateSuggestedName(); + }); lastControl = wProject; + Label wlCanvasText = new Label(shell, SWT.RIGHT); + PropsUi.setLook(wlCanvasText); + wlCanvasText.setText( + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Label.CanvasText")); + FormData fdlCanvasText = new FormData(); + fdlCanvasText.left = new FormAttachment(0, 0); + fdlCanvasText.right = new FormAttachment(middle, 0); + fdlCanvasText.top = new FormAttachment(lastControl, margin); + wlCanvasText.setLayoutData(fdlCanvasText); + wCanvasText = new Text(shell, SWT.SINGLE | SWT.BORDER | SWT.LEFT); + PropsUi.setLook(wCanvasText); + wCanvasText.setToolTipText( + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.ToolTip.CanvasText")); + FormData fdCanvasText = new FormData(); + fdCanvasText.left = new FormAttachment(middle, margin); + fdCanvasText.right = new FormAttachment(100, 0); + fdCanvasText.top = new FormAttachment(wlCanvasText, 0, SWT.CENTER); + wCanvasText.setLayoutData(fdCanvasText); + lastControl = wCanvasText; + Label wlConfigFiles = new Label(shell, SWT.LEFT); PropsUi.setLook(wlConfigFiles); wlConfigFiles.setText( @@ -189,15 +242,48 @@ public String open() { fdlConfigFiles.top = new FormAttachment(lastControl, margin); wlConfigFiles.setLayoutData(fdlConfigFiles); + // Bottons to the right. + // + wbImportVariables = new Button(shell, SWT.PUSH); + PropsUi.setLook(wbImportVariables); + wbImportVariables.setText( + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Button.ImportVariables")); + FormData fdImportVariables = new FormData(); + fdImportVariables.right = new FormAttachment(100, 0); + fdImportVariables.top = new FormAttachment(wlConfigFiles, margin); + wbImportVariables.setLayoutData(fdImportVariables); + wbImportVariables.addListener(SWT.Selection, this::importVariables); + Button wbSelect = new Button(shell, SWT.PUSH); PropsUi.setLook(wbSelect); wbSelect.setText(BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Button.Select")); FormData fdAdd = new FormData(); + fdAdd.left = new FormAttachment(wbImportVariables, 0, SWT.LEFT); fdAdd.right = new FormAttachment(100, 0); - fdAdd.top = new FormAttachment(wlConfigFiles, margin); + fdAdd.top = new FormAttachment(wbImportVariables, margin); wbSelect.setLayoutData(fdAdd); wbSelect.addListener(SWT.Selection, this::addConfigFile); + Button wbNew = new Button(shell, SWT.PUSH); + PropsUi.setLook(wbNew); + wbNew.setText(BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Button.New")); + FormData fdNew = new FormData(); + fdNew.left = new FormAttachment(wbImportVariables, 0, SWT.LEFT); + fdNew.right = new FormAttachment(100, 0); + fdNew.top = new FormAttachment(wbSelect, margin); + wbNew.setLayoutData(fdNew); + wbNew.addListener(SWT.Selection, this::newConfigFile); + + wbEdit = new Button(shell, SWT.PUSH); + PropsUi.setLook(wbEdit); + wbEdit.setText(BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Button.Edit")); + FormData fdEdit = new FormData(); + fdEdit.left = new FormAttachment(wbImportVariables, 0, SWT.LEFT); + fdEdit.right = new FormAttachment(100, 0); + fdEdit.top = new FormAttachment(wbNew, margin); + wbEdit.setLayoutData(fdEdit); + wbEdit.addListener(SWT.Selection, this::editConfigFile); + ColumnInfo[] columnInfo = new ColumnInfo[] { new ColumnInfo( @@ -220,32 +306,12 @@ public String open() { PropsUi.setLook(wConfigFiles); FormData fdConfigFiles = new FormData(); fdConfigFiles.left = new FormAttachment(0, 0); - fdConfigFiles.right = new FormAttachment(wbSelect, -2 * margin); + fdConfigFiles.right = new FormAttachment(wbImportVariables, -2 * margin); fdConfigFiles.top = new FormAttachment(wlConfigFiles, margin); fdConfigFiles.bottom = new FormAttachment(wOK, -margin * 2); wConfigFiles.setLayoutData(fdConfigFiles); wConfigFiles.table.addListener(SWT.Selection, this::setButtonStates); - Button wbNew = new Button(shell, SWT.PUSH); - PropsUi.setLook(wbNew); - wbNew.setText(BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Button.New")); - FormData fdNew = new FormData(); - fdNew.left = new FormAttachment(wConfigFiles, 2 * margin); - fdNew.right = new FormAttachment(100, 0); - fdNew.top = new FormAttachment(wbSelect, margin); - wbNew.setLayoutData(fdNew); - wbNew.addListener(SWT.Selection, this::newConfigFile); - - wbEdit = new Button(shell, SWT.PUSH); - PropsUi.setLook(wbEdit); - wbEdit.setText(BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Button.Edit")); - FormData fdEdit = new FormData(); - fdEdit.left = new FormAttachment(wConfigFiles, 2 * margin); - fdEdit.right = new FormAttachment(100, 0); - fdEdit.top = new FormAttachment(wbNew, margin); - wbEdit.setLayoutData(fdEdit); - wbEdit.addListener(SWT.Selection, this::editConfigFile); - getData(); wName.setFocus(); @@ -371,6 +437,189 @@ private void setButtonStates(Event event) { wbEdit.setGrayed(index < 0); } + /** + * Crawl the project for {@code ${VARIABLE}} expressions not yet defined in environment config + * files, review them, and write into a new or existing configuration file. + */ + private void importVariables(Event event) { + shell.setCursor(shell.getDisplay().getSystemCursor(SWT.CURSOR_WAIT)); + try { + String projectName = wProject.getText(); + if (StringUtils.isEmpty(projectName)) { + MessageBox box = new MessageBox(shell, SWT.OK | SWT.ICON_ERROR); + box.setText( + BaseMessages.getString( + PKG, "LifecycleEnvironmentDialog.ImportVariables.ProjectRequired.Title")); + box.setMessage( + BaseMessages.getString( + PKG, "LifecycleEnvironmentDialog.ImportVariables.ProjectRequired.Message")); + box.open(); + return; + } + + ProjectConfig projectConfig = + ProjectsConfigSingleton.getConfig().findProjectConfig(projectName); + if (projectConfig == null) { + throw new HopException( + "Project '" + projectName + "' is not configured in Hop (projects config)."); + } + + List configurationFiles = new ArrayList<>(); + for (TableItem item : wConfigFiles.getNonEmptyItems()) { + configurationFiles.add(item.getText(1)); + } + + List proposed = + EnvironmentVariablesImportHelper.findMissingVariables( + projectConfig, variables, configurationFiles, wName.getText()); + + // Restore the pointer before modal dialogs so the user can interact normally. + shell.setCursor(null); + + if (proposed.isEmpty()) { + MessageBox box = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION); + box.setText( + BaseMessages.getString( + PKG, "LifecycleEnvironmentDialog.ImportVariables.NoneFound.Title")); + box.setMessage( + BaseMessages.getString( + PKG, "LifecycleEnvironmentDialog.ImportVariables.NoneFound.Message")); + box.open(); + return; + } + + HopDescribedVariablesDialog variablesDialog = + new HopDescribedVariablesDialog( + shell, + BaseMessages.getString( + PKG, "LifecycleEnvironmentDialog.ImportVariables.DialogMessage", projectName), + proposed, + null); + List confirmed = variablesDialog.open(); + if (confirmed == null) { + return; + } + + String configFilename = chooseConfigFileForImport(configurationFiles, projectName); + if (StringUtils.isEmpty(configFilename)) { + return; + } + + shell.setCursor(shell.getDisplay().getSystemCursor(SWT.CURSOR_WAIT)); + String realConfigFilename = variables.resolve(configFilename); + EnvironmentVariablesImportHelper.saveVariablesToConfigFile(realConfigFilename, confirmed); + shell.setCursor(null); + + // Ensure the path is listed on the environment + boolean alreadyListed = false; + for (TableItem item : wConfigFiles.getNonEmptyItems()) { + if (configFilename.equals(item.getText(1)) + || realConfigFilename.equals(variables.resolve(item.getText(1)))) { + alreadyListed = true; + break; + } + } + if (!alreadyListed) { + TableItem item = new TableItem(wConfigFiles.table, SWT.NONE); + item.setText(1, configFilename); + wConfigFiles.removeEmptyRows(); + wConfigFiles.setRowNums(); + wConfigFiles.optWidth(true); + wConfigFiles.table.setSelection(item); + } + + needingEnvironmentRefresh = true; + + MessageBox done = new MessageBox(shell, SWT.OK | SWT.ICON_INFORMATION); + done.setText( + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.ImportVariables.Saved.Title")); + done.setMessage( + BaseMessages.getString( + PKG, + "LifecycleEnvironmentDialog.ImportVariables.Saved.Message", + Integer.toString(confirmed.size()), + realConfigFilename)); + done.open(); + } catch (Exception e) { + new ErrorDialog( + shell, + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.ImportVariables.Error.Title"), + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.ImportVariables.Error.Message"), + e); + } finally { + if (shell != null && !shell.isDisposed()) { + shell.setCursor(null); + } + } + } + + /** + * Pick an existing configuration file from the environment list, or create a new one. + * + * @return selected/created path (may contain variables), or null if cancelled + */ + private String chooseConfigFileForImport(List configurationFiles, String projectName) + throws Exception { + String createNewLabel = + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.ImportVariables.CreateNewOption"); + + if (configurationFiles != null && !configurationFiles.isEmpty()) { + List choices = new ArrayList<>(configurationFiles); + choices.add(createNewLabel); + EnterSelectionDialog selectionDialog = + new EnterSelectionDialog( + shell, + choices.toArray(new String[0]), + BaseMessages.getString( + PKG, "LifecycleEnvironmentDialog.ImportVariables.SelectConfig.Title"), + BaseMessages.getString( + PKG, "LifecycleEnvironmentDialog.ImportVariables.SelectConfig.Message")); + selectionDialog.setAvoidQuickSearch(); + String selected = selectionDialog.open(); + if (selected == null) { + return null; + } + if (!createNewLabel.equals(selected)) { + return selected; + } + } + + return promptNewConfigFilename(projectName); + } + + private String promptNewConfigFilename(String projectName) throws Exception { + String environmentName = Const.NVL(wName.getText(), projectName); + String defaultName = EnvironmentVariablesImportHelper.defaultConfigFilename(environmentName); + + FileObject startFile; + ProjectConfig projectConfig = + ProjectsConfigSingleton.getConfig().findProjectConfig(projectName); + if (projectConfig != null) { + String filename = + projectConfig.getProjectHome() + + Const.FILE_SEPARATOR + + ".." + + Const.FILE_SEPARATOR + + defaultName; + startFile = HopVfs.getFileObject(variables.resolve(filename)); + } else { + startFile = HopVfs.getFileObject(defaultName); + } + + return BaseDialog.presentFileDialog( + true, + shell, + null, + variables, + startFile, + new String[] {"*.json", "*"}, + new String[] { + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.ImportVariables.FileFilter.Json"), + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.ImportVariables.FileFilter.All") + }, + true); + } + private void ok() { try { @@ -408,18 +657,53 @@ public void dispose() { private void getData() { ProjectsConfig config = ProjectsConfigSingleton.getConfig(); + String developmentLabel = + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Purpose.Text.Development"); + String testingLabel = + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Purpose.Text.Testing"); + String acceptanceLabel = + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Purpose.Text.Acceptance"); + String productionLabel = + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Purpose.Text.Production"); + String continuousIntegrationLabel = + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Purpose.Text.CI"); + String commonBuildLabel = + BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Purpose.Text.CB"); + + knownPurposeSuffixes = + LifecycleEnvironmentNaming.knownPurposeSuffixes( + developmentLabel, + testingLabel, + acceptanceLabel, + productionLabel, + continuousIntegrationLabel, + commonBuildLabel); + wProject.setItems(config.listProjectConfigNames().toArray(new String[0])); wPurpose.setItems( - BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Purpose.Text.Development"), - BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Purpose.Text.Testing"), - BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Purpose.Text.Acceptance"), - BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Purpose.Text.Production"), - BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Purpose.Text.CI"), - BaseMessages.getString(PKG, "LifecycleEnvironmentDialog.Purpose.Text.CB")); - - wName.setText(Const.NVL(environment.getName(), "")); + developmentLabel, + testingLabel, + acceptanceLabel, + productionLabel, + continuousIntegrationLabel, + commonBuildLabel); + + // Setting project/purpose may fire Modify and update the suggested name for new envs. wPurpose.setText(Const.NVL(environment.getPurpose(), "")); wProject.setText(Const.NVL(environment.getProjectName(), "")); + wCanvasText.setText(Const.NVL(environment.getCanvasText(), "")); + + if (StringUtils.isNotEmpty(environment.getName())) { + // Provided name (edit or rare pre-fill): do not auto-overwrite. + nameAutoManaged = false; + setNameText(environment.getName()); + } else if (isNewEnvironment()) { + nameAutoManaged = true; + // Project/purpose listeners may already have suggested a name; ensure we do so if not. + updateSuggestedName(); + } else { + setNameText(""); + } for (int i = 0; i < environment.getConfigurationFiles().size(); i++) { String configurationFile = environment.getConfigurationFiles().get(i); @@ -437,10 +721,58 @@ private void getData() { } } + private boolean isNewEnvironment() { + return StringUtils.isEmpty(originalName); + } + + /** + * When creating a new environment, keep the name field in sync with project + purpose until the + * user edits the name manually. + */ + private void updateSuggestedName() { + if (!isNewEnvironment() + || !nameAutoManaged + || wName == null + || wProject == null + || wPurpose == null) { + return; + } + + String suggested = + LifecycleEnvironmentNaming.suggestEnvironmentName( + wProject.getText(), wPurpose.getText(), knownPurposeSuffixes); + lastSuggestedName = suggested; + setNameText(suggested); + } + + private void setNameText(String text) { + updatingSuggestedName = true; + try { + wName.setText(Const.NVL(text, "")); + } finally { + updatingSuggestedName = false; + } + } + + private void onNameModified() { + if (updatingSuggestedName || !isNewEnvironment()) { + return; + } + String current = wName.getText(); + // Cleared field or still matching last suggestion → keep auto-managing. + if (StringUtils.isEmpty(current) + || (lastSuggestedName != null && current.equals(lastSuggestedName))) { + nameAutoManaged = true; + } else { + nameAutoManaged = false; + } + } + private void getInfo(LifecycleEnvironment env) { env.setName(wName.getText()); env.setPurpose(wPurpose.getText()); env.setProjectName(wProject.getText()); + env.setCanvasText(wCanvasText.getText()); env.getConfigurationFiles().clear(); for (TableItem item : wConfigFiles.getNonEmptyItems()) { diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironmentNaming.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironmentNaming.java new file mode 100644 index 00000000000..0337957b462 --- /dev/null +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/environment/LifecycleEnvironmentNaming.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.environment; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; + +/** + * Suggests default environment names from a project name and purpose. Used when creating a new + * environment so the name field can be pre-filled and updated as the user picks a purpose. + */ +public final class LifecycleEnvironmentNaming { + + /** Fixed English technical suffixes for standard purposes. */ + public static final String SUFFIX_DEVELOPMENT = "development"; + + public static final String SUFFIX_TEST = "test"; + public static final String SUFFIX_ACCEPTANCE = "acceptance"; + public static final String SUFFIX_PRODUCTION = "production"; + public static final String SUFFIX_CONTINUOUS_INTEGRATION = "continuous-integration"; + public static final String SUFFIX_COMMON_BUILD = "common-build"; + + private LifecycleEnvironmentNaming() { + // utility + } + + /** + * Builds a map from known purpose display labels (localized) to fixed English suffixes. + * + * @param developmentLabel localized "Development" (or equivalent) + * @param testingLabel localized "Testing" + * @param acceptanceLabel localized "Acceptance" + * @param productionLabel localized "Production" + * @param continuousIntegrationLabel localized "Continuous Integration" + * @param commonBuildLabel localized "Common Build" + * @return unmodifiable map of label → suffix + */ + public static Map knownPurposeSuffixes( + String developmentLabel, + String testingLabel, + String acceptanceLabel, + String productionLabel, + String continuousIntegrationLabel, + String commonBuildLabel) { + Map map = new LinkedHashMap<>(); + putIfNotBlank(map, developmentLabel, SUFFIX_DEVELOPMENT); + putIfNotBlank(map, testingLabel, SUFFIX_TEST); + putIfNotBlank(map, acceptanceLabel, SUFFIX_ACCEPTANCE); + putIfNotBlank(map, productionLabel, SUFFIX_PRODUCTION); + putIfNotBlank(map, continuousIntegrationLabel, SUFFIX_CONTINUOUS_INTEGRATION); + putIfNotBlank(map, commonBuildLabel, SUFFIX_COMMON_BUILD); + return Collections.unmodifiableMap(map); + } + + private static void putIfNotBlank(Map map, String label, String suffix) { + if (StringUtils.isNotBlank(label)) { + map.put(label, suffix); + } + } + + /** + * Maps a purpose string to a technical suffix for use in an environment name. + * + *

Known purpose labels (from {@code knownPurposeLabelsToSuffix}) map to fixed English + * suffixes. Free-text purposes are sanitized to a lowercase slug. + * + * @param purpose purpose combo text (may be localized label or free text) + * @param knownPurposeLabelsToSuffix map of known display labels → fixed suffixes + * @return suffix without leading dash, or empty string if purpose is blank + */ + public static String purposeToSuffix( + String purpose, Map knownPurposeLabelsToSuffix) { + if (StringUtils.isBlank(purpose)) { + return ""; + } + String trimmed = purpose.trim(); + if (knownPurposeLabelsToSuffix != null) { + String known = knownPurposeLabelsToSuffix.get(trimmed); + if (known != null) { + return known; + } + } + return sanitizePurposeSuffix(trimmed); + } + + /** + * Suggests an environment name from project and purpose suffix. + * + * @param projectName project name (required for a non-empty suggestion) + * @param purposeSuffix technical suffix without leading dash; empty means project only + * @return suggested name, or empty string if project is blank + */ + public static String suggestEnvironmentName(String projectName, String purposeSuffix) { + if (StringUtils.isBlank(projectName)) { + return ""; + } + String project = projectName.trim(); + if (StringUtils.isBlank(purposeSuffix)) { + return project; + } + return project + "-" + purposeSuffix.trim(); + } + + /** + * Suggests an environment name from project and purpose text. + * + * @param projectName project name + * @param purpose purpose combo text + * @param knownPurposeLabelsToSuffix map of known display labels → fixed suffixes + * @return suggested environment name + */ + public static String suggestEnvironmentName( + String projectName, String purpose, Map knownPurposeLabelsToSuffix) { + return suggestEnvironmentName( + projectName, purposeToSuffix(purpose, knownPurposeLabelsToSuffix)); + } + + /** + * Sanitizes free-text purpose into a stable technical suffix (lowercase, hyphenated). + * + * @param purpose free-text purpose + * @return sanitized suffix, or empty if nothing usable remains + */ + public static String sanitizePurposeSuffix(String purpose) { + if (StringUtils.isBlank(purpose)) { + return ""; + } + String sanitized = + purpose + .trim() + .toLowerCase(Locale.ROOT) + .replaceAll("[^a-z0-9._-]+", "-") + .replaceAll("-+", "-"); + return StringUtils.strip(sanitized, "-"); + } +} diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java index b4fd7e21305..68001603d6e 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/gui/ProjectsGuiPlugin.java @@ -166,7 +166,12 @@ public static void enableHopGuiProject( return; } - // Close's all + // Save execution perspective state (toolbar filters + open tabs) under the current project + // namespace before closing tabs / switching namespace. + // + ExecutionPerspective.getInstance().saveState(); + + // Close's all (including execution information tabs) // hopGui.fileDelegate.closeAllFiles(); @@ -179,7 +184,6 @@ public static void enableHopGuiProject( // Save explorer perspective state for the current project before switching namespace // ExplorerPerspective.getInstance().saveExplorerStateOnShutdown(); - ExecutionPerspective.getInstance().saveState(); // This is called only in Hop GUI so we want to start with a new set of variables // It avoids variables from one project showing up in another @@ -1655,6 +1659,7 @@ public void menuProjectExport() { && !name.contains("HOP_AUDIT_FOLDER") && !name.contains("HOP_CONFIG_FOLDER") && !name.contains("PROJECT_HOME") + && !name.contains("PARENT_PROJECT_NAME") && !name.contains("HOP_PROJECTS") && !name.contains("HOP_PLATFORM_OS") && !name.contains("HOP_PROJECT_NAME") diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ManageProjectsOptionPlugin.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ManageProjectsOptionPlugin.java index e925f03431e..6607ddc8648 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ManageProjectsOptionPlugin.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/ManageProjectsOptionPlugin.java @@ -386,7 +386,13 @@ private void createProject(ILogChannel log, ProjectsConfig config, IVariables va log.logBasic(CONST_PROJECT + projectName + "' was created for home folder : " + projectHome); Project project = projectConfig.loadProject(variables); - project.setParentProjectName(config.getStandardParentProject()); + // Keep an existing parent from a pre-existing project-config.json (Docker + // --project-keep-config-file). Only fall back to the standard parent when none is set. + // --project-parent still wins via modifyProjectSettings below. + // + if (StringUtils.isEmpty(project.getParentProjectName())) { + project.setParentProjectName(config.getStandardParentProject()); + } modifyProjectSettings(project); // Check to see if there's not a loop in the project parent hierarchy diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java index 7c76fa8872e..f2ceff0df1c 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/project/Project.java @@ -180,10 +180,11 @@ public void modifyVariables( // definition as well // Project parentProject = null; + ProjectConfig parentProjectConfig = null; String realParentProjectName = variables.resolve(parentProjectName); if (StringUtils.isNotEmpty(realParentProjectName)) { - ProjectConfig parentProjectConfig = + parentProjectConfig = ProjectsConfigSingleton.getConfig().findProjectConfig(realParentProjectName); if (parentProjectConfig != null) { try { @@ -199,6 +200,20 @@ public void modifyVariables( } } + // Expose the immediate parent project (if any) for path references like + // ${PARENT_PROJECT_HOME}/shared/pipeline.hpl. Clear when missing so project + // switches do not leave stale values. + // + if (parentProjectConfig != null && parentProject != null) { + variables.setVariable( + ProjectsUtil.VARIABLE_PARENT_PROJECT_NAME, Const.NVL(realParentProjectName, "")); + String parentHome = variables.resolve(parentProjectConfig.getProjectHome()); + variables.setVariable(ProjectsUtil.VARIABLE_PARENT_PROJECT_HOME, Const.NVL(parentHome, "")); + } else { + variables.setVariable(ProjectsUtil.VARIABLE_PARENT_PROJECT_NAME, ""); + variables.setVariable(ProjectsUtil.VARIABLE_PARENT_PROJECT_HOME, ""); + } + // Set the name of the active environment // variables.setVariable( diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/util/PathVariableReplacer.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/util/PathVariableReplacer.java new file mode 100644 index 00000000000..52e7e3f13ba --- /dev/null +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/util/PathVariableReplacer.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.util; + +import java.util.Locale; +import java.util.Set; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.vfs2.FileObject; +import org.apache.hop.core.Const; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.HopVfs; + +/** + * Rewrites absolute paths to use the most specific matching path-like variable from the active + * variable space (for example {@code ${PROJECT_HOME}/file.csv} or {@code + * ${SOURCE_FILES}/input.csv}). + * + *

Used by Hop GUI file/directory browse extension points as a best-practice aid. + */ +public final class PathVariableReplacer { + + private static final Set SYSTEM_NAME_PREFIXES = + Set.of( + "java.", "sun.", "user.", "os.", "file.", "jdk.", "http.", "awt.", "path.", "line.", + "ftp."); + + private static final Set NON_PATH_VARIABLE_NAMES = + Set.of( + Defaults.VARIABLE_HOP_PROJECT_NAME, + Defaults.VARIABLE_HOP_ENVIRONMENT_NAME, + ProjectsUtil.VARIABLE_PARENT_PROJECT_NAME); + + private PathVariableReplacer() { + // Utility class + } + + /** + * If {@code path} is under one or more path-like variable values in {@code variables}, rewrite + * the longest matching prefix to {@code ${NAME}} or {@code ${NAME}/remainder}. + * + * @param variables active variables (may be null) + * @param path selected file or directory path (may be null/empty) + * @return rewritten path, or the original path when no replacement applies + */ + public static String replacePathWithVariable(IVariables variables, String path) { + if (variables == null || StringUtils.isEmpty(path)) { + return path; + } + + try { + String absolutePath = normalizeAbsolutePath(path); + if (StringUtils.isEmpty(absolutePath)) { + return path; + } + + PathMatch best = null; + for (String name : variables.getVariableNames()) { + if (!isCandidateVariableName(name)) { + continue; + } + String rawValue = variables.getVariable(name); + if (StringUtils.isEmpty(rawValue)) { + continue; + } + String resolved = variables.resolve(rawValue); + if (!isCandidatePathValue(resolved)) { + continue; + } + + String absoluteRoot; + try { + absoluteRoot = normalizeAbsolutePath(resolved); + } catch (Exception e) { + continue; + } + if (!isUsablePathRoot(absoluteRoot)) { + continue; + } + + if (absolutePath.equals(absoluteRoot)) { + PathMatch match = new PathMatch(name, absoluteRoot, ""); + best = betterMatch(best, match); + } else if (absolutePath.startsWith(absoluteRoot + "/")) { + String remainder = absolutePath.substring(absoluteRoot.length() + 1); + PathMatch match = new PathMatch(name, absoluteRoot, remainder); + best = betterMatch(best, match); + } + } + + if (best == null) { + return path; + } + if (StringUtils.isEmpty(best.remainder)) { + return "${" + best.variableName + "}"; + } + return "${" + best.variableName + "}/" + best.remainder; + } catch (Exception e) { + return path; + } + } + + /** + * Whether a variable name is eligible to be used as a path replacement root. + * + * @param name variable name + * @return true if the name may be considered + */ + public static boolean isCandidateVariableName(String name) { + if (StringUtils.isBlank(name)) { + return false; + } + if (name.startsWith(Const.INTERNAL_VARIABLE_PREFIX)) { + return false; + } + if (NON_PATH_VARIABLE_NAMES.contains(name)) { + return false; + } + String lower = name.toLowerCase(Locale.ROOT); + for (String prefix : SYSTEM_NAME_PREFIXES) { + if (lower.startsWith(prefix)) { + return false; + } + } + return true; + } + + /** + * Whether a resolved variable value looks like a usable path root before VFS normalization. + * + * @param value resolved variable value + * @return true if the value may be a path + */ + public static boolean isCandidatePathValue(String value) { + if (StringUtils.isBlank(value)) { + return false; + } + // Incomplete resolution or multi-folder lists (e.g. HOP_METADATA_FOLDER parent,child) + if (value.contains("${") || value.contains(",")) { + return false; + } + // Absolute local path, Windows drive path, UNC, or VFS URI + if (value.startsWith("/") || value.startsWith("\\")) { + return true; + } + if (value.length() >= 3 + && Character.isLetter(value.charAt(0)) + && value.charAt(1) == ':' + && (value.charAt(2) == '/' || value.charAt(2) == '\\')) { + return true; + } + // VFS / URL style: scheme:// + int schemeSep = value.indexOf("://"); + if (schemeSep > 0) { + return true; + } + // Relative paths and bare names are not used as replacement roots + return false; + } + + static boolean isUsablePathRoot(String absoluteRoot) { + if (StringUtils.isEmpty(absoluteRoot)) { + return false; + } + // Too broad: filesystem root only + if ("/".equals(absoluteRoot) || absoluteRoot.matches("^[A-Za-z]:$")) { + return false; + } + // Windows root like C:/ or C:\ + if (absoluteRoot.matches("^[A-Za-z]:[\\\\/]$")) { + return false; + } + return true; + } + + static String normalizeAbsolutePath(String path) throws Exception { + FileObject file = HopVfs.getFileObject(path); + String absolute = file.getName().getPath(); + // Strip trailing slash except for root + while (absolute.length() > 1 && absolute.endsWith("/")) { + absolute = absolute.substring(0, absolute.length() - 1); + } + return absolute; + } + + private static PathMatch betterMatch(PathMatch current, PathMatch candidate) { + if (current == null) { + return candidate; + } + int lengthCompare = Integer.compare(candidate.rootLength(), current.rootLength()); + if (lengthCompare > 0) { + return candidate; + } + if (lengthCompare < 0) { + return current; + } + // Equal length: prefer PROJECT_HOME, then stable name order + if (ProjectsUtil.VARIABLE_PROJECT_HOME.equals(candidate.variableName) + && !ProjectsUtil.VARIABLE_PROJECT_HOME.equals(current.variableName)) { + return candidate; + } + if (ProjectsUtil.VARIABLE_PROJECT_HOME.equals(current.variableName) + && !ProjectsUtil.VARIABLE_PROJECT_HOME.equals(candidate.variableName)) { + return current; + } + return candidate.variableName.compareTo(current.variableName) < 0 ? candidate : current; + } + + private static final class PathMatch { + final String variableName; + final String absoluteRoot; + final String remainder; + + PathMatch(String variableName, String absoluteRoot, String remainder) { + this.variableName = variableName; + this.absoluteRoot = absoluteRoot; + this.remainder = remainder; + } + + int rootLength() { + return absoluteRoot.length(); + } + } +} diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/util/ProjectsUtil.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/util/ProjectsUtil.java index d6a6e709711..ac30f082a83 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/util/ProjectsUtil.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/util/ProjectsUtil.java @@ -46,6 +46,8 @@ public class ProjectsUtil { public static final String VARIABLE_PROJECT_HOME = "PROJECT_HOME"; + public static final String VARIABLE_PARENT_PROJECT_HOME = "PARENT_PROJECT_HOME"; + public static final String VARIABLE_PARENT_PROJECT_NAME = "PARENT_PROJECT_NAME"; public static final String VARIABLE_HOP_DATASETS_FOLDER = "HOP_DATASETS_FOLDER"; public static final String VARIABLE_HOP_UNIT_TESTS_FOLDER = "HOP_UNIT_TESTS_FOLDER"; diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/DrawEnvironmentCanvasTextExtension.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/DrawEnvironmentCanvasTextExtension.java new file mode 100644 index 00000000000..ff142adc525 --- /dev/null +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/DrawEnvironmentCanvasTextExtension.java @@ -0,0 +1,89 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.xp; + +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.gui.BasePainter; +import org.apache.hop.core.gui.IGc; +import org.apache.hop.core.gui.Point; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.projects.config.ProjectsConfigSingleton; +import org.apache.hop.projects.environment.LifecycleEnvironment; +import org.apache.hop.projects.util.Defaults; + +/** + * Shared helper that draws optional environment canvas text as a large watermark in the top-right + * of the pipeline/workflow canvas (under graph content when used from *PainterStart). + */ +final class DrawEnvironmentCanvasTextExtension { + + private DrawEnvironmentCanvasTextExtension() { + // utility + } + + static void draw(BasePainter painter, IVariables variables) { + if (painter == null || variables == null) { + return; + } + + String environmentName = variables.getVariable(Defaults.VARIABLE_HOP_ENVIRONMENT_NAME); + if (StringUtils.isEmpty(environmentName)) { + return; + } + + LifecycleEnvironment environment = + ProjectsConfigSingleton.getConfig().findEnvironment(environmentName); + if (environment == null || StringUtils.isEmpty(environment.getCanvasText())) { + return; + } + + String text = variables.resolve(environment.getCanvasText()); + if (StringUtils.isEmpty(text)) { + return; + } + + IGc gc = painter.getGc(); + Point area = painter.getArea(); + if (gc == null || area == null || area.x <= 0 || area.y <= 0) { + return; + } + + // PainterStart runs under graph magnification. Draw in screen pixels so the label + // stays fixed-size while zooming (same approach as the navigation view). + float mag = gc.getMagnification(); + gc.setTransform(0.0f, 0.0f, 1.0f); + + try { + gc.setFont("Sans", 48, true, false); + Point extent = gc.textExtent(text); + + int margin = 16; + int x = area.x - extent.x - margin; + int y = margin; + + int oldAlpha = gc.getAlpha(); + gc.setAlpha(100); + gc.setForeground(IGc.EColor.DARKGRAY); + gc.drawText(text, x, y, true); + gc.setAlpha(oldAlpha); + } finally { + // Restore graph magnification for the rest of the painter + gc.setTransform(0.0f, 0.0f, mag); + } + } +} diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/DrawEnvironmentOnPipelineExtensionPoint.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/DrawEnvironmentOnPipelineExtensionPoint.java new file mode 100644 index 00000000000..5b702862b49 --- /dev/null +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/DrawEnvironmentOnPipelineExtensionPoint.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.xp; + +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.extension.ExtensionPoint; +import org.apache.hop.core.extension.IExtensionPoint; +import org.apache.hop.core.logging.ILogChannel; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.pipeline.PipelinePainter; + +@ExtensionPoint( + id = "DrawEnvironmentOnPipelineExtensionPoint", + description = "Draws optional environment canvas text top-right under pipeline graph content", + extensionPointId = "PipelinePainterStart") +public class DrawEnvironmentOnPipelineExtensionPoint implements IExtensionPoint { + + @Override + public void callExtensionPoint( + ILogChannel log, IVariables variables, PipelinePainter pipelinePainter) throws HopException { + DrawEnvironmentCanvasTextExtension.draw(pipelinePainter, variables); + } +} diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/DrawEnvironmentOnWorkflowExtensionPoint.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/DrawEnvironmentOnWorkflowExtensionPoint.java new file mode 100644 index 00000000000..518a7b20ad3 --- /dev/null +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/DrawEnvironmentOnWorkflowExtensionPoint.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.xp; + +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.extension.ExtensionPoint; +import org.apache.hop.core.extension.IExtensionPoint; +import org.apache.hop.core.logging.ILogChannel; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.workflow.WorkflowPainter; + +@ExtensionPoint( + id = "DrawEnvironmentOnWorkflowExtensionPoint", + description = "Draws optional environment canvas text top-right under workflow graph content", + extensionPointId = "WorkflowPainterStart") +public class DrawEnvironmentOnWorkflowExtensionPoint implements IExtensionPoint { + + @Override + public void callExtensionPoint( + ILogChannel log, IVariables variables, WorkflowPainter workflowPainter) throws HopException { + DrawEnvironmentCanvasTextExtension.draw(workflowPainter, variables); + } +} diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopGuiDirectoryReplaceHomeVariable.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopGuiDirectoryReplaceHomeVariable.java index 282520a36e5..53ca0a95489 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopGuiDirectoryReplaceHomeVariable.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopGuiDirectoryReplaceHomeVariable.java @@ -17,25 +17,24 @@ package org.apache.hop.projects.xp; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.vfs2.FileObject; import org.apache.hop.core.extension.ExtensionPoint; import org.apache.hop.core.extension.IExtensionPoint; import org.apache.hop.core.logging.ILogChannel; import org.apache.hop.core.util.StringUtil; import org.apache.hop.core.variables.IVariables; -import org.apache.hop.core.vfs.HopVfs; import org.apache.hop.projects.config.ProjectsConfig; import org.apache.hop.projects.config.ProjectsConfigSingleton; import org.apache.hop.projects.project.ProjectConfig; -import org.apache.hop.projects.util.ProjectsUtil; +import org.apache.hop.projects.util.PathVariableReplacer; import org.apache.hop.ui.core.gui.HopNamespace; import org.apache.hop.ui.hopgui.delegates.HopGuiDirectorySelectedExtension; @ExtensionPoint( id = "HopGuiDirectoryReplaceHomeVariable", extensionPointId = "HopGuiDirectorySelected", - description = "Replace ${PROJECT_HOME} in selected directory as a best practice aid") + description = + "Replace path-like project/environment variables (e.g. ${PROJECT_HOME}, ${SOURCE_FILES})" + + " in selected directory as a best practice aid") public class HopGuiDirectoryReplaceHomeVariable implements IExtensionPoint { @@ -56,39 +55,17 @@ public void callExtensionPoint( if (projectConfig == null) { return; } - String homeFolder; - - if (variables != null) { - homeFolder = variables.resolve(projectConfig.getProjectHome()); - } else { - homeFolder = projectConfig.getProjectHome(); - } try { - if (StringUtils.isNotEmpty(homeFolder)) { - - FileObject file = HopVfs.getFileObject(ext.folderName); - String absoluteFile = file.getName().getPath(); - - FileObject home = HopVfs.getFileObject(homeFolder); - String absoluteHome = home.getName().getPath(); - // Make the URI always end with a / - if (!absoluteHome.endsWith("/")) { - absoluteHome += "/"; - } - - // Replace the project home variable in the filename - // - if (absoluteFile.startsWith(absoluteHome)) { - ext.folderName = - "${" - + ProjectsUtil.VARIABLE_PROJECT_HOME - + "}/" - + absoluteFile.substring(absoluteHome.length()); - } + IVariables vars = + HopGuiFileReplaceHomeVariable.ensureProjectHomeVariable( + variables, projectConfig, ext.variables); + String replaced = PathVariableReplacer.replacePathWithVariable(vars, ext.folderName); + if (replaced != null) { + ext.folderName = replaced; } } catch (Exception e) { - log.logError("Error setting default folder for project " + projectName, e); + log.logError("Error replacing path variables for project " + projectName, e); } } } diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopGuiFileReplaceHomeVariable.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopGuiFileReplaceHomeVariable.java index 8ccd78e4a89..2c1932c82dc 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopGuiFileReplaceHomeVariable.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/HopGuiFileReplaceHomeVariable.java @@ -17,17 +17,16 @@ package org.apache.hop.projects.xp; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.vfs2.FileObject; import org.apache.hop.core.extension.ExtensionPoint; import org.apache.hop.core.extension.IExtensionPoint; import org.apache.hop.core.logging.ILogChannel; import org.apache.hop.core.util.StringUtil; import org.apache.hop.core.variables.IVariables; -import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.core.variables.Variables; import org.apache.hop.projects.config.ProjectsConfig; import org.apache.hop.projects.config.ProjectsConfigSingleton; import org.apache.hop.projects.project.ProjectConfig; +import org.apache.hop.projects.util.PathVariableReplacer; import org.apache.hop.projects.util.ProjectsUtil; import org.apache.hop.ui.core.gui.HopNamespace; import org.apache.hop.ui.hopgui.delegates.HopGuiFileOpenedExtension; @@ -35,7 +34,9 @@ @ExtensionPoint( id = "HopGuiFileReplaceHomeVariable", extensionPointId = "HopGuiFileOpenedDialog", - description = "Replace ${PROJECT_HOME} in selected filenames as a best practice aid") + description = + "Replace path-like project/environment variables (e.g. ${PROJECT_HOME}, ${SOURCE_FILES})" + + " in selected filenames as a best practice aid") public class HopGuiFileReplaceHomeVariable implements IExtensionPoint { // TODO make this optional @@ -55,39 +56,36 @@ public void callExtensionPoint( if (projectConfig == null) { return; } - String homeFolder; - - if (variables != null) { - homeFolder = variables.resolve(projectConfig.getProjectHome()); - } else { - homeFolder = projectConfig.getProjectHome(); - } try { - if (StringUtils.isNotEmpty(homeFolder)) { - - FileObject file = HopVfs.getFileObject(ext.filename); - String absoluteFile = file.getName().getPath(); - - FileObject home = HopVfs.getFileObject(homeFolder); - String absoluteHome = home.getName().getPath(); - // Make the URI always end with a / - if (!absoluteHome.endsWith("/")) { - absoluteHome += "/"; - } - - // Replace the project home variable in the filename - // - if (absoluteFile.startsWith(absoluteHome)) { - ext.filename = - "${" - + ProjectsUtil.VARIABLE_PROJECT_HOME - + "}/" - + absoluteFile.substring(absoluteHome.length()); - } + IVariables vars = ensureProjectHomeVariable(variables, projectConfig, ext.variables); + String replaced = PathVariableReplacer.replacePathWithVariable(vars, ext.filename); + if (replaced != null) { + ext.filename = replaced; } } catch (Exception e) { - log.logError("Error setting default folder for project " + projectName, e); + log.logError("Error replacing path variables for project " + projectName, e); + } + } + + /** + * Ensure {@code PROJECT_HOME} is available even when the dialog passed a null variable space. + * Prefer the extension's own variables when present. + */ + static IVariables ensureProjectHomeVariable( + IVariables callVariables, ProjectConfig projectConfig, IVariables extensionVariables) { + IVariables vars = extensionVariables != null ? extensionVariables : callVariables; + if (vars == null) { + vars = new Variables(); + vars.initializeFrom(null); + } + // Expose PROJECT_HOME from the active project config when missing so replacement works + // even if the dialog variable space is incomplete. + if (StringUtil.isEmpty(vars.getVariable(ProjectsUtil.VARIABLE_PROJECT_HOME)) + && projectConfig.getProjectHome() != null) { + vars.setVariable( + ProjectsUtil.VARIABLE_PROJECT_HOME, vars.resolve(projectConfig.getProjectHome())); } + return vars; } } diff --git a/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/ProjectsVariablesControlSpaceSortOrderPrefix.java b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/ProjectsVariablesControlSpaceSortOrderPrefix.java index 26087335986..7ca0f82a69d 100644 --- a/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/ProjectsVariablesControlSpaceSortOrderPrefix.java +++ b/plugins/misc/projects/src/main/java/org/apache/hop/projects/xp/ProjectsVariablesControlSpaceSortOrderPrefix.java @@ -37,6 +37,8 @@ public void callExtensionPoint( ILogChannel log, IVariables variables, Map prefixMap) throws HopException { prefixMap.put(ProjectsUtil.VARIABLE_PROJECT_HOME, "310_"); + prefixMap.put(ProjectsUtil.VARIABLE_PARENT_PROJECT_HOME, "311_"); + prefixMap.put(ProjectsUtil.VARIABLE_PARENT_PROJECT_NAME, "312_"); prefixMap.put(Defaults.VARIABLE_HOP_PROJECT_NAME, "450_"); prefixMap.put(Defaults.VARIABLE_HOP_ENVIRONMENT_NAME, "450_"); prefixMap.put(ProjectsUtil.VARIABLE_HOP_DATASETS_FOLDER, "450_"); diff --git a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/environment/messages/messages_en_US.properties b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/environment/messages/messages_en_US.properties index 4bd84281148..a4ee2c7c00a 100644 --- a/plugins/misc/projects/src/main/resources/org/apache/hop/projects/environment/messages/messages_en_US.properties +++ b/plugins/misc/projects/src/main/resources/org/apache/hop/projects/environment/messages/messages_en_US.properties @@ -17,6 +17,7 @@ # LifecycleEnvironmentDialog.Button.Edit=Edit... +LifecycleEnvironmentDialog.Button.ImportVariables=Import variables LifecycleEnvironmentDialog.Button.New=New... LifecycleEnvironmentDialog.Button.Select=Select... LifecycleEnvironmentDialog.DetailTable.Label.Filename=Filename @@ -24,6 +25,8 @@ LifecycleEnvironmentDialog.Group.Label.ConfigurationFiles=Configuration files: LifecycleEnvironmentDialog.Label.EnvironmentName=Name LifecycleEnvironmentDialog.Label.EnvironmentPurpose=Purpose LifecycleEnvironmentDialog.Label.ReferencedProject=Project +LifecycleEnvironmentDialog.Label.CanvasText=Canvas text +LifecycleEnvironmentDialog.ToolTip.CanvasText=Large text drawn in the top-right of pipeline and workflow canvases when this environment is active LifecycleEnvironmentDialog.Purpose.Text.Acceptance=Acceptance LifecycleEnvironmentDialog.Purpose.Text.CB=Common Build LifecycleEnvironmentDialog.Purpose.Text.CI=Continuous Integration @@ -31,3 +34,17 @@ LifecycleEnvironmentDialog.Purpose.Text.Development=Development LifecycleEnvironmentDialog.Purpose.Text.Production=Production LifecycleEnvironmentDialog.Purpose.Text.Testing=Testing LifecycleEnvironmentDialog.Shell.Name=Environment Properties +LifecycleEnvironmentDialog.ImportVariables.ProjectRequired.Title=Project required +LifecycleEnvironmentDialog.ImportVariables.ProjectRequired.Message=Please select the project associated with this environment before importing variables. +LifecycleEnvironmentDialog.ImportVariables.NoneFound.Title=No variables to import +LifecycleEnvironmentDialog.ImportVariables.NoneFound.Message=No '${VARIABLE}' expressions were found in the project that are not already defined in the environment configuration files. +LifecycleEnvironmentDialog.ImportVariables.DialogMessage=Variables used in project ''{0}'' that are not yet in an environment configuration file. Set values as needed, then OK to save. +LifecycleEnvironmentDialog.ImportVariables.SelectConfig.Title=Select configuration file +LifecycleEnvironmentDialog.ImportVariables.SelectConfig.Message=Choose an existing environment configuration file, or create a new one. +LifecycleEnvironmentDialog.ImportVariables.CreateNewOption=(Create new configuration file\u2026) +LifecycleEnvironmentDialog.ImportVariables.FileFilter.Json=Config JSON files +LifecycleEnvironmentDialog.ImportVariables.FileFilter.All=All files +LifecycleEnvironmentDialog.ImportVariables.Saved.Title=Variables saved +LifecycleEnvironmentDialog.ImportVariables.Saved.Message={0} variable(s) were saved to configuration file:\n{1} +LifecycleEnvironmentDialog.ImportVariables.Error.Title=Error +LifecycleEnvironmentDialog.ImportVariables.Error.Message=Error importing variables from the project diff --git a/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EnvironmentVariablesImportHelperTest.java b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EnvironmentVariablesImportHelperTest.java new file mode 100644 index 00000000000..c5a9c2366c8 --- /dev/null +++ b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/EnvironmentVariablesImportHelperTest.java @@ -0,0 +1,193 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.environment; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; +import org.apache.hop.core.Const; +import org.apache.hop.core.config.DescribedVariablesConfigFile; +import org.apache.hop.core.search.ISearchResult; +import org.apache.hop.core.search.ISearchable; +import org.apache.hop.core.search.ISearchableCallback; +import org.apache.hop.core.search.SearchResult; +import org.apache.hop.core.variables.DescribedVariable; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.projects.util.ProjectsUtil; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class EnvironmentVariablesImportHelperTest { + + @TempDir Path tempDir; + + @Test + void extractVariableNamesUnixOnly() { + String text = + "host=${DB_HOST} port=%%DB_PORT%% url=#{resolver:x} path=${PROJECT_HOME}/data nested=${A}${B}"; + Set names = EnvironmentVariablesImportHelper.extractVariableNames(text); + assertEquals(Set.of("DB_HOST", "PROJECT_HOME", "A", "B"), names); + assertFalse(names.contains("DB_PORT")); + assertFalse(names.contains("resolver:x")); + } + + @Test + void extractVariableNamesEmpty() { + assertTrue(EnvironmentVariablesImportHelper.extractVariableNames(null).isEmpty()); + assertTrue(EnvironmentVariablesImportHelper.extractVariableNames("").isEmpty()); + assertTrue( + EnvironmentVariablesImportHelper.extractVariableNames("no variables here").isEmpty()); + } + + @Test + void variableExpressionRegexMatchesPropertyValues() { + Pattern pattern = Pattern.compile(EnvironmentVariablesImportHelper.VARIABLE_EXPRESSION_REGEX); + assertTrue(pattern.matcher("jdbc:postgresql://${DB_HOST}:5432/db").find()); + assertTrue(pattern.matcher("${ONLY}").find()); + assertFalse(pattern.matcher("%%WINDOWS%%").find()); + assertFalse(pattern.matcher("#{resolver:x}").find()); + assertFalse(pattern.matcher("plain text").find()); + } + + @Test + void isManagedVariable() { + assertTrue( + EnvironmentVariablesImportHelper.isManagedVariable(ProjectsUtil.VARIABLE_PROJECT_HOME)); + assertTrue(EnvironmentVariablesImportHelper.isManagedVariable(Const.HOP_METADATA_FOLDER)); + assertTrue( + EnvironmentVariablesImportHelper.isManagedVariable( + Const.INTERNAL_VARIABLE_PIPELINE_FILENAME_DIRECTORY)); + assertTrue( + EnvironmentVariablesImportHelper.isManagedVariable( + EnvironmentVariablesImportHelper.JAVA_IO_TMPDIR)); + assertTrue(EnvironmentVariablesImportHelper.isManagedVariable("")); + assertFalse(EnvironmentVariablesImportHelper.isManagedVariable("DB_HOSTNAME")); + } + + @Test + void formatFoundLocationIncludesTypeNameAndField() { + ISearchable searchable = + new ISearchable<>() { + @Override + public String getLocation() { + return "project"; + } + + @Override + public String getName() { + return "load-data"; + } + + @Override + public String getType() { + return "Pipeline"; + } + + @Override + public String getFilename() { + return "/p/load-data.hpl"; + } + + @Override + public String getSearchableObject() { + return "meta"; + } + + @Override + public ISearchableCallback getSearchCallback() { + return null; + } + }; + ISearchResult result = + new SearchResult( + searchable, "${DB_HOST}", "hostname : host name", "Table input", "${DB_HOST}"); + assertEquals( + "Pipeline 'load-data' → Table input → host name", + EnvironmentVariablesImportHelper.formatFoundLocation(result)); + } + + @Test + void formatLocationsDescriptionJoinsAndCaps() { + assertEquals("", EnvironmentVariablesImportHelper.formatLocationsDescription(null)); + assertEquals( + "a; b", + EnvironmentVariablesImportHelper.formatLocationsDescription( + new LinkedHashSet<>(List.of("a", "b")))); + + Set many = new LinkedHashSet<>(); + for (int i = 1; i <= 12; i++) { + many.add("loc" + i); + } + String description = EnvironmentVariablesImportHelper.formatLocationsDescription(many); + assertTrue(description.startsWith("loc1; ")); + assertTrue(description.endsWith("(+2 more)")); + } + + @Test + void defaultConfigFilename() { + assertEquals("dev-config.json", EnvironmentVariablesImportHelper.defaultConfigFilename("dev")); + assertEquals( + "my-env-config.json", EnvironmentVariablesImportHelper.defaultConfigFilename("My Env")); + assertEquals( + "environment-config.json", EnvironmentVariablesImportHelper.defaultConfigFilename("")); + } + + @Test + void loadConfiguredVariableNamesAndSaveMerge() throws Exception { + Path configPath = tempDir.resolve("env-config.json"); + DescribedVariablesConfigFile created = new DescribedVariablesConfigFile(configPath.toString()); + created.setDescribedVariable(new DescribedVariable("EXISTING", "one", "already there")); + created.saveToFile(); + + Variables variables = new Variables(); + Set names = + EnvironmentVariablesImportHelper.loadConfiguredVariableNames( + List.of(configPath.toString()), variables); + assertTrue(names.contains("EXISTING")); + assertEquals(1, names.size()); + + List toAdd = new ArrayList<>(); + toAdd.add(new DescribedVariable("NEW_VAR", "two", "imported")); + toAdd.add(new DescribedVariable("EXISTING", "updated", "overwritten")); + EnvironmentVariablesImportHelper.saveVariablesToConfigFile(configPath.toString(), toAdd); + + DescribedVariablesConfigFile reloaded = new DescribedVariablesConfigFile(configPath.toString()); + reloaded.readFromFile(); + assertEquals("updated", reloaded.findDescribedVariableValue("EXISTING")); + assertEquals("two", reloaded.findDescribedVariableValue("NEW_VAR")); + assertEquals(2, reloaded.getDescribedVariables().size()); + assertTrue(Files.size(configPath) > 0); + } + + @Test + void loadConfiguredVariableNamesSkipsMissingFiles() throws Exception { + Variables variables = new Variables(); + Set names = + EnvironmentVariablesImportHelper.loadConfiguredVariableNames( + List.of(tempDir.resolve("does-not-exist.json").toString()), variables); + assertTrue(names.isEmpty()); + } +} diff --git a/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/LifecycleEnvironmentNamingTest.java b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/LifecycleEnvironmentNamingTest.java new file mode 100644 index 00000000000..2a75c2940ed --- /dev/null +++ b/plugins/misc/projects/src/test/java/org/apache/hop/projects/environment/LifecycleEnvironmentNamingTest.java @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.environment; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +import java.util.Map; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class LifecycleEnvironmentNamingTest { + + private Map knownSuffixes; + + @BeforeEach + void setUp() { + knownSuffixes = + LifecycleEnvironmentNaming.knownPurposeSuffixes( + "Development", + "Testing", + "Acceptance", + "Production", + "Continuous Integration", + "Common Build"); + } + + @Test + void suggestProjectOnlyWhenPurposeEmpty() { + assertEquals( + "my-project", + LifecycleEnvironmentNaming.suggestEnvironmentName("my-project", "", knownSuffixes)); + assertEquals( + "my-project", + LifecycleEnvironmentNaming.suggestEnvironmentName("my-project", null, knownSuffixes)); + assertEquals( + "my-project", + LifecycleEnvironmentNaming.suggestEnvironmentName("my-project", " ", knownSuffixes)); + } + + @Test + void suggestEmptyWhenProjectBlank() { + assertEquals( + "", LifecycleEnvironmentNaming.suggestEnvironmentName(null, "Development", knownSuffixes)); + assertEquals( + "", LifecycleEnvironmentNaming.suggestEnvironmentName("", "Development", knownSuffixes)); + assertEquals( + "", LifecycleEnvironmentNaming.suggestEnvironmentName(" ", "Development", knownSuffixes)); + } + + @Test + void suggestKnownPurposesUseEnglishSuffixes() { + assertEquals( + "sales-development", + LifecycleEnvironmentNaming.suggestEnvironmentName("sales", "Development", knownSuffixes)); + assertEquals( + "sales-test", + LifecycleEnvironmentNaming.suggestEnvironmentName("sales", "Testing", knownSuffixes)); + assertEquals( + "sales-acceptance", + LifecycleEnvironmentNaming.suggestEnvironmentName("sales", "Acceptance", knownSuffixes)); + assertEquals( + "sales-production", + LifecycleEnvironmentNaming.suggestEnvironmentName("sales", "Production", knownSuffixes)); + assertEquals( + "sales-continuous-integration", + LifecycleEnvironmentNaming.suggestEnvironmentName( + "sales", "Continuous Integration", knownSuffixes)); + assertEquals( + "sales-common-build", + LifecycleEnvironmentNaming.suggestEnvironmentName("sales", "Common Build", knownSuffixes)); + } + + @Test + void knownLocalizedLabelsMapToEnglishSuffixes() { + Map italian = + LifecycleEnvironmentNaming.knownPurposeSuffixes( + "Sviluppo", + "Testing", + "Accettazione", + "Produzione", + "Continuous Integration", + "Common Build"); + + assertEquals( + "app-development", + LifecycleEnvironmentNaming.suggestEnvironmentName("app", "Sviluppo", italian)); + assertEquals( + "app-test", LifecycleEnvironmentNaming.suggestEnvironmentName("app", "Testing", italian)); + assertEquals( + "app-acceptance", + LifecycleEnvironmentNaming.suggestEnvironmentName("app", "Accettazione", italian)); + assertEquals( + "app-production", + LifecycleEnvironmentNaming.suggestEnvironmentName("app", "Produzione", italian)); + } + + @Test + void freeTextPurposeIsSanitized() { + assertEquals( + "my-project-staging", + LifecycleEnvironmentNaming.suggestEnvironmentName("my-project", "Staging", knownSuffixes)); + assertEquals( + "my-project-pre-prod", + LifecycleEnvironmentNaming.suggestEnvironmentName("my-project", "Pre Prod", knownSuffixes)); + assertEquals( + "my-project-qa.1", + LifecycleEnvironmentNaming.suggestEnvironmentName("my-project", "QA.1", knownSuffixes)); + } + + @Test + void purposeToSuffixKnownAndFreeText() { + assertEquals( + LifecycleEnvironmentNaming.SUFFIX_TEST, + LifecycleEnvironmentNaming.purposeToSuffix("Testing", knownSuffixes)); + assertEquals( + LifecycleEnvironmentNaming.SUFFIX_DEVELOPMENT, + LifecycleEnvironmentNaming.purposeToSuffix("Development", knownSuffixes)); + assertEquals( + "custom-env", LifecycleEnvironmentNaming.purposeToSuffix("Custom Env", knownSuffixes)); + assertEquals("", LifecycleEnvironmentNaming.purposeToSuffix(null, knownSuffixes)); + assertEquals("", LifecycleEnvironmentNaming.purposeToSuffix("", knownSuffixes)); + } + + @Test + void sanitizePurposeSuffix() { + assertEquals("hello-world", LifecycleEnvironmentNaming.sanitizePurposeSuffix(" Hello World ")); + assertEquals("a-b-c", LifecycleEnvironmentNaming.sanitizePurposeSuffix("A/B\\C")); + assertEquals("", LifecycleEnvironmentNaming.sanitizePurposeSuffix("@@@")); + assertEquals("", LifecycleEnvironmentNaming.sanitizePurposeSuffix(null)); + } + + @Test + void suggestWithSuffixDirectly() { + assertEquals( + "p-development", + LifecycleEnvironmentNaming.suggestEnvironmentName( + "p", LifecycleEnvironmentNaming.SUFFIX_DEVELOPMENT)); + assertEquals("p", LifecycleEnvironmentNaming.suggestEnvironmentName("p", "")); + } +} diff --git a/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectTest.java b/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectTest.java index 4cb65c6110c..a9e67cc5cbb 100644 --- a/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectTest.java +++ b/plugins/misc/projects/src/test/java/org/apache/hop/projects/project/ProjectTest.java @@ -17,15 +17,46 @@ package org.apache.hop.projects.project; +import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.stream.Stream; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.variables.Variables; +import org.apache.hop.projects.config.ProjectsConfig; +import org.apache.hop.projects.config.ProjectsConfigSingleton; +import org.apache.hop.projects.util.Defaults; +import org.apache.hop.projects.util.ProjectsUtil; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; public class ProjectTest { + private final java.util.List registeredProjectNames = new ArrayList<>(); + private Path tempRoot; + + @AfterEach + public void tearDown() throws Exception { + ProjectsConfig config = ProjectsConfigSingleton.getConfig(); + for (String name : registeredProjectNames) { + config.removeProjectConfig(name); + } + registeredProjectNames.clear(); + if (tempRoot != null && Files.exists(tempRoot)) { + try (Stream walk = Files.walk(tempRoot)) { + walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + tempRoot = null; + } + } + @Test public void testEnforcingExecutionInHomeSerialization() throws Exception { File tempFile = Files.createTempFile("project-config-test", ".json").toFile(); @@ -54,4 +85,120 @@ public void testEnforcingExecutionInHomeSerialization() throws Exception { tempFile.delete(); } } + + @Test + public void testParentProjectVariablesWhenNoParent() throws Exception { + tempRoot = Files.createTempDirectory("hop-project-no-parent"); + Path home = tempRoot.resolve("child"); + Files.createDirectories(home); + writeMinimalConfig(home, null); + + ProjectConfig projectConfig = + new ProjectConfig( + "orphan", home.toString(), ProjectsConfig.DEFAULT_PROJECT_CONFIG_FILENAME); + registerProject(projectConfig); + + Project project = projectConfig.loadProject(new Variables()); + IVariables variables = new Variables(); + project.modifyVariables(variables, projectConfig, new ArrayList<>(), null); + + assertEquals("orphan", variables.getVariable(Defaults.VARIABLE_HOP_PROJECT_NAME)); + assertEquals(home.toString(), variables.getVariable(ProjectsUtil.VARIABLE_PROJECT_HOME)); + assertEquals("", variables.getVariable(ProjectsUtil.VARIABLE_PARENT_PROJECT_NAME)); + assertEquals("", variables.getVariable(ProjectsUtil.VARIABLE_PARENT_PROJECT_HOME)); + } + + @Test + public void testParentProjectVariablesWithParent() throws Exception { + tempRoot = Files.createTempDirectory("hop-project-with-parent"); + Path parentHome = tempRoot.resolve("parent"); + Path childHome = tempRoot.resolve("child"); + Files.createDirectories(parentHome); + Files.createDirectories(childHome); + writeMinimalConfig(parentHome, null); + writeMinimalConfig(childHome, "parent-proj"); + + ProjectConfig parentConfig = + new ProjectConfig( + "parent-proj", parentHome.toString(), ProjectsConfig.DEFAULT_PROJECT_CONFIG_FILENAME); + ProjectConfig childConfig = + new ProjectConfig( + "child-proj", childHome.toString(), ProjectsConfig.DEFAULT_PROJECT_CONFIG_FILENAME); + registerProject(parentConfig); + registerProject(childConfig); + + Project child = childConfig.loadProject(new Variables()); + IVariables variables = new Variables(); + child.modifyVariables(variables, childConfig, new ArrayList<>(), null); + + assertEquals("child-proj", variables.getVariable(Defaults.VARIABLE_HOP_PROJECT_NAME)); + assertEquals(childHome.toString(), variables.getVariable(ProjectsUtil.VARIABLE_PROJECT_HOME)); + assertEquals("parent-proj", variables.getVariable(ProjectsUtil.VARIABLE_PARENT_PROJECT_NAME)); + assertEquals( + parentHome.toString(), variables.getVariable(ProjectsUtil.VARIABLE_PARENT_PROJECT_HOME)); + } + + @Test + public void testParentProjectVariablesImmediateParentOnly() throws Exception { + tempRoot = Files.createTempDirectory("hop-project-chain"); + Path grandHome = tempRoot.resolve("grand"); + Path parentHome = tempRoot.resolve("parent"); + Path childHome = tempRoot.resolve("child"); + Files.createDirectories(grandHome); + Files.createDirectories(parentHome); + Files.createDirectories(childHome); + writeMinimalConfig(grandHome, null); + writeMinimalConfig(parentHome, "grand-proj"); + writeMinimalConfig(childHome, "parent-proj"); + + registerProject( + new ProjectConfig( + "grand-proj", grandHome.toString(), ProjectsConfig.DEFAULT_PROJECT_CONFIG_FILENAME)); + registerProject( + new ProjectConfig( + "parent-proj", parentHome.toString(), ProjectsConfig.DEFAULT_PROJECT_CONFIG_FILENAME)); + ProjectConfig childConfig = + new ProjectConfig( + "child-proj", childHome.toString(), ProjectsConfig.DEFAULT_PROJECT_CONFIG_FILENAME); + registerProject(childConfig); + + Project child = childConfig.loadProject(new Variables()); + IVariables variables = new Variables(); + child.modifyVariables(variables, childConfig, new ArrayList<>(), null); + + // Immediate parent only — not the grandparent + assertEquals("parent-proj", variables.getVariable(ProjectsUtil.VARIABLE_PARENT_PROJECT_NAME)); + assertEquals( + parentHome.toString(), variables.getVariable(ProjectsUtil.VARIABLE_PARENT_PROJECT_HOME)); + assertEquals("child-proj", variables.getVariable(Defaults.VARIABLE_HOP_PROJECT_NAME)); + assertEquals(childHome.toString(), variables.getVariable(ProjectsUtil.VARIABLE_PROJECT_HOME)); + } + + private void registerProject(ProjectConfig projectConfig) { + ProjectsConfigSingleton.getConfig().addProjectConfig(projectConfig); + registeredProjectNames.add(projectConfig.getProjectName()); + } + + private static void writeMinimalConfig(Path projectHome, String parentProjectName) + throws Exception { + String parentJson = + parentProjectName == null ? "null" : "\"" + parentProjectName.replace("\"", "\\\"") + "\""; + String json = + "{\n" + + " \"metadataBaseFolder\" : \"${PROJECT_HOME}/metadata\",\n" + + " \"unitTestsBasePath\" : \"${PROJECT_HOME}\",\n" + + " \"dataSetsCsvFolder\" : \"${PROJECT_HOME}/datasets\",\n" + + " \"enforcingExecutionInHome\" : true,\n" + + " \"parentProjectName\" : " + + parentJson + + ",\n" + + " \"config\" : {\n" + + " \"variables\" : [ ]\n" + + " }\n" + + "}\n"; + Files.writeString( + projectHome.resolve(ProjectsConfig.DEFAULT_PROJECT_CONFIG_FILENAME), + json, + StandardCharsets.UTF_8); + } } diff --git a/plugins/misc/projects/src/test/java/org/apache/hop/projects/util/PathVariableReplacerTest.java b/plugins/misc/projects/src/test/java/org/apache/hop/projects/util/PathVariableReplacerTest.java new file mode 100644 index 00000000000..f917843086a --- /dev/null +++ b/plugins/misc/projects/src/test/java/org/apache/hop/projects/util/PathVariableReplacerTest.java @@ -0,0 +1,171 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.projects.util; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.nio.file.Files; +import java.nio.file.Path; +import org.apache.hop.core.Const; +import org.apache.hop.core.variables.Variables; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class PathVariableReplacerTest { + + @TempDir Path tempDir; + + private Path projectHome; + private Path sourceFiles; + private Variables variables; + + @BeforeEach + void setUp() throws Exception { + projectHome = tempDir.resolve("project"); + sourceFiles = tempDir.resolve("source"); + Files.createDirectories(projectHome.resolve("data")); + Files.createDirectories(sourceFiles.resolve("input")); + Files.writeString(projectHome.resolve("file.csv"), "a"); + Files.writeString(sourceFiles.resolve("input").resolve("data.csv"), "b"); + + variables = new Variables(); + // Avoid loading full system property set into candidates for most tests: + // set only the variables we care about without initializeFrom(). + variables.setVariable(ProjectsUtil.VARIABLE_PROJECT_HOME, projectHome.toString()); + variables.setVariable("SOURCE_FILES", sourceFiles.toString()); + } + + @Test + void replacesProjectHomeChildPath() { + String selected = projectHome.resolve("file.csv").toString(); + assertEquals( + "${PROJECT_HOME}/file.csv", + PathVariableReplacer.replacePathWithVariable(variables, selected)); + } + + @Test + void replacesProjectHomeNestedPath() { + String selected = projectHome.resolve("data").resolve("x.txt").toString(); + // create nested file path (dir already exists) + assertEquals( + "${PROJECT_HOME}/data/x.txt", + PathVariableReplacer.replacePathWithVariable(variables, selected)); + } + + @Test + void replacesEnvironmentVariableOutsideProject() { + String selected = sourceFiles.resolve("input").resolve("data.csv").toString(); + assertEquals( + "${SOURCE_FILES}/input/data.csv", + PathVariableReplacer.replacePathWithVariable(variables, selected)); + } + + @Test + void longestPrefixWinsOverProjectHome() throws Exception { + Path nested = projectHome.resolve("data"); + variables.setVariable("DATA_FOLDER", nested.toString()); + String selected = nested.resolve("report.csv").toString(); + assertEquals( + "${DATA_FOLDER}/report.csv", + PathVariableReplacer.replacePathWithVariable(variables, selected)); + } + + @Test + void exactDirectoryMatchUsesVariableOnly() { + assertEquals( + "${PROJECT_HOME}", + PathVariableReplacer.replacePathWithVariable(variables, projectHome.toString())); + assertEquals( + "${SOURCE_FILES}", + PathVariableReplacer.replacePathWithVariable(variables, sourceFiles.toString())); + } + + @Test + void noMatchLeavesPathUnchanged() { + Path outside = tempDir.resolve("other").resolve("file.txt"); + String selected = outside.toString(); + assertEquals(selected, PathVariableReplacer.replacePathWithVariable(variables, selected)); + } + + @Test + void nullOrEmptyInputsUnchanged() { + assertEquals("/tmp/x", PathVariableReplacer.replacePathWithVariable(null, "/tmp/x")); + assertEquals(null, PathVariableReplacer.replacePathWithVariable(variables, null)); + assertEquals("", PathVariableReplacer.replacePathWithVariable(variables, "")); + } + + @Test + void ignoresSystemPropertiesAsCandidates() { + Variables withSystem = new Variables(); + withSystem.setVariable(ProjectsUtil.VARIABLE_PROJECT_HOME, projectHome.toString()); + withSystem.setVariable("java.io.tmpdir", tempDir.toString()); + withSystem.setVariable("user.home", tempDir.toString()); + + // Path under tempDir (where java.io.tmpdir points) but not under project home + Path other = tempDir.resolve("not-in-project"); + String selected = other.toString(); + assertEquals(selected, PathVariableReplacer.replacePathWithVariable(withSystem, selected)); + } + + @Test + void ignoresNonPathManagedNames() { + assertFalse(PathVariableReplacer.isCandidateVariableName(Defaults.VARIABLE_HOP_PROJECT_NAME)); + assertFalse( + PathVariableReplacer.isCandidateVariableName(Defaults.VARIABLE_HOP_ENVIRONMENT_NAME)); + assertFalse( + PathVariableReplacer.isCandidateVariableName(ProjectsUtil.VARIABLE_PARENT_PROJECT_NAME)); + assertFalse( + PathVariableReplacer.isCandidateVariableName( + Const.INTERNAL_VARIABLE_PIPELINE_FILENAME_DIRECTORY)); + assertTrue(PathVariableReplacer.isCandidateVariableName(ProjectsUtil.VARIABLE_PROJECT_HOME)); + assertTrue(PathVariableReplacer.isCandidateVariableName("SOURCE_FILES")); + } + + @Test + void ignoresNonPathValues() { + assertFalse(PathVariableReplacer.isCandidatePathValue("")); + assertFalse(PathVariableReplacer.isCandidatePathValue("localhost")); + assertFalse(PathVariableReplacer.isCandidatePathValue("${PROJECT_HOME}/data")); + assertFalse(PathVariableReplacer.isCandidatePathValue("/meta1,/meta2")); + assertTrue(PathVariableReplacer.isCandidatePathValue("/tmp/source")); + assertTrue(PathVariableReplacer.isCandidatePathValue("file:///tmp/source")); + assertTrue(PathVariableReplacer.isCandidatePathValue("C:\\data\\files")); + } + + @Test + void resolvesNestedVariableValuesBeforeMatching() throws Exception { + Path dataDir = projectHome.resolve("data"); + variables.setVariable("DATA_FOLDER", "${PROJECT_HOME}/data"); + String selected = dataDir.resolve("report.csv").toString(); + assertEquals( + "${DATA_FOLDER}/report.csv", + PathVariableReplacer.replacePathWithVariable(variables, selected)); + } + + @Test + void trailingSlashOnVariableValueStillMatches() { + variables.setVariable("SOURCE_FILES", sourceFiles.toString() + "/"); + String selected = sourceFiles.resolve("input").resolve("data.csv").toString(); + assertEquals( + "${SOURCE_FILES}/input/data.csv", + PathVariableReplacer.replacePathWithVariable(variables, selected)); + } +} diff --git a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroinput/AvroFileInput.java b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroinput/AvroFileInput.java index 4bf828f5e0d..8c0b2b6888a 100644 --- a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroinput/AvroFileInput.java +++ b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroinput/AvroFileInput.java @@ -82,7 +82,7 @@ public boolean processRow() throws HopException { + "' doesn't exist in the input of this transform"); } - data.rowsLimit = Const.toInt(resolve(meta.getRowsLimit()), -1); + data.rowsLimit = Const.toIntExpanded(resolve(meta.getRowsLimit()), -1); data.outputRowMeta = getInputRowMeta().clone(); meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); diff --git a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroinput/AvroFileInputDialog.java b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroinput/AvroFileInputDialog.java index e7ba1665547..7fafd88f6b4 100644 --- a/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroinput/AvroFileInputDialog.java +++ b/plugins/tech/avro/src/main/java/org/apache/hop/avro/transforms/avroinput/AvroFileInputDialog.java @@ -107,6 +107,7 @@ public String open() { fdlRowsLimit.top = new FormAttachment(lastControl, margin); wlRowsLimit.setLayoutData(fdlRowsLimit); wRowsLimit = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wRowsLimit.enableExpandedInteger(); wRowsLimit.setText(transformName); PropsUi.setLook(wRowsLimit); FormData fdRowsLimit = new FormData(); diff --git a/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/listen/AzureListener.java b/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/listen/AzureListener.java index 2628e5c4530..fa0992eea4c 100644 --- a/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/listen/AzureListener.java +++ b/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/listen/AzureListener.java @@ -58,8 +58,8 @@ public AzureListener( @Override public boolean init() { - data.batchSize = Const.toInt(resolve(meta.getBatchSize()), 100); - data.prefetchSize = Const.toInt(resolve(meta.getPrefetchSize()), -1); + data.batchSize = Const.toIntExpanded(resolve(meta.getBatchSize()), 100); + data.prefetchSize = Const.toIntExpanded(resolve(meta.getPrefetchSize()), -1); data.list = new LinkedList<>(); return super.init(); @@ -209,11 +209,11 @@ public void rowWrittenEvent(IRowMeta rowMeta, Object[] row) options.setExceptionNotification(new AzureListenerErrorNotificationHandler(AzureListener.this)); if (!StringUtils.isNotEmpty(meta.getBatchSize())) { - options.setMaxBatchSize(Const.toInt(resolve(meta.getBatchSize()), 100)); + options.setMaxBatchSize(Const.toIntExpanded(resolve(meta.getBatchSize()), 100)); } if (!StringUtils.isNotEmpty(meta.getPrefetchSize())) { - options.setPrefetchCount(Const.toInt(resolve(meta.getPrefetchSize()), 100)); + options.setPrefetchCount(Const.toIntExpanded(resolve(meta.getPrefetchSize()), 100)); } data.executorService = Executors.newSingleThreadScheduledExecutor(); diff --git a/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/listen/AzureListenerDialog.java b/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/listen/AzureListenerDialog.java index f03702bcb98..4e7e8a322a4 100644 --- a/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/listen/AzureListenerDialog.java +++ b/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/listen/AzureListenerDialog.java @@ -249,6 +249,7 @@ public String open() { fdlBatchSize.top = new FormAttachment(lastControl, margin); wlBatchSize.setLayoutData(fdlBatchSize); wBatchSize = new TextVar(variables, wContent, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wBatchSize.enableExpandedInteger(); PropsUi.setLook(wBatchSize); wBatchSize.addModifyListener(lsMod); FormData fdBatchSize = new FormData(); @@ -267,6 +268,7 @@ public String open() { fdlPrefetchSize.top = new FormAttachment(lastControl, margin); wlPrefetchSize.setLayoutData(fdlPrefetchSize); wPrefetchSize = new TextVar(variables, wContent, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wPrefetchSize.enableExpandedInteger(); PropsUi.setLook(wPrefetchSize); wPrefetchSize.addModifyListener(lsMod); FormData fdPrefetchSize = new FormData(); diff --git a/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/write/AzureWrite.java b/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/write/AzureWrite.java index 1797b999aa5..63cc492f78b 100644 --- a/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/write/AzureWrite.java +++ b/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/write/AzureWrite.java @@ -48,7 +48,7 @@ public AzureWrite( @Override public boolean init() { - data.batchSize = Const.toLong(resolve(meta.getBatchSize()), 1); + data.batchSize = Const.toLongExpanded(resolve(meta.getBatchSize()), 1); data.list = new LinkedList<>(); return super.init(); diff --git a/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/write/AzureWriterDialog.java b/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/write/AzureWriterDialog.java index 33ea72d7602..8dacf4fe61f 100644 --- a/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/write/AzureWriterDialog.java +++ b/plugins/tech/azure/src/main/java/org/apache/hop/pipeline/transforms/eventhubs/write/AzureWriterDialog.java @@ -175,6 +175,7 @@ public String open() { fdlBatchSize.top = new FormAttachment(lastControl, margin); wlBatchSize.setLayoutData(fdlBatchSize); wBatchSize = new TextVar(variables, wContent, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wBatchSize.enableExpandedInteger(); PropsUi.setLook(wBatchSize); wBatchSize.addModifyListener(lsMod); FormData fdBatchSize = new FormData(); diff --git a/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalyticsDialog.java b/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalyticsDialog.java index c7939c008f8..6a4b252d455 100644 --- a/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalyticsDialog.java +++ b/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalyticsDialog.java @@ -609,7 +609,7 @@ private void getInfo(GoogleAnalyticsMeta meta) { googleAnalyticsFields.add(field); } meta.setGoogleAnalyticsFields(googleAnalyticsFields); - meta.setRowLimit(Const.toInt(wLimit.getText(), 0)); + meta.setRowLimit(Const.toIntExpanded(wLimit.getText(), 0)); } // Preview the data diff --git a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypher/Cypher.java b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypher/Cypher.java index ca07694dc29..e1d6cb61643 100644 --- a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypher/Cypher.java +++ b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypher/Cypher.java @@ -97,7 +97,7 @@ public boolean init() { return false; } - data.batchSize = Const.toLong(resolve(meta.getBatchSize()), 1); + data.batchSize = Const.toLongExpanded(resolve(meta.getBatchSize()), 1); // Try at least once and then do retries as needed // diff --git a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypher/CypherDialog.java b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypher/CypherDialog.java index 89c57ed997f..ee321918953 100644 --- a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypher/CypherDialog.java +++ b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypher/CypherDialog.java @@ -186,6 +186,7 @@ private void addOptionsTab() { fdlBatchSize.top = new FormAttachment(lastControl, margin); wlBatchSize.setLayoutData(fdlBatchSize); wBatchSize = new TextVar(variables, wOptionsComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wBatchSize.enableExpandedInteger(); PropsUi.setLook(wBatchSize); FormData fdBatchSize = new FormData(); fdBatchSize.left = new FormAttachment(middle, 0); diff --git a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypherbuilder/CypherBuilder.java b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypherbuilder/CypherBuilder.java index 7d90265eaae..a4ae0ce1190 100644 --- a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypherbuilder/CypherBuilder.java +++ b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypherbuilder/CypherBuilder.java @@ -182,7 +182,7 @@ private void prepareOnFirstRow(Object[] row) throws HopException { data.neoTypes.add(neoType); } - data.batchSize = Const.toInt(resolve(meta.getBatchSize()), 1); + data.batchSize = Const.toIntExpanded(resolve(meta.getBatchSize()), 1); data.cypher = meta.getCypher(this); diff --git a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypherbuilder/CypherBuilderDialog.java b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypherbuilder/CypherBuilderDialog.java index 3e9097f6ab5..7f431f4b74c 100644 --- a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypherbuilder/CypherBuilderDialog.java +++ b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/cypherbuilder/CypherBuilderDialog.java @@ -205,6 +205,7 @@ private void addOptionsTab() { fdlBatchSize.top = new FormAttachment(lastControl, margin); wlBatchSize.setLayoutData(fdlBatchSize); wBatchSize = new TextVar(variables, wOptionsComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wBatchSize.enableExpandedInteger(); PropsUi.setLook(wBatchSize); FormData fdBatchSize = new FormData(); fdBatchSize.left = new FormAttachment(middle, 0); diff --git a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/graph/GraphOutput.java b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/graph/GraphOutput.java index 70a0489ab82..36546916c65 100644 --- a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/graph/GraphOutput.java +++ b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/graph/GraphOutput.java @@ -109,7 +109,7 @@ public boolean init() { return false; } - data.batchSize = Const.toLong(resolve(meta.getBatchSize()), 1); + data.batchSize = Const.toLongExpanded(resolve(meta.getBatchSize()), 1); } if (StringUtils.isEmpty(meta.getModel())) { diff --git a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/graph/GraphOutputDialog.java b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/graph/GraphOutputDialog.java index fb85a2bc819..f208eca0a91 100644 --- a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/graph/GraphOutputDialog.java +++ b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/graph/GraphOutputDialog.java @@ -172,6 +172,7 @@ public String open() { fdlBatchSize.top = new FormAttachment(lastControl, margin); wlBatchSize.setLayoutData(fdlBatchSize); wBatchSize = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wBatchSize.enableExpandedInteger(); PropsUi.setLook(wBatchSize); FormData fdBatchSize = new FormData(); fdBatchSize.left = new FormAttachment(middle, 0); diff --git a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/output/Neo4JOutput.java b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/output/Neo4JOutput.java index b0cc49bbefb..de3f76276c6 100644 --- a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/output/Neo4JOutput.java +++ b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/output/Neo4JOutput.java @@ -861,7 +861,7 @@ public boolean init() { return false; } - data.batchSize = Const.toLong(resolve(meta.getBatchSize()), 1); + data.batchSize = Const.toLongExpanded(resolve(meta.getBatchSize()), 1); } return super.init(); diff --git a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/output/Neo4JOutputDialog.java b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/output/Neo4JOutputDialog.java index ff2087e8381..b6327ac6aa6 100755 --- a/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/output/Neo4JOutputDialog.java +++ b/plugins/tech/neo4j/src/main/java/org/apache/hop/neo4j/transforms/output/Neo4JOutputDialog.java @@ -178,6 +178,7 @@ public String open() { fdlBatchSize.top = new FormAttachment(lastControl, margin); wlBatchSize.setLayoutData(fdlBatchSize); wBatchSize = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wBatchSize.enableExpandedInteger(); PropsUi.setLook(wBatchSize); wBatchSize.addModifyListener(lsMod); FormData fdBatchSize = new FormData(); diff --git a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java index 3e07a38f6de..704b28047a3 100644 --- a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java +++ b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java @@ -65,14 +65,14 @@ public boolean init() { // Pre-calculate some values... // data.pageSize = - Const.toInt(resolve(meta.getDataPageSize()), ParquetProperties.DEFAULT_PAGE_SIZE); + Const.toIntExpanded(resolve(meta.getDataPageSize()), ParquetProperties.DEFAULT_PAGE_SIZE); data.dictionaryPageSize = - Const.toInt( + Const.toIntExpanded( resolve(meta.getDictionaryPageSize()), ParquetProperties.DEFAULT_DICTIONARY_PAGE_SIZE); data.rowGroupSize = - Const.toInt( + Const.toIntExpanded( resolve(meta.getRowGroupSize()), ParquetProperties.DEFAULT_PAGE_ROW_COUNT_LIMIT); - data.maxSplitSizeRows = Const.toLong(resolve(meta.getFileSplitSize()), -1); + data.maxSplitSizeRows = Const.toLongExpanded(resolve(meta.getFileSplitSize()), -1); return super.init(); } diff --git a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutputDialog.java b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutputDialog.java index 2fac6d25799..b1cd5baf68a 100644 --- a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutputDialog.java +++ b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutputDialog.java @@ -277,6 +277,7 @@ public String open() { fdlFilenameSplitSize.top = new FormAttachment(lastControl, margin); wlFilenameSplitSize.setLayoutData(fdlFilenameSplitSize); wFilenameSplitSize = new TextVar(variables, wFileGroup, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wFilenameSplitSize.enableExpandedInteger(); PropsUi.setLook(wFilenameSplitSize); FormData fdFilenameSplitSize = new FormData(); fdFilenameSplitSize.left = new FormAttachment(middle, 0); @@ -356,6 +357,7 @@ public String open() { fdlRowGroupSize.top = new FormAttachment(lastControl, margin); wlRowGroupSize.setLayoutData(fdlRowGroupSize); wRowGroupSize = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wRowGroupSize.enableExpandedInteger(); PropsUi.setLook(wRowGroupSize); FormData fdRowGroupSize = new FormData(); fdRowGroupSize.left = new FormAttachment(middle, 0); @@ -373,6 +375,7 @@ public String open() { fdlDataPageSize.top = new FormAttachment(lastControl, margin); wlDataPageSize.setLayoutData(fdlDataPageSize); wDataPageSize = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wDataPageSize.enableExpandedInteger(); PropsUi.setLook(wDataPageSize); FormData fdDataPageSize = new FormData(); fdDataPageSize.left = new FormAttachment(middle, 0); @@ -391,6 +394,7 @@ public String open() { fdlDictionaryPageSize.top = new FormAttachment(lastControl, margin); wlDictionaryPageSize.setLayoutData(fdlDictionaryPageSize); wDictionaryPageSize = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wDictionaryPageSize.enableExpandedInteger(); PropsUi.setLook(wDictionaryPageSize); FormData fdDictionaryPageSize = new FormData(); fdDictionaryPageSize.left = new FormAttachment(middle, 0); diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforceinput/SalesforceInput.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforceinput/SalesforceInput.java index 6c41242414e..aaf5bf10817 100644 --- a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforceinput/SalesforceInput.java +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforceinput/SalesforceInput.java @@ -392,7 +392,7 @@ public boolean init() { } } - data.limit = Const.toLong(resolve(meta.getRowLimit()), 0); + data.limit = Const.toLongExpanded(resolve(meta.getRowLimit()), 0); // Do we have to query for all records included deleted records data.connection.setQueryAll(meta.isQueryAll()); diff --git a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforceinput/SalesforceInputDialog.java b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforceinput/SalesforceInputDialog.java index 11ff886bb28..2be97edc74e 100644 --- a/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforceinput/SalesforceInputDialog.java +++ b/plugins/tech/salesforce/src/main/java/org/apache/hop/pipeline/transforms/salesforceinput/SalesforceInputDialog.java @@ -1165,6 +1165,7 @@ public void widgetSelected(SelectionEvent e) { fdlLimit.right = new FormAttachment(middle, -margin); wlLimit.setLayoutData(fdlLimit); wLimit = new TextVar(variables, wContentComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wLimit.enableExpandedInteger(); PropsUi.setLook(wLimit); wLimit.addModifyListener(lsMod); FormData fdLimit = new FormData(); diff --git a/plugins/transforms/clonerow/src/main/java/org/apache/hop/pipeline/transforms/clonerow/CloneRow.java b/plugins/transforms/clonerow/src/main/java/org/apache/hop/pipeline/transforms/clonerow/CloneRow.java index 7a622ff1b79..668b369a240 100644 --- a/plugins/transforms/clonerow/src/main/java/org/apache/hop/pipeline/transforms/clonerow/CloneRow.java +++ b/plugins/transforms/clonerow/src/main/java/org/apache/hop/pipeline/transforms/clonerow/CloneRow.java @@ -101,7 +101,7 @@ public boolean processRow() throws HopException { } } else { String nrclonesString = resolve(meta.getNrClones()); - data.nrclones = Const.toInt(nrclonesString, 0); + data.nrclones = Const.toIntExpanded(nrclonesString, 0); if (isDebug()) { logDebug(BaseMessages.getString(PKG, "CloneRow.Log.NrClones", "" + data.nrclones)); } diff --git a/plugins/transforms/clonerow/src/main/java/org/apache/hop/pipeline/transforms/clonerow/CloneRowDialog.java b/plugins/transforms/clonerow/src/main/java/org/apache/hop/pipeline/transforms/clonerow/CloneRowDialog.java index 912903a366f..97bc24fa97e 100644 --- a/plugins/transforms/clonerow/src/main/java/org/apache/hop/pipeline/transforms/clonerow/CloneRowDialog.java +++ b/plugins/transforms/clonerow/src/main/java/org/apache/hop/pipeline/transforms/clonerow/CloneRowDialog.java @@ -90,6 +90,7 @@ public String open() { wlnrClone.setLayoutData(fdlnrClone); wnrClone = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wnrClone.enableExpandedInteger(); PropsUi.setLook(wnrClone); wnrClone.setToolTipText(BaseMessages.getString(PKG, "CloneRowDialog.nrClone.Tooltip")); wnrClone.addModifyListener(lsMod); diff --git a/plugins/transforms/cubeinput/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInput.java b/plugins/transforms/cubeinput/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInput.java index 55632567abf..06998a138bf 100644 --- a/plugins/transforms/cubeinput/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInput.java +++ b/plugins/transforms/cubeinput/src/main/java/org/apache/hop/pipeline/transforms/cubeinput/CubeInput.java @@ -56,7 +56,7 @@ public boolean processRow() throws HopException { if (first) { first = false; - realRowLimit = Const.toInt(resolve(meta.getRowLimit()), 0); + realRowLimit = Const.toIntExpanded(resolve(meta.getRowLimit()), 0); } try { diff --git a/plugins/transforms/databasejoin/src/main/java/org/apache/hop/pipeline/transforms/databasejoin/DatabaseJoinDialog.java b/plugins/transforms/databasejoin/src/main/java/org/apache/hop/pipeline/transforms/databasejoin/DatabaseJoinDialog.java index a4304b77332..d11b0bd5a5c 100644 --- a/plugins/transforms/databasejoin/src/main/java/org/apache/hop/pipeline/transforms/databasejoin/DatabaseJoinDialog.java +++ b/plugins/transforms/databasejoin/src/main/java/org/apache/hop/pipeline/transforms/databasejoin/DatabaseJoinDialog.java @@ -454,7 +454,7 @@ private void ok() { input.setConnection(wConnection.getText()); input.setCached(wCache.getSelection()); input.setCacheSize(Const.toInt(wCacheSize.getText(), 0)); - input.setRowLimit(Const.toInt(wLimit.getText(), 0)); + input.setRowLimit(Const.toIntExpanded(wLimit.getText(), 0)); input.setSql(wSql.getText()); input.setOuterJoin(wOuter.getSelection()); input.setReplaceVariables(wUseVars.getSelection()); diff --git a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java index cea01f0aa5c..1b664b0a107 100644 --- a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java +++ b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteDialog.java @@ -244,6 +244,7 @@ private void addGeneralTab( fdlCommit.right = new FormAttachment(middle, -margin); wlCommit.setLayoutData(fdlCommit); wCommit = new TextVar(variables, composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wCommit.enableExpandedInteger(); PropsUi.setLook(wCommit); wCommit.addModifyListener(lsMod); FormData fdCommit = new FormData(); diff --git a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteMeta.java b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteMeta.java index abeefd633a3..e78a022c178 100644 --- a/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteMeta.java +++ b/plugins/transforms/delete/src/main/java/org/apache/hop/pipeline/transforms/delete/DeleteMeta.java @@ -105,7 +105,9 @@ public String getCommitSizeVar() { public int getCommitSize(IVariables vs) { // this happens when the transform is created via API and no setDefaults was called commitSize = (commitSize == null) ? "0" : commitSize; - return Integer.parseInt(vs.resolve(commitSize)); + String resolved = vs.resolve(commitSize); + String expanded = Const.expandIntegerString(resolved); + return Integer.parseInt(expanded != null ? expanded : resolved); } public DeleteMeta(DeleteMeta obj) { diff --git a/plugins/transforms/dynamicsqlrow/src/main/java/org/apache/hop/pipeline/transforms/dynamicsqlrow/DynamicSqlRowDialog.java b/plugins/transforms/dynamicsqlrow/src/main/java/org/apache/hop/pipeline/transforms/dynamicsqlrow/DynamicSqlRowDialog.java index a0c76d2d8e1..f7c1c67e8aa 100644 --- a/plugins/transforms/dynamicsqlrow/src/main/java/org/apache/hop/pipeline/transforms/dynamicsqlrow/DynamicSqlRowDialog.java +++ b/plugins/transforms/dynamicsqlrow/src/main/java/org/apache/hop/pipeline/transforms/dynamicsqlrow/DynamicSqlRowDialog.java @@ -416,7 +416,7 @@ private void ok() { } input.setConnection(wConnection.getText()); - input.setRowLimit(Const.toInt(wLimit.getText(), 0)); + input.setRowLimit(Const.toIntExpanded(wLimit.getText(), 0)); input.setSql(wSqlComposite.getText()); input.setSqlFieldName(wSqlFieldName.getText()); input.setOuterJoin(wOuter.getSelection()); diff --git a/plugins/transforms/excel/src/main/java/org/apache/hop/pipeline/transforms/excelinput/ExcelInputDialog.java b/plugins/transforms/excel/src/main/java/org/apache/hop/pipeline/transforms/excelinput/ExcelInputDialog.java index a3f9a37a76e..9de9d58cc37 100644 --- a/plugins/transforms/excel/src/main/java/org/apache/hop/pipeline/transforms/excelinput/ExcelInputDialog.java +++ b/plugins/transforms/excel/src/main/java/org/apache/hop/pipeline/transforms/excelinput/ExcelInputDialog.java @@ -1210,7 +1210,7 @@ private void getInfo(ExcelInputMeta meta) { transformName = wTransformName.getText(); // return value // copy info to Meta class (input) - meta.setRowLimit(Const.toLong(wLimit.getText(), 0)); + meta.setRowLimit(Const.toLongExpanded(wLimit.getText(), 0)); meta.setEncoding(wEncoding.getText()); meta.setSchemaDefinition(wSchemaDefinition.getText()); meta.setIgnoreFields(wIgnoreFields.getSelection()); diff --git a/plugins/transforms/filemetadata/src/main/java/org/apache/hop/pipeline/transforms/filemetadata/FileMetadata.java b/plugins/transforms/filemetadata/src/main/java/org/apache/hop/pipeline/transforms/filemetadata/FileMetadata.java index 4a91610b2c1..6d32fd42643 100644 --- a/plugins/transforms/filemetadata/src/main/java/org/apache/hop/pipeline/transforms/filemetadata/FileMetadata.java +++ b/plugins/transforms/filemetadata/src/main/java/org/apache/hop/pipeline/transforms/filemetadata/FileMetadata.java @@ -30,6 +30,7 @@ import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import org.apache.hop.core.Const; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.exception.HopFileException; import org.apache.hop.core.exception.HopRuntimeException; @@ -193,7 +194,8 @@ private void buildOutputRows() throws HopException { if (strLimitRows.trim().isEmpty()) { limitRows = 0; } else { - limitRows = Long.parseLong(strLimitRows); + String expandedLimit = Const.expandIntegerString(strLimitRows); + limitRows = Long.parseLong(expandedLimit != null ? expandedLimit : strLimitRows); } defaultCharset = Charset.forName(resolve(meta.getDefaultCharset())); diff --git a/plugins/transforms/filemetadata/src/main/java/org/apache/hop/pipeline/transforms/filemetadata/FileMetadataDialog.java b/plugins/transforms/filemetadata/src/main/java/org/apache/hop/pipeline/transforms/filemetadata/FileMetadataDialog.java index 22f70d418a0..8a91936a048 100644 --- a/plugins/transforms/filemetadata/src/main/java/org/apache/hop/pipeline/transforms/filemetadata/FileMetadataDialog.java +++ b/plugins/transforms/filemetadata/src/main/java/org/apache/hop/pipeline/transforms/filemetadata/FileMetadataDialog.java @@ -250,6 +250,7 @@ public void focusGained(FocusEvent e) { fdlLimit.top = new FormAttachment(0, margin); wlLimit.setLayoutData(fdlLimit); wLimit = new TextVar(variables, gDelimitedLayout, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wLimit.enableExpandedInteger(); wLimit.setToolTipText( BaseMessages.getString(PKG, "FileMetadata.methods.DELIMITED_FIELDS.limit.tooltip")); PropsUi.setLook(wLimit); diff --git a/plugins/transforms/getfilenames/src/main/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesDialog.java b/plugins/transforms/getfilenames/src/main/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesDialog.java index f88e1d32ed4..cea24ae3115 100644 --- a/plugins/transforms/getfilenames/src/main/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesDialog.java +++ b/plugins/transforms/getfilenames/src/main/java/org/apache/hop/pipeline/transforms/getfilenames/GetFileNamesDialog.java @@ -1019,7 +1019,7 @@ private void getInfo(GetFileNamesMeta in) { in.setDynamicExcludeWildcardField(wExcludeWildcardField.getText()); in.setFileField(wFileField.getSelection()); in.setRowNumberField(wInclRownum.getSelection() ? wInclRownumField.getText() : ""); - in.setRowLimit(Const.toLong(wLimit.getText(), 0L)); + in.setRowLimit(Const.toLongExpanded(wLimit.getText(), 0L)); in.setDynamicIncludeSubFolders(wIncludeSubFolder.getSelection()); in.setDoNotFailIfNoFile(wDoNotFailIfNoFile.getSelection()); in.setRaiseAnExceptionIfNoFile(wRaiseAnExceptionIfNoFile.getSelection()); diff --git a/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersDialog.java b/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersDialog.java index 3343eef2351..eecca4abdb9 100644 --- a/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersDialog.java +++ b/plugins/transforms/getsubfolders/src/main/java/org/apache/hop/pipeline/transforms/getsubfolders/GetSubFoldersDialog.java @@ -584,7 +584,7 @@ private void getInfo(GetSubFoldersMeta in) { in.setDynamicFolderNameField(wFolderNameField.getText()); in.setFolderNameDynamic(wFolderField.getSelection()); in.setRowNumberField(wInclRowNumberField.getText()); - in.setRowLimit(Const.toLong(wLimit.getText(), 0L)); + in.setRowLimit(Const.toLongExpanded(wLimit.getText(), 0L)); } // Preview the data diff --git a/plugins/transforms/insertupdate/src/main/java/org/apache/hop/pipeline/transforms/insertupdate/InsertUpdateDialog.java b/plugins/transforms/insertupdate/src/main/java/org/apache/hop/pipeline/transforms/insertupdate/InsertUpdateDialog.java index 371fb7b8081..321a75df956 100644 --- a/plugins/transforms/insertupdate/src/main/java/org/apache/hop/pipeline/transforms/insertupdate/InsertUpdateDialog.java +++ b/plugins/transforms/insertupdate/src/main/java/org/apache/hop/pipeline/transforms/insertupdate/InsertUpdateDialog.java @@ -216,6 +216,7 @@ public void widgetSelected(SelectionEvent e) { PropsUi.setLook(wlCommit); wCommit = new TextVar(variables, composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wCommit.enableExpandedInteger(); wCommit.addModifyListener(lsMod); wCommit.setLayoutData( new FormDataBuilder().left(middle, 0).top(wTable, margin).right().result()); diff --git a/plugins/transforms/insertupdate/src/main/java/org/apache/hop/pipeline/transforms/insertupdate/InsertUpdateMeta.java b/plugins/transforms/insertupdate/src/main/java/org/apache/hop/pipeline/transforms/insertupdate/InsertUpdateMeta.java index e04276c45ad..d3fbee73023 100644 --- a/plugins/transforms/insertupdate/src/main/java/org/apache/hop/pipeline/transforms/insertupdate/InsertUpdateMeta.java +++ b/plugins/transforms/insertupdate/src/main/java/org/apache/hop/pipeline/transforms/insertupdate/InsertUpdateMeta.java @@ -113,7 +113,9 @@ public String getCommitSize() { public int getCommitSizeVar(IVariables vs) { // this happens when the transform is created via API and no setDefaults was called commitSize = (commitSize == null) ? "0" : commitSize; - return Integer.parseInt(vs.resolve(commitSize)); + String resolved = vs.resolve(commitSize); + String expanded = Const.expandIntegerString(resolved); + return Integer.parseInt(expanded != null ? expanded : resolved); } /** diff --git a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputDialog.java b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputDialog.java index eb6ad94bf9e..4f737f02e2f 100644 --- a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputDialog.java +++ b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsoninput/JsonInputDialog.java @@ -1293,7 +1293,7 @@ private void ok() { private void getInfo(JsonInputMeta in) { transformName = wTransformName.getText(); // return value - in.setRowLimit(Const.toLong(wLimit.getText(), 0L)); + in.setRowLimit(Const.toLongExpanded(wLimit.getText(), 0L)); in.setFilenameField(wInclFilenameField.getText()); in.setRowNumberField(wInclRownumField.getText()); in.setAddResultFile(wAddResult.getSelection()); diff --git a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/JsonNormalizeInputDialog.java b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/JsonNormalizeInputDialog.java index c2d4cfc28ef..a43d2efe4ef 100644 --- a/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/JsonNormalizeInputDialog.java +++ b/plugins/transforms/json/src/main/java/org/apache/hop/pipeline/transforms/jsonnormalize/JsonNormalizeInputDialog.java @@ -1419,7 +1419,7 @@ private void ok() { private void getInfo(JsonNormalizeInputMeta in) { transformName = wTransformName.getText(); // return value - in.setRowLimit(Const.toLong(wLimit.getText(), 0L)); + in.setRowLimit(Const.toLongExpanded(wLimit.getText(), 0L)); in.setFilenameField(wInclFilenameField.getText()); in.setRowNumberField(wInclRownumField.getText()); in.setAddResultFile(wAddResult.getSelection()); diff --git a/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInput.java b/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInput.java index 7a49d845029..375e7dc521c 100644 --- a/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInput.java +++ b/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInput.java @@ -84,7 +84,7 @@ public boolean init() { data.incomingRowsBuffer = new ArrayList<>(); data.batchDuration = Const.toInt(resolve(meta.getBatchDuration()), 0); - data.batchSize = Const.toInt(resolve(meta.getBatchSize()), 0); + data.batchSize = Const.toIntExpanded(resolve(meta.getBatchSize()), 0); data.stopWhenIdle = meta.isStopWhenIdle(); data.maxIdleTimeMs = Const.toLong(resolve(meta.getMaxIdleTimeMs()), 500L); data.lastRecordTime = System.currentTimeMillis(); @@ -252,7 +252,7 @@ public static Consumer buildKafkaConsumer(IVariables variables, KafkaConsumerInp // The batch size : max poll size // - int batch = Const.toInt(variables.resolve(meta.getBatchSize()), 0); + int batch = Const.toIntExpanded(variables.resolve(meta.getBatchSize()), 0); if (batch > 0) { config.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, batch); } diff --git a/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputDialog.java b/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputDialog.java index 1a5c7e67a2c..fb3f48a5d4b 100644 --- a/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputDialog.java +++ b/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputDialog.java @@ -611,6 +611,7 @@ private void buildBatchTab() { wlBatchSize.setLayoutData(fdlBatchSize); wBatchSize = new TextVar(variables, wBatchComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wBatchSize.enableExpandedInteger(); PropsUi.setLook(wBatchSize); wBatchSize.addModifyListener(lsMod); FormData fdBatchSize = new FormData(); diff --git a/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputMeta.java b/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputMeta.java index c596d463d6e..303fc186715 100644 --- a/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputMeta.java +++ b/plugins/transforms/kafka/src/main/java/org/apache/hop/pipeline/transforms/kafka/consumer/KafkaConsumerInputMeta.java @@ -401,7 +401,9 @@ public void check( long size = Long.MIN_VALUE; try { - size = Long.parseLong(variables.resolve(getBatchSize())); + String batchSizeResolved = variables.resolve(getBatchSize()); + String batchSizeExpanded = Const.expandIntegerString(batchSizeResolved); + size = Long.parseLong(batchSizeExpanded != null ? batchSizeExpanded : batchSizeResolved); } catch (NumberFormatException e) { remarks.add( new CheckResult( diff --git a/plugins/transforms/ldap/src/main/java/org/apache/hop/pipeline/transforms/ldapinput/LdapInputDialog.java b/plugins/transforms/ldap/src/main/java/org/apache/hop/pipeline/transforms/ldapinput/LdapInputDialog.java index 5a94cb5a94f..88d8b82dbfa 100644 --- a/plugins/transforms/ldap/src/main/java/org/apache/hop/pipeline/transforms/ldapinput/LdapInputDialog.java +++ b/plugins/transforms/ldap/src/main/java/org/apache/hop/pipeline/transforms/ldapinput/LdapInputDialog.java @@ -1381,7 +1381,7 @@ private void getInfo(LdapInputMeta in) { in.setTrustAllCertificates(wTrustAll.getSelection()); // copy info to TextFileInputMeta class (input) - in.setRowLimit(Const.toInt(wLimit.getText(), 0)); + in.setRowLimit(Const.toIntExpanded(wLimit.getText(), 0)); in.setTimeLimit(Const.toInt(wTimeLimit.getText(), 0)); in.setMultiValuedSeparator(wMultiValuedSeparator.getText()); in.setIncludeRowNumber(wInclRownum.getSelection()); diff --git a/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java b/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java index 6f2767b833c..f160346c716 100644 --- a/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java +++ b/plugins/transforms/loadfileinput/src/main/java/org/apache/hop/pipeline/transforms/loadfileinput/LoadFileInputDialog.java @@ -1248,7 +1248,7 @@ private void getInfo(LoadFileInputMeta in) { transformName = wTransformName.getText(); // return value // copy info to TextFileInputMeta class (input) - in.setRowLimit(Const.toLong(wLimit.getText(), 0L)); + in.setRowLimit(Const.toLongExpanded(wLimit.getText(), 0L)); in.setEncoding(wEncoding.getText()); in.setRowNumberField(wInclRownumField.getText()); in.setAddingResultFile(wAddResult.getSelection()); diff --git a/plugins/transforms/pipelineexecutor/src/main/java/org/apache/hop/pipeline/transforms/pipelineexecutor/PipelineExecutor.java b/plugins/transforms/pipelineexecutor/src/main/java/org/apache/hop/pipeline/transforms/pipelineexecutor/PipelineExecutor.java index 07cef4bc097..92f613ad9e7 100644 --- a/plugins/transforms/pipelineexecutor/src/main/java/org/apache/hop/pipeline/transforms/pipelineexecutor/PipelineExecutor.java +++ b/plugins/transforms/pipelineexecutor/src/main/java/org/apache/hop/pipeline/transforms/pipelineexecutor/PipelineExecutor.java @@ -530,7 +530,7 @@ public boolean init() { // How many rows do we group together for the pipeline? if (!Utils.isEmpty(meta.getGroupSize())) { - pipelineExecutorData.groupSize = Const.toInt(resolve(meta.getGroupSize()), -1); + pipelineExecutorData.groupSize = Const.toIntExpanded(resolve(meta.getGroupSize()), -1); } else { pipelineExecutorData.groupSize = -1; } diff --git a/plugins/transforms/pipelineexecutor/src/main/java/org/apache/hop/pipeline/transforms/pipelineexecutor/PipelineExecutorDialog.java b/plugins/transforms/pipelineexecutor/src/main/java/org/apache/hop/pipeline/transforms/pipelineexecutor/PipelineExecutorDialog.java index c5fa2b36f33..7edf90a73a6 100644 --- a/plugins/transforms/pipelineexecutor/src/main/java/org/apache/hop/pipeline/transforms/pipelineexecutor/PipelineExecutorDialog.java +++ b/plugins/transforms/pipelineexecutor/src/main/java/org/apache/hop/pipeline/transforms/pipelineexecutor/PipelineExecutorDialog.java @@ -741,6 +741,7 @@ private void addRowGroupTab() { wlGroupSize.setLayoutData(fdlGroupSize); wGroupSize = new TextVar(variables, wInputComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wGroupSize.enableExpandedInteger(); PropsUi.setLook(wGroupSize); FormData fdGroupSize = new FormData(); fdGroupSize.right = new FormAttachment(100); @@ -1129,7 +1130,7 @@ private void setFlags() { || wGroupTime == null) { return; } - boolean enableSize = Const.toInt(variables.resolve(wGroupSize.getText()), -1) >= 0; + boolean enableSize = Const.toIntExpanded(variables.resolve(wGroupSize.getText()), -1) >= 0; boolean enableField = !Utils.isEmpty(wGroupField.getText()); wlGroupSize.setEnabled(true); diff --git a/plugins/transforms/propertyinput/src/main/java/org/apache/hop/pipeline/transforms/propertyinput/PropertyInputDialog.java b/plugins/transforms/propertyinput/src/main/java/org/apache/hop/pipeline/transforms/propertyinput/PropertyInputDialog.java index 0cf144f1222..9c595a6e3ce 100644 --- a/plugins/transforms/propertyinput/src/main/java/org/apache/hop/pipeline/transforms/propertyinput/PropertyInputDialog.java +++ b/plugins/transforms/propertyinput/src/main/java/org/apache/hop/pipeline/transforms/propertyinput/PropertyInputDialog.java @@ -1339,7 +1339,7 @@ private void getInfo(PropertyInputMeta in) { transformName = wTransformName.getText(); // return value // copy info to PropertyInputMeta class (input) - in.setRowLimit(Const.toLong(wLimit.getText(), 0L)); + in.setRowLimit(Const.toLongExpanded(wLimit.getText(), 0L)); in.setIncludingFilename(wInclFilename.getSelection()); in.setFilenameField(wInclFilenameField.getText()); in.setIncludeRowNumber(wInclRowNum.getSelection()); diff --git a/plugins/transforms/reservoirsampling/src/main/java/org/apache/hop/pipeline/transforms/reservoirsampling/ReservoirSampling.java b/plugins/transforms/reservoirsampling/src/main/java/org/apache/hop/pipeline/transforms/reservoirsampling/ReservoirSampling.java index 1f4330068cd..71561ee98b7 100644 --- a/plugins/transforms/reservoirsampling/src/main/java/org/apache/hop/pipeline/transforms/reservoirsampling/ReservoirSampling.java +++ b/plugins/transforms/reservoirsampling/src/main/java/org/apache/hop/pipeline/transforms/reservoirsampling/ReservoirSampling.java @@ -19,6 +19,7 @@ import java.util.Arrays; import java.util.List; +import org.apache.hop.core.Const; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.exception.HopTransformException; import org.apache.hop.pipeline.Pipeline; @@ -91,7 +92,10 @@ public boolean processRow() throws HopException { data.setOutputRowMeta(getInputRowMeta().clone()); String sampleSize = resolve(meta.getSampleSize()); String seed = resolve(meta.getSeed()); - data.initialize(Integer.parseInt(sampleSize), Integer.parseInt(seed)); + String expandedSampleSize = Const.expandIntegerString(sampleSize); + data.initialize( + Integer.parseInt(expandedSampleSize != null ? expandedSampleSize : sampleSize), + Integer.parseInt(seed)); // no real reason to determine the output fields here // as we don't add/delete any fields diff --git a/plugins/transforms/reservoirsampling/src/main/java/org/apache/hop/pipeline/transforms/reservoirsampling/ReservoirSamplingDialog.java b/plugins/transforms/reservoirsampling/src/main/java/org/apache/hop/pipeline/transforms/reservoirsampling/ReservoirSamplingDialog.java index 9fa17ec5346..cf0a603afc3 100644 --- a/plugins/transforms/reservoirsampling/src/main/java/org/apache/hop/pipeline/transforms/reservoirsampling/ReservoirSamplingDialog.java +++ b/plugins/transforms/reservoirsampling/src/main/java/org/apache/hop/pipeline/transforms/reservoirsampling/ReservoirSamplingDialog.java @@ -89,6 +89,7 @@ public String open() { mWlSampleSize.setLayoutData(mFdlSampleSize); wSampleSize = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wSampleSize.enableExpandedInteger(); PropsUi.setLook(wSampleSize); wSampleSize.addModifyListener(lsMod); FormData mFdSampleSize = new FormData(); diff --git a/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGenerator.java b/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGenerator.java index 500c6bbe6bb..313ecd093ab 100644 --- a/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGenerator.java +++ b/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGenerator.java @@ -263,9 +263,9 @@ public boolean init() { if (super.init()) { // Determine the number of rows to generate... - data.rowLimit = Const.toLong(resolve(meta.getRowLimit()), -1L); + data.rowLimit = Const.toLongExpanded(resolve(meta.getRowLimit()), -1L); data.rowsWritten = 0L; - data.delay = Const.toLong(resolve(meta.getIntervalInMs()), -1L); + data.delay = Const.toLongExpanded(resolve(meta.getIntervalInMs()), -1L); if (!meta.isNeverEnding() && data.rowLimit < 0L) { // Unable to parse logError(BaseMessages.getString(PKG, "RowGenerator.Wrong.RowLimit.Number")); diff --git a/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorDialog.java b/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorDialog.java index b59130a565a..7a6a06f7b04 100644 --- a/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorDialog.java +++ b/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorDialog.java @@ -95,6 +95,7 @@ public String open() { fdlLimit.top = new FormAttachment(lastControl, margin); wlLimit.setLayoutData(fdlLimit); wLimit = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wLimit.enableExpandedInteger(); PropsUi.setLook(wLimit); FormData fdLimit = new FormData(); fdLimit.left = new FormAttachment(middle, 0); diff --git a/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorMeta.java b/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorMeta.java index 7bd6fd0679a..6be2aed62dd 100644 --- a/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorMeta.java +++ b/plugins/transforms/rowgenerator/src/main/java/org/apache/hop/pipeline/transforms/rowgenerator/RowGeneratorMeta.java @@ -168,7 +168,7 @@ public void check( remarks.add(cr); String strLimit = variables.resolve(rowLimit); - if (Const.toLong(strLimit, -1L) <= 0) { + if (Const.toLongExpanded(strLimit, -1L) <= 0) { cr = new CheckResult( ICheckResult.TYPE_RESULT_WARNING, diff --git a/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInput.java b/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInput.java index b7895785be1..6a3b0da60bc 100644 --- a/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInput.java +++ b/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInput.java @@ -93,7 +93,7 @@ public boolean processRow() throws HopException { data.outputRowMeta = getInputRowMeta().clone(); meta.getFields(data.outputRowMeta, getTransformName(), null, null, this, metadataProvider); - data.limit = Const.toLong(resolve(meta.getLimit()), -1); + data.limit = Const.toLongExpanded(resolve(meta.getLimit()), -1); } String rawFilename = getInputRowMeta().getString(fileRowData, meta.getAcceptingField(), null); diff --git a/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputDialog.java b/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputDialog.java index 7c5058785e1..b2d971b4f95 100644 --- a/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputDialog.java +++ b/plugins/transforms/sasinput/src/main/java/org/apache/hop/pipeline/transforms/sasinput/SasInputDialog.java @@ -133,6 +133,7 @@ public String open() { fdlLimit.right = new FormAttachment(middle, -margin); wlLimit.setLayoutData(fdlLimit); wLimit = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wLimit.enableExpandedInteger(); wLimit.setToolTipText(BaseMessages.getString(PKG, "SASInputDialog.Limit.Tooltip")); PropsUi.setLook(wLimit); FormData fdLimit = new FormData(); diff --git a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java index 025570177e6..1dec1ed67c5 100644 --- a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java +++ b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRows.java @@ -518,7 +518,7 @@ public boolean init() { return false; } - data.sortSize = Const.toInt(resolve(meta.getSortSize()), -1); + data.sortSize = Const.toIntExpanded(resolve(meta.getSortSize()), -1); data.freeMemoryPctLimit = Const.toInt(meta.getFreeMemoryLimit(), -1); if (data.sortSize <= 0 && data.freeMemoryPctLimit <= 0) { // Prefer the memory limit as it should never fail diff --git a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java index 02cd8d3e634..b73d57bd684 100644 --- a/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java +++ b/plugins/transforms/sort/src/main/java/org/apache/hop/pipeline/transforms/sort/SortRowsDialog.java @@ -153,6 +153,7 @@ public String open() { fdlSortSize.top = new FormAttachment(wPrefix, margin); wlSortSize.setLayoutData(fdlSortSize); wSortSize = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wSortSize.enableExpandedInteger(); PropsUi.setLook(wSortSize); wSortSize.addModifyListener(lsMod); FormData fdSortSize = new FormData(); diff --git a/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMerge.java b/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMerge.java index 25a9d097d56..bba2613f9e4 100644 --- a/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMerge.java +++ b/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMerge.java @@ -972,7 +972,10 @@ public boolean init() { data.releaseSavepoint = false; } - data.commitSize = Integer.parseInt(resolve(meta.getCommitSize())); + String commitSize = resolve(meta.getCommitSize()); + String expandedCommitSize = Const.expandIntegerString(commitSize); + data.commitSize = + Integer.parseInt(expandedCommitSize != null ? expandedCommitSize : commitSize); data.batchMode = data.commitSize > 0 && meta.isUsingBatchUpdates(); // Batch updates are not supported on PostgreSQL (and look-a-likes) together with error diff --git a/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergeDialog.java b/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergeDialog.java index e33d9cc88aa..e49edb866b2 100644 --- a/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergeDialog.java +++ b/plugins/transforms/synchronizeaftermerge/src/main/java/org/apache/hop/pipeline/transforms/synchronizeaftermerge/SynchronizeAfterMergeDialog.java @@ -618,6 +618,7 @@ private void addGeneralTab( wlCommit.setLayoutData(fdlCommit); wCommit = new TextVar(variables, wGeneralComp, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wCommit.enableExpandedInteger(); PropsUi.setLook(wCommit); wCommit.addModifyListener(lsMod); FormData fdCommit = new FormData(); diff --git a/plugins/transforms/tableinput/src/main/java/org/apache/hop/pipeline/transforms/tableinput/TableInput.java b/plugins/transforms/tableinput/src/main/java/org/apache/hop/pipeline/transforms/tableinput/TableInput.java index ece2fcfec35..ce0d4c721c6 100644 --- a/plugins/transforms/tableinput/src/main/java/org/apache/hop/pipeline/transforms/tableinput/TableInput.java +++ b/plugins/transforms/tableinput/src/main/java/org/apache/hop/pipeline/transforms/tableinput/TableInput.java @@ -341,7 +341,7 @@ public boolean init() { DatabaseMeta databaseMeta = getPipelineMeta().findDatabase(meta.getConnection(), variables); data.db = new Database(this, this, databaseMeta); - data.db.setQueryLimit(Const.toInt(resolve(meta.getRowLimit()), 0)); + data.db.setQueryLimit(Const.toIntExpanded(resolve(meta.getRowLimit()), 0)); // Statement timeout is for transform dialog / pipeline preview only (Hop GUI sets preview). // Normal pipeline runs use JDBC driver default (0 = no explicit timeout on the statement). if (getPipeline() != null && getPipeline().isPreview()) { diff --git a/plugins/transforms/tableinput/src/main/java/org/apache/hop/pipeline/transforms/tableinput/TableInputDialog.java b/plugins/transforms/tableinput/src/main/java/org/apache/hop/pipeline/transforms/tableinput/TableInputDialog.java index 348dd338107..e613f837ab9 100644 --- a/plugins/transforms/tableinput/src/main/java/org/apache/hop/pipeline/transforms/tableinput/TableInputDialog.java +++ b/plugins/transforms/tableinput/src/main/java/org/apache/hop/pipeline/transforms/tableinput/TableInputDialog.java @@ -166,6 +166,7 @@ public String open() { fdlLimit.bottom = new FormAttachment(wOk, -margin); wlLimit.setLayoutData(fdlLimit); wLimit = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wLimit.enableExpandedInteger(); PropsUi.setLook(wLimit); wLimit.addModifyListener(lsMod); FormData fdLimit = new FormData(); diff --git a/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java b/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java index 197954d1597..104379385b4 100644 --- a/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java +++ b/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutput.java @@ -491,7 +491,10 @@ public boolean init() { if (super.init()) { try { - data.commitSize = Integer.parseInt(resolve(meta.getCommitSize())); + String commitSize = resolve(meta.getCommitSize()); + String expandedCommitSize = Const.expandIntegerString(commitSize); + data.commitSize = + Integer.parseInt(expandedCommitSize != null ? expandedCommitSize : commitSize); if (Utils.isEmpty(meta.getConnection())) throw new HopException(BaseMessages.getString(PKG, "TableOutput.Init.ConnectionMissing")); diff --git a/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutputDialog.java b/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutputDialog.java index e074ecf46ce..b8fa0f0c5c6 100644 --- a/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutputDialog.java +++ b/plugins/transforms/tableoutput/src/main/java/org/apache/hop/pipeline/transforms/tableoutput/TableOutputDialog.java @@ -296,6 +296,7 @@ public void widgetSelected(SelectionEvent e) { fdlCommit.top = new FormAttachment(wbTable, margin); wlCommit.setLayoutData(fdlCommit); wCommit = new TextVar(variables, wContentComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wCommit.enableExpandedInteger(); PropsUi.setLook(wCommit); FormData fdCommit = new FormData(); fdCommit.left = new FormAttachment(middle, 0); diff --git a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInput.java b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInput.java index 5157b7a5e77..cc3072abe07 100644 --- a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInput.java +++ b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInput.java @@ -29,6 +29,7 @@ import org.apache.commons.io.input.BOMInputStream; import org.apache.commons.vfs2.FileObject; import org.apache.commons.vfs2.provider.local.LocalFile; +import org.apache.hop.core.Const; import org.apache.hop.core.ResultFile; import org.apache.hop.core.exception.HopConversionException; import org.apache.hop.core.exception.HopException; @@ -938,7 +939,10 @@ public boolean init() { if (super.init()) { // see if a variable is used as encoding value String realEncoding = resolve(meta.getEncoding()); - data.preferredBufferSize = Integer.parseInt(resolve(meta.getBufferSize())); + String bufferSize = resolve(meta.getBufferSize()); + String expandedBufferSize = Const.expandIntegerString(bufferSize); + data.preferredBufferSize = + Integer.parseInt(expandedBufferSize != null ? expandedBufferSize : bufferSize); // If the transform doesn't have any previous transforms, we just get the filename. // Otherwise, we'll grab the list of file names later... diff --git a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputDialog.java b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputDialog.java index 12553925168..a53bbc009e3 100644 --- a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputDialog.java +++ b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/csvinput/CsvInputDialog.java @@ -316,6 +316,7 @@ public void widgetSelected(SelectionEvent e) { fdlBufferSize.right = new FormAttachment(middle, -margin); wlBufferSize.setLayoutData(fdlBufferSize); wBufferSize = new TextVar(variables, shell, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wBufferSize.enableExpandedInteger(); PropsUi.setLook(wBufferSize); wBufferSize.addModifyListener(lsMod); FormData fdBufferSize = new FormData(); diff --git a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileInputDialog.java b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileInputDialog.java index 4c0be1472fb..20e72be6f6f 100644 --- a/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileInputDialog.java +++ b/plugins/transforms/textfile/src/main/java/org/apache/hop/pipeline/transforms/fileinput/text/TextFileInputDialog.java @@ -2522,7 +2522,7 @@ private void getInfo(TextFileInputMeta meta, boolean preview) { meta.getContent().setEnclosure(wEnclosure.getText()); meta.getContent().setEscapeCharacter(wEscape.getText()); meta.getContent().setBreakInEnclosureAllowed(wEnclBreaks.getSelection()); - meta.getContent().setRowLimit(Const.toLong(wLimit.getText(), 0L)); + meta.getContent().setRowLimit(Const.toLongExpanded(wLimit.getText(), 0L)); meta.getContent().setFilenameField(wInclFilenameField.getText()); meta.getContent().setRowNumberField(wInclRownumField.getText()); meta.getFileInput().setAddingResult(wAddResult.getSelection()); diff --git a/plugins/transforms/tika/src/main/java/org/apache/hop/pipeline/transforms/tika/TikaDialog.java b/plugins/transforms/tika/src/main/java/org/apache/hop/pipeline/transforms/tika/TikaDialog.java index 58aa4688bc5..965b3474353 100755 --- a/plugins/transforms/tika/src/main/java/org/apache/hop/pipeline/transforms/tika/TikaDialog.java +++ b/plugins/transforms/tika/src/main/java/org/apache/hop/pipeline/transforms/tika/TikaDialog.java @@ -841,7 +841,7 @@ private void getInfo(TikaMeta in) throws HopException { transformName = wTransformName.getText(); // return value // copy info to TikaMeta class (input) - in.setRowLimit(Const.toLong(wLimit.getText(), 0L)); + in.setRowLimit(Const.toLongExpanded(wLimit.getText(), 0L)); in.setEncoding(wEncoding.getText()); in.setOutputFormat(wOutputFormat.getText()); in.setAddingResultFile(wAddResult.getSelection()); diff --git a/plugins/transforms/update/src/main/java/org/apache/hop/pipeline/transforms/update/UpdateDialog.java b/plugins/transforms/update/src/main/java/org/apache/hop/pipeline/transforms/update/UpdateDialog.java index 79e398bb21f..e5d2fd89391 100644 --- a/plugins/transforms/update/src/main/java/org/apache/hop/pipeline/transforms/update/UpdateDialog.java +++ b/plugins/transforms/update/src/main/java/org/apache/hop/pipeline/transforms/update/UpdateDialog.java @@ -263,6 +263,7 @@ public void widgetSelected(SelectionEvent e) { fdlCommit.right = new FormAttachment(middle, -margin); wlCommit.setLayoutData(fdlCommit); wCommit = new TextVar(variables, composite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wCommit.enableExpandedInteger(); PropsUi.setLook(wCommit); wCommit.addModifyListener(lsMod); FormData fdCommit = new FormData(); diff --git a/plugins/transforms/update/src/main/java/org/apache/hop/pipeline/transforms/update/UpdateMeta.java b/plugins/transforms/update/src/main/java/org/apache/hop/pipeline/transforms/update/UpdateMeta.java index 1a40147fa1e..79c7148e0e4 100644 --- a/plugins/transforms/update/src/main/java/org/apache/hop/pipeline/transforms/update/UpdateMeta.java +++ b/plugins/transforms/update/src/main/java/org/apache/hop/pipeline/transforms/update/UpdateMeta.java @@ -125,7 +125,9 @@ public String getCommitSizeVar() { public int getCommitSize(IVariables vs) { // this happens when the transform is created via API and no setDefaults was called commitSize = (commitSize == null) ? "0" : commitSize; - return Integer.parseInt(vs.resolve(commitSize)); + String resolved = vs.resolve(commitSize); + String expanded = Const.expandIntegerString(resolved); + return Integer.parseInt(expanded != null ? expanded : resolved); } public UpdateMeta() { diff --git a/plugins/transforms/workflowexecutor/src/main/java/org/apache/hop/pipeline/transforms/workflowexecutor/WorkflowExecutor.java b/plugins/transforms/workflowexecutor/src/main/java/org/apache/hop/pipeline/transforms/workflowexecutor/WorkflowExecutor.java index 33d16ec86ff..109e8d91fec 100644 --- a/plugins/transforms/workflowexecutor/src/main/java/org/apache/hop/pipeline/transforms/workflowexecutor/WorkflowExecutor.java +++ b/plugins/transforms/workflowexecutor/src/main/java/org/apache/hop/pipeline/transforms/workflowexecutor/WorkflowExecutor.java @@ -442,7 +442,7 @@ public boolean init() { // data.groupSize = -1; if (!Utils.isEmpty(meta.getGroupSize())) { - data.groupSize = Const.toInt(resolve(meta.getGroupSize()), -1); + data.groupSize = Const.toIntExpanded(resolve(meta.getGroupSize()), -1); } // Is there a grouping time set? diff --git a/plugins/transforms/workflowexecutor/src/main/java/org/apache/hop/pipeline/transforms/workflowexecutor/WorkflowExecutorDialog.java b/plugins/transforms/workflowexecutor/src/main/java/org/apache/hop/pipeline/transforms/workflowexecutor/WorkflowExecutorDialog.java index c47cae9232c..b6cb1402e9a 100644 --- a/plugins/transforms/workflowexecutor/src/main/java/org/apache/hop/pipeline/transforms/workflowexecutor/WorkflowExecutorDialog.java +++ b/plugins/transforms/workflowexecutor/src/main/java/org/apache/hop/pipeline/transforms/workflowexecutor/WorkflowExecutorDialog.java @@ -620,6 +620,7 @@ private void addRowGroupTab() { wlGroupSize.setLayoutData(fdlGroupSize); wGroupSize = new TextVar(variables, wInputComposite, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wGroupSize.enableExpandedInteger(); PropsUi.setLook(wGroupSize); FormData fdGroupSize = new FormData(); fdGroupSize.width = 250; @@ -1013,7 +1014,7 @@ private void setFlags() { || wGroupTime == null) { return; } - boolean enableSize = Const.toInt(variables.resolve(wGroupSize.getText()), -1) >= 0; + boolean enableSize = Const.toIntExpanded(variables.resolve(wGroupSize.getText()), -1) >= 0; boolean enableField = !Utils.isEmpty(wGroupField.getText()); wlGroupSize.setEnabled(true); diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXmlDataDialog.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXmlDataDialog.java index f14822bceb9..046e938c503 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXmlDataDialog.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/getxmldata/GetXmlDataDialog.java @@ -1804,7 +1804,7 @@ private void getInfo(GetXmlDataMeta in) { transformName = wTransformName.getText(); // return value // copy info to TextFileInputMeta class (input) - in.setRowLimit(Const.toLong(wLimit.getText(), 0L)); + in.setRowLimit(Const.toLongExpanded(wLimit.getText(), 0L)); in.setPrunePath(wPrunePath.getText()); in.setLoopXPath(wLoopXPath.getText()); in.setEncoding(wEncoding.getText()); diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmlinputstream/XmlInputStream.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmlinputstream/XmlInputStream.java index c712df93a25..7550606ff61 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmlinputstream/XmlInputStream.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmlinputstream/XmlInputStream.java @@ -670,8 +670,8 @@ public boolean init() { data.filenames = null; } - data.nrRowsToSkip = Const.toLong(this.resolve(meta.getNrRowsToSkip()), 0); - data.rowLimit = Const.toLong(this.resolve(meta.getRowLimit()), 0); + data.nrRowsToSkip = Const.toLongExpanded(this.resolve(meta.getNrRowsToSkip()), 0); + data.rowLimit = Const.toLongExpanded(this.resolve(meta.getRowLimit()), 0); data.encoding = this.resolve(meta.getEncoding()); data.outputRowMeta = new RowMeta(); diff --git a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmlinputstream/XmlInputStreamDialog.java b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmlinputstream/XmlInputStreamDialog.java index 67a0266c359..c8ed7f678e0 100644 --- a/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmlinputstream/XmlInputStreamDialog.java +++ b/plugins/transforms/xml/src/main/java/org/apache/hop/pipeline/transforms/xml/xmlinputstream/XmlInputStreamDialog.java @@ -307,6 +307,7 @@ public String open() { fdlRowsToSkip.right = new FormAttachment(middle, -margin); wlRowsToSkip.setLayoutData(fdlRowsToSkip); wRowsToSkip = new TextVar(variables, wContent, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wRowsToSkip.enableExpandedInteger(); PropsUi.setLook(wRowsToSkip); wRowsToSkip.addModifyListener(lsMod); FormData fdRowsToSkip = new FormData(); @@ -327,6 +328,7 @@ public String open() { fdlLimit.right = new FormAttachment(middle, -margin); wlLimit.setLayoutData(fdlLimit); wLimit = new TextVar(variables, wContent, SWT.SINGLE | SWT.LEFT | SWT.BORDER); + wLimit.enableExpandedInteger(); PropsUi.setLook(wLimit); wLimit.addModifyListener(lsMod); FormData fdLimit = new FormData(); diff --git a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java index b2aacde4900..057634a7ca1 100644 --- a/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java +++ b/plugins/transforms/yamlinput/src/main/java/org/apache/hop/pipeline/transforms/yamlinput/YamlInputDialog.java @@ -1117,7 +1117,7 @@ private void getInfo(YamlInputMeta in) { transformName = wTransformName.getText(); // copy info to TextFileInputMeta class (input) - in.setRowLimit(Const.toLong(wLimit.getText(), 0L)); + in.setRowLimit(Const.toLongExpanded(wLimit.getText(), 0L)); in.setFilenameField(wInclFilenameField.getText()); in.setRowNumberField(wInclRowNumField.getText()); in.setAddingResultFile(wAddResult.getSelection()); diff --git a/ui/src/main/java/org/apache/hop/ui/core/database/DatabaseConnectionVariablesHelper.java b/ui/src/main/java/org/apache/hop/ui/core/database/DatabaseConnectionVariablesHelper.java new file mode 100644 index 00000000000..0db65bdc8c3 --- /dev/null +++ b/ui/src/main/java/org/apache/hop/ui/core/database/DatabaseConnectionVariablesHelper.java @@ -0,0 +1,175 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.ui.core.database; + +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.hop.core.Const; +import org.apache.hop.core.variables.DescribedVariable; + +/** + * Builds proposed environment variables for a named relational database connection (for example + * {@code EDW_HOSTNAME}) and maps dialog results back to connection fields by name suffix. + */ +public final class DatabaseConnectionVariablesHelper { + + public static final String SUFFIX_HOSTNAME = "_HOSTNAME"; + public static final String SUFFIX_PORT = "_PORT"; + public static final String SUFFIX_DATABASE = "_DATABASE"; + public static final String SUFFIX_USERNAME = "_USERNAME"; + public static final String SUFFIX_PASSWORD = "_PASSWORD"; + public static final String SUFFIX_URL = "_URL"; + + private static final String[] FIELD_SUFFIXES = { + SUFFIX_HOSTNAME, SUFFIX_PORT, SUFFIX_DATABASE, SUFFIX_USERNAME, SUFFIX_PASSWORD, SUFFIX_URL, + }; + + private DatabaseConnectionVariablesHelper() { + // Utility class + } + + /** + * Sanitize a connection name into a variable name prefix. + * + *

Example: {@code "My DB"} → {@code "MY_DB"}, {@code "EDW"} → {@code "EDW"}. + * + * @param name connection name + * @return uppercase prefix safe for variable names, or empty string if name is blank + */ + public static String sanitizeConnectionNamePrefix(String name) { + if (StringUtils.isBlank(name)) { + return ""; + } + String prefix = name.trim().toUpperCase(Locale.ROOT).replaceAll("[^A-Z0-9_]+", "_"); + prefix = prefix.replaceAll("_+", "_"); + return StringUtils.strip(prefix, "_"); + } + + /** + * @param varName variable name without braces + * @return Hop variable expression such as {@code ${EDW_HOSTNAME}} + */ + public static String expressionFor(String varName) { + return "${" + varName + "}"; + } + + /** + * Build proposed described variables from the current connection field values. + * + * @param connectionName human-readable connection name (used in descriptions) + * @param hostname current hostname value + * @param port current port value + * @param databaseName current database name value + * @param username current username value + * @param password current password value + * @param manualUrl current manual URL value (variable proposed only when non-blank) + * @param includeUsername whether username is part of the editor + * @param includePassword whether password is part of the editor + * @param descriptionsBySuffix map of suffix → description text (e.g. {@code _HOSTNAME} → "…") + * @return ordered list of proposed variables + */ + public static List buildProposedVariables( + String connectionName, + String hostname, + String port, + String databaseName, + String username, + String password, + String manualUrl, + boolean includeUsername, + boolean includePassword, + Map descriptionsBySuffix) { + String prefix = sanitizeConnectionNamePrefix(connectionName); + List variables = new ArrayList<>(); + if (StringUtils.isEmpty(prefix)) { + return variables; + } + + addVariable(variables, prefix, SUFFIX_HOSTNAME, hostname, descriptionsBySuffix); + addVariable(variables, prefix, SUFFIX_PORT, port, descriptionsBySuffix); + addVariable(variables, prefix, SUFFIX_DATABASE, databaseName, descriptionsBySuffix); + + if (includeUsername) { + addVariable(variables, prefix, SUFFIX_USERNAME, username, descriptionsBySuffix); + } + if (includePassword) { + addVariable(variables, prefix, SUFFIX_PASSWORD, password, descriptionsBySuffix); + } + if (StringUtils.isNotBlank(manualUrl)) { + addVariable(variables, prefix, SUFFIX_URL, manualUrl, descriptionsBySuffix); + } + + return variables; + } + + /** + * Map variable names from a (possibly edited) list by known field suffix. If multiple names share + * a suffix, the first match wins (stable order of the list). + * + * @param variables variables returned from the review dialog + * @return suffix → full variable name + */ + public static Map findVariableNamesBySuffix(List variables) { + Map bySuffix = new LinkedHashMap<>(); + if (variables == null) { + return bySuffix; + } + for (DescribedVariable variable : variables) { + if (variable == null || StringUtils.isBlank(variable.getName())) { + continue; + } + String name = variable.getName().trim(); + for (String suffix : FIELD_SUFFIXES) { + if (name.endsWith(suffix) && !bySuffix.containsKey(suffix)) { + bySuffix.put(suffix, name); + break; + } + } + } + return bySuffix; + } + + /** + * Suggest a default config filename for the connection prefix (not path-resolved). + * + * @param prefix sanitized connection prefix + * @return e.g. {@code EDW-config.json} + */ + public static String defaultConfigFilename(String prefix) { + if (StringUtils.isEmpty(prefix)) { + return "database-config.json"; + } + return prefix + "-config.json"; + } + + private static void addVariable( + List variables, + String prefix, + String suffix, + String value, + Map descriptionsBySuffix) { + String name = prefix + suffix; + String description = + descriptionsBySuffix != null ? Const.NVL(descriptionsBySuffix.get(suffix), "") : ""; + variables.add(new DescribedVariable(name, Const.NVL(value, ""), description)); + } +} diff --git a/ui/src/main/java/org/apache/hop/ui/core/database/DatabaseMetaEditor.java b/ui/src/main/java/org/apache/hop/ui/core/database/DatabaseMetaEditor.java index 865d9cf17f8..0d2fdbb23e4 100644 --- a/ui/src/main/java/org/apache/hop/ui/core/database/DatabaseMetaEditor.java +++ b/ui/src/main/java/org/apache/hop/ui/core/database/DatabaseMetaEditor.java @@ -22,12 +22,15 @@ import java.util.Arrays; import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.vfs2.FileObject; import org.apache.hop.core.Const; import org.apache.hop.core.Props; +import org.apache.hop.core.config.DescribedVariablesConfigFile; import org.apache.hop.core.database.BaseDatabaseMeta; import org.apache.hop.core.database.DatabaseMeta; import org.apache.hop.core.database.DatabasePluginType; @@ -38,11 +41,15 @@ import org.apache.hop.core.plugins.IPlugin; import org.apache.hop.core.plugins.PluginRegistry; import org.apache.hop.core.util.Utils; +import org.apache.hop.core.variables.DescribedVariable; import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.HopVfs; import org.apache.hop.i18n.BaseMessages; import org.apache.hop.ui.core.PropsUi; import org.apache.hop.ui.core.database.dialog.DatabaseExplorerDialog; +import org.apache.hop.ui.core.dialog.BaseDialog; import org.apache.hop.ui.core.dialog.ErrorDialog; +import org.apache.hop.ui.core.dialog.HopDescribedVariablesDialog; import org.apache.hop.ui.core.dialog.MessageBox; import org.apache.hop.ui.core.dialog.ShowMessageDialog; import org.apache.hop.ui.core.gui.GuiCompositeWidgets; @@ -1413,6 +1420,11 @@ private String[] getConnectionTypes() { @Override public Button[] createButtonsForButtonBar(Composite parent) { + Button wGenerateVariables = new Button(parent, SWT.PUSH); + wGenerateVariables.setText( + BaseMessages.getString(PKG, "DatabaseDialog.button.GenerateVariables")); + wGenerateVariables.addListener(SWT.Selection, e -> generateVariables()); + Button wExplore = new Button(parent, SWT.PUSH); wExplore.setText(BaseMessages.getString(PKG, "DatabaseDialog.button.Explore")); wExplore.addListener(SWT.Selection, e -> explore()); @@ -1421,7 +1433,206 @@ public Button[] createButtonsForButtonBar(Composite parent) { wTest.setText(BaseMessages.getString(PKG, "System.Button.Test")); wTest.addListener(SWT.Selection, e -> test()); - return new Button[] {wExplore, wTest}; + return new Button[] {wGenerateVariables, wExplore, wTest}; + } + + /** + * Propose connection-scoped environment variables (for example {@code EDW_HOSTNAME}), optionally + * write them to a described-variables JSON config file for DTAP environments, and replace the + * editor fields with {@code ${…}} expressions. + */ + private void generateVariables() { + String connectionName = wName.getText(); + if (StringUtils.isBlank(connectionName)) { + MessageBox box = new MessageBox(getShell(), SWT.OK | SWT.ICON_ERROR); + box.setText( + BaseMessages.getString(PKG, "DatabaseDialog.GenerateVariables.NameRequired.Title")); + box.setMessage( + BaseMessages.getString(PKG, "DatabaseDialog.GenerateVariables.NameRequired.Message")); + box.open(); + return; + } + + DatabaseMeta meta = getMetadata(); + getWidgetsContent(meta); + + boolean includeUsername = + wUsername != null && !excludedElementIds.contains(BaseDatabaseMeta.ELEMENT_ID_USERNAME); + boolean includePassword = + wPassword != null && !excludedElementIds.contains(BaseDatabaseMeta.ELEMENT_ID_PASSWORD); + + Map descriptions = new LinkedHashMap<>(); + descriptions.put( + DatabaseConnectionVariablesHelper.SUFFIX_HOSTNAME, + BaseMessages.getString( + PKG, "DatabaseDialog.GenerateVariables.Description.Hostname", connectionName)); + descriptions.put( + DatabaseConnectionVariablesHelper.SUFFIX_PORT, + BaseMessages.getString( + PKG, "DatabaseDialog.GenerateVariables.Description.Port", connectionName)); + descriptions.put( + DatabaseConnectionVariablesHelper.SUFFIX_DATABASE, + BaseMessages.getString( + PKG, "DatabaseDialog.GenerateVariables.Description.Database", connectionName)); + descriptions.put( + DatabaseConnectionVariablesHelper.SUFFIX_USERNAME, + BaseMessages.getString( + PKG, "DatabaseDialog.GenerateVariables.Description.Username", connectionName)); + descriptions.put( + DatabaseConnectionVariablesHelper.SUFFIX_PASSWORD, + BaseMessages.getString( + PKG, "DatabaseDialog.GenerateVariables.Description.Password", connectionName)); + descriptions.put( + DatabaseConnectionVariablesHelper.SUFFIX_URL, + BaseMessages.getString( + PKG, "DatabaseDialog.GenerateVariables.Description.Url", connectionName)); + + List proposed = + DatabaseConnectionVariablesHelper.buildProposedVariables( + connectionName, + meta.getHostname(), + meta.getPort(), + meta.getDatabaseName(), + meta.getUsername(), + meta.getPassword(), + meta.getManualUrl(), + includeUsername, + includePassword, + descriptions); + + if (proposed.isEmpty()) { + MessageBox box = new MessageBox(getShell(), SWT.OK | SWT.ICON_ERROR); + box.setText( + BaseMessages.getString(PKG, "DatabaseDialog.GenerateVariables.NameRequired.Title")); + box.setMessage( + BaseMessages.getString(PKG, "DatabaseDialog.GenerateVariables.NameRequired.Message")); + box.open(); + return; + } + + HopDescribedVariablesDialog variablesDialog = + new HopDescribedVariablesDialog( + getShell(), + BaseMessages.getString( + PKG, "DatabaseDialog.GenerateVariables.DialogMessage", connectionName), + proposed, + null); + List confirmed = variablesDialog.open(); + if (confirmed == null) { + return; + } + + // Optional: create or update a described-variables JSON file for DTAP / deployment use. + // Values may be left empty in the dialog to produce a VCS template; environment-specific + // copies are typically deployed per stage and not auto-registered on a lifecycle environment. + MessageBox createConfigBox = new MessageBox(getShell(), SWT.YES | SWT.NO | SWT.ICON_QUESTION); + createConfigBox.setText( + BaseMessages.getString(PKG, "DatabaseDialog.GenerateVariables.CreateConfig.Title")); + createConfigBox.setMessage( + BaseMessages.getString(PKG, "DatabaseDialog.GenerateVariables.CreateConfig.Message")); + if (createConfigBox.open() == SWT.YES) { + saveVariablesToConfigFile(confirmed, connectionName); + } + + applyVariableExpressionsToEditor(meta, confirmed); + setWidgetsContent(); + setChanged(); + MetadataPerspective.getInstance().updateEditor(this); + } + + private void saveVariablesToConfigFile(List variables, String connectionName) { + try { + String prefix = + DatabaseConnectionVariablesHelper.sanitizeConnectionNamePrefix(connectionName); + String defaultName = DatabaseConnectionVariablesHelper.defaultConfigFilename(prefix); + + FileObject startFile; + String projectHome = manager.getVariables().resolve(Const.VAR_PROJECT_HOME); + if (StringUtils.isNotEmpty(projectHome) + && !Const.VAR_PROJECT_HOME.equals(projectHome) + && HopVfs.fileExists(projectHome)) { + startFile = HopVfs.getFileObject(projectHome + Const.FILE_SEPARATOR + defaultName); + } else { + startFile = HopVfs.getFileObject(defaultName); + } + + String configFilename = + BaseDialog.presentFileDialog( + true, + getShell(), + null, + manager.getVariables(), + startFile, + new String[] {"*.json", "*"}, + new String[] { + BaseMessages.getString(PKG, "DatabaseDialog.GenerateVariables.FileFilter.Json"), + BaseMessages.getString(PKG, "DatabaseDialog.GenerateVariables.FileFilter.All") + }, + true); + if (StringUtils.isEmpty(configFilename)) { + return; + } + + // Resolve variables (e.g. ${PROJECT_HOME}) before VFS read/write + String realConfigFilename = manager.getVariables().resolve(configFilename); + + DescribedVariablesConfigFile configFile = + new DescribedVariablesConfigFile(realConfigFilename); + if (HopVfs.fileExists(realConfigFilename)) { + configFile.readFromFile(); + } + for (DescribedVariable variable : variables) { + if (variable != null && StringUtils.isNotBlank(variable.getName())) { + configFile.setDescribedVariable(variable); + } + } + configFile.saveToFile(); + + MessageBox done = new MessageBox(getShell(), SWT.OK | SWT.ICON_INFORMATION); + done.setText( + BaseMessages.getString(PKG, "DatabaseDialog.GenerateVariables.ConfigSaved.Title")); + done.setMessage( + BaseMessages.getString( + PKG, "DatabaseDialog.GenerateVariables.ConfigSaved.Message", realConfigFilename)); + done.open(); + } catch (Exception e) { + new ErrorDialog( + getShell(), + BaseMessages.getString(PKG, "DatabaseDialog.GenerateVariables.ConfigError.Title"), + BaseMessages.getString(PKG, "DatabaseDialog.GenerateVariables.ConfigError.Message"), + e); + } + } + + private void applyVariableExpressionsToEditor( + DatabaseMeta meta, List variables) { + Map namesBySuffix = + DatabaseConnectionVariablesHelper.findVariableNamesBySuffix(variables); + + String hostnameVar = namesBySuffix.get(DatabaseConnectionVariablesHelper.SUFFIX_HOSTNAME); + if (hostnameVar != null) { + meta.setHostname(DatabaseConnectionVariablesHelper.expressionFor(hostnameVar)); + } + String portVar = namesBySuffix.get(DatabaseConnectionVariablesHelper.SUFFIX_PORT); + if (portVar != null) { + meta.setPort(DatabaseConnectionVariablesHelper.expressionFor(portVar)); + } + String databaseVar = namesBySuffix.get(DatabaseConnectionVariablesHelper.SUFFIX_DATABASE); + if (databaseVar != null) { + meta.setDBName(DatabaseConnectionVariablesHelper.expressionFor(databaseVar)); + } + String usernameVar = namesBySuffix.get(DatabaseConnectionVariablesHelper.SUFFIX_USERNAME); + if (usernameVar != null) { + meta.setUsername(DatabaseConnectionVariablesHelper.expressionFor(usernameVar)); + } + String passwordVar = namesBySuffix.get(DatabaseConnectionVariablesHelper.SUFFIX_PASSWORD); + if (passwordVar != null) { + meta.setPassword(DatabaseConnectionVariablesHelper.expressionFor(passwordVar)); + } + String urlVar = namesBySuffix.get(DatabaseConnectionVariablesHelper.SUFFIX_URL); + if (urlVar != null) { + meta.setManualUrl(DatabaseConnectionVariablesHelper.expressionFor(urlVar)); + } } @Override diff --git a/ui/src/main/java/org/apache/hop/ui/core/dialog/HopDescribedVariablesDialog.java b/ui/src/main/java/org/apache/hop/ui/core/dialog/HopDescribedVariablesDialog.java index af433170483..d89f5e48b19 100644 --- a/ui/src/main/java/org/apache/hop/ui/core/dialog/HopDescribedVariablesDialog.java +++ b/ui/src/main/java/org/apache/hop/ui/core/dialog/HopDescribedVariablesDialog.java @@ -229,6 +229,10 @@ private void encodeSelectedValue() { for (int index : wFields.getSelectionIndices()) { TableItem item = wFields.table.getItem(index); String value = item.getText(2); + // Already encrypted? Skip so double-click / re-encode does not nest encryption + if (value != null && value.startsWith(Encr.PASSWORD_ENCRYPTED_PREFIX)) { + continue; + } String encoded = encoder.encode(value, true); item.setText(2, Const.NVL(encoded, "")); } diff --git a/ui/src/main/java/org/apache/hop/ui/core/gui/GuiResource.java b/ui/src/main/java/org/apache/hop/ui/core/gui/GuiResource.java index 78bea972f9f..4c4ee136df8 100644 --- a/ui/src/main/java/org/apache/hop/ui/core/gui/GuiResource.java +++ b/ui/src/main/java/org/apache/hop/ui/core/gui/GuiResource.java @@ -143,6 +143,7 @@ public class GuiResource { private SwtUniversalImage imageMissing; private SwtUniversalImage imageDeprecated; private SwtUniversalImage imageVariable; + private SwtUniversalImage imageHash; private SwtUniversalImage imagePipeline; private SwtUniversalImage imagePipelineDisabled; private SwtUniversalImage imagePartitionSchema; @@ -444,6 +445,7 @@ private void dispose() { imageFolder.dispose(); imageMissing.dispose(); imageVariable.dispose(); + imageHash.dispose(); imagePipeline.dispose(); imagePipelineDisabled.dispose(); imagePartitionSchema.dispose(); @@ -829,6 +831,7 @@ private void loadCommonImages() { imageFalseDisabled = SwtSvgImageUtil.getImageAsResource(display, "ui/images/false-disabled.svg"); imageVariable = SwtSvgImageUtil.getImageAsResource(display, "ui/images/variable.svg"); + imageHash = SwtSvgImageUtil.getImageAsResource(display, "ui/images/hash.svg"); imageFile = SwtSvgImageUtil.getImageAsResource(display, "ui/images/file.svg"); imageFolder = SwtSvgImageUtil.getImageAsResource(display, "ui/images/folder.svg"); imagePartitionSchema = @@ -1188,6 +1191,18 @@ public Image getImageVariableMini() { return getZoomedImaged(imageVariable, display, 12, 12); } + /** + * @return the hash / expanded-integer indicator image at standard small icon size + */ + public Image getImageHash() { + return getZoomedImaged(imageHash, display, ConstUi.SMALL_ICON_SIZE, ConstUi.SMALL_ICON_SIZE); + } + + /** Mini hash icon for TextVar expanded-integer notation indicator (matches variable mini). */ + public Image getImageHashMini() { + return getZoomedImaged(imageHash, display, 12, 12); + } + public Image getImagePipeline() { return getZoomedImaged( imagePipeline, display, ConstUi.SMALL_ICON_SIZE, ConstUi.SMALL_ICON_SIZE); diff --git a/ui/src/main/java/org/apache/hop/ui/core/widget/TextVar.java b/ui/src/main/java/org/apache/hop/ui/core/widget/TextVar.java index 3a8e7c16e90..33623234234 100644 --- a/ui/src/main/java/org/apache/hop/ui/core/widget/TextVar.java +++ b/ui/src/main/java/org/apache/hop/ui/core/widget/TextVar.java @@ -59,6 +59,14 @@ public class TextVar extends Composite { protected Text wText; + /** Variable ($) indicator; always present. */ + protected Label wVariableImage; + + /** Optional expanded-integer (#) indicator; created by {@link #enableExpandedInteger()}. */ + protected Label wHashImage; + + protected FormData fdText; + protected ModifyListener modifyListenerTooltipText; public TextVar(IVariables variables, Composite composite, int flags) { @@ -158,22 +166,22 @@ protected void initialize( // Add the variable $ image on the top right of the control // - Label wImage = new Label(this, SWT.NONE); - PropsUi.setLook(wImage); - wImage.setImage(GuiResource.getInstance().getImageVariableMini()); - wImage.setToolTipText(BaseMessages.getString(PKG, "TextVar.tooltip.InsertVariable")); + wVariableImage = new Label(this, SWT.NONE); + PropsUi.setLook(wVariableImage); + wVariableImage.setImage(GuiResource.getInstance().getImageVariableMini()); + wVariableImage.setToolTipText(BaseMessages.getString(PKG, "TextVar.tooltip.InsertVariable")); FormData fdlImage = new FormData(); fdlImage.top = new FormAttachment(0, 0); fdlImage.right = new FormAttachment(100, 0); - wImage.setLayoutData(fdlImage); + wVariableImage.setLayoutData(fdlImage); // add a text field on it... wText = new Text(this, flags); PropsUi.setLook(wText); - FormData fdText = new FormData(); + fdText = new FormData(); fdText.top = new FormAttachment(0, 0); fdText.left = new FormAttachment(0, 0); - fdText.right = new FormAttachment(wImage, 0); + fdText.right = new FormAttachment(wVariableImage, 0); fdText.bottom = new FormAttachment(100, 0); wText.setLayoutData(fdText); @@ -186,6 +194,38 @@ protected void initialize( wText.addKeyListener(controlSpaceKeyAdapter); } + /** + * Show a mini {@code #} indicator that this field accepts expanded integer notation (grouping + * separators, k/m/g/b suffixes, scientific forms). Call after construction on fields that use + * {@link Const#toIntExpanded(String, int)} / {@link Const#toLongExpanded(String, long)}. + * + *

Layout becomes: {@code [ text ........ ] [#] [$]}. Safe to call more than once. + * + * @return this widget for chaining + */ + public TextVar enableExpandedInteger() { + if (wHashImage != null || wVariableImage == null || wText == null) { + return this; + } + + wHashImage = new Label(this, SWT.NONE); + PropsUi.setLook(wHashImage); + wHashImage.setImage(GuiResource.getInstance().getImageHashMini()); + wHashImage.setToolTipText(BaseMessages.getString(PKG, "TextVar.tooltip.ExpandedInteger")); + FormData fdlHash = new FormData(); + fdlHash.top = new FormAttachment(0, 0); + fdlHash.right = new FormAttachment(wVariableImage, 0); + wHashImage.setLayoutData(fdlHash); + + // Rebind text field to end at the hash icon (left of $) + if (fdText != null) { + fdText.right = new FormAttachment(wHashImage, 0); + wText.setLayoutData(fdText); + } + layout(true, true); + return this; + } + /** * @return the getCaretPositionInterface */ diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/delegates/HopGuiFileDelegate.java b/ui/src/main/java/org/apache/hop/ui/hopgui/delegates/HopGuiFileDelegate.java index a02835547aa..4b22a448486 100644 --- a/ui/src/main/java/org/apache/hop/ui/hopgui/delegates/HopGuiFileDelegate.java +++ b/ui/src/main/java/org/apache/hop/ui/hopgui/delegates/HopGuiFileDelegate.java @@ -277,6 +277,17 @@ public void closeAllFiles() { } } } + + // Execution Information tabs are not IHopFileTypeHandlers, so they never appear in + // getItems(). Close them explicitly so project switches (and File → Close All) do not leave + // viewers from the previous project open. Callers that need to remember tabs (project switch) + // must call ExecutionPerspective.saveState() first. + // + ExecutionPerspective executionPerspective = ExecutionPerspective.getInstance(); + if (executionPerspective != null) { + executionPerspective.closeAllTabs(); + } + this.isClosing = false; } diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/execution/ExecutionPerspective.java b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/execution/ExecutionPerspective.java index 6f2293cde62..d18587f127e 100644 --- a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/execution/ExecutionPerspective.java +++ b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/execution/ExecutionPerspective.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import lombok.Getter; +import org.apache.commons.lang3.StringUtils; import org.apache.hop.core.Const; import org.apache.hop.core.Props; import org.apache.hop.core.exception.HopException; @@ -48,6 +49,9 @@ import org.apache.hop.execution.IExecutionInfoLocation; import org.apache.hop.execution.IExecutionSelector; import org.apache.hop.execution.LastPeriod; +import org.apache.hop.execution.caching.CachingFileExecutionInfoLocation; +import org.apache.hop.execution.local.FileExecutionInfoLocation; +import org.apache.hop.history.AuditList; import org.apache.hop.history.AuditManager; import org.apache.hop.history.AuditState; import org.apache.hop.history.AuditStateMap; @@ -82,6 +86,7 @@ import org.apache.hop.ui.hopgui.perspective.IHopPerspective; import org.apache.hop.ui.hopgui.perspective.TabClosable; import org.apache.hop.ui.hopgui.perspective.TabCloseHandler; +import org.apache.hop.ui.hopgui.shared.BaseExecutionViewer; import org.apache.hop.ui.hopgui.shared.SashFormMemory; import org.apache.hop.workflow.WorkflowMeta; import org.eclipse.swt.SWT; @@ -91,12 +96,16 @@ import org.eclipse.swt.custom.CTabItem; import org.eclipse.swt.custom.SashForm; import org.eclipse.swt.graphics.Cursor; +import org.eclipse.swt.graphics.Point; +import org.eclipse.swt.graphics.Rectangle; import org.eclipse.swt.layout.FormAttachment; import org.eclipse.swt.layout.FormData; import org.eclipse.swt.layout.FormLayout; import org.eclipse.swt.widgets.Combo; import org.eclipse.swt.widgets.Composite; import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Menu; +import org.eclipse.swt.widgets.MenuItem; import org.eclipse.swt.widgets.Shell; import org.eclipse.swt.widgets.Text; import org.eclipse.swt.widgets.ToolBar; @@ -121,6 +130,8 @@ public class ExecutionPerspective implements IHopPerspective, TabClosable { public static final String GUI_PLUGIN_TOOLBAR_PARENT_ID = "ExecutionPerspective-Toolbar"; + public static final String TOOLBAR_ITEM_COPY_FILENAME = + "ExecutionPerspective-Toolbar-10000-CopyFilename"; public static final String TOOLBAR_ITEM_EDIT = "ExecutionPerspective-Toolbar-10010-Edit"; public static final String TOOLBAR_ITEM_DUPLICATE = "ExecutionPerspective-Toolbar-10030-Duplicate"; @@ -152,7 +163,11 @@ public class ExecutionPerspective implements IHopPerspective, TabClosable { public static final String CONST_ERROR1 = "Error"; public static final String FILTER_NAME_DATE_ID = "name - date - ID"; + /** Tree item data key for the empty-state "configure locations" placeholder. */ + public static final String CONFIGURE_LOCATIONS = "configure-locations"; + private static final String EXECUTION_AUDIT_TYPE = "execution-perspective-gui"; + private static final String EXECUTION_TABS_AUDIT_TYPE = "execution-perspective-tabs"; private static final String AUDIT_EXECUTION_TOOLBAR = "toolbar"; private static final String AUDIT_ONLY_PARENTS = "only-parents"; private static final String AUDIT_ONLY_FAILED = "only-failed"; @@ -162,6 +177,8 @@ public class ExecutionPerspective implements IHopPerspective, TabClosable { private static final String AUDIT_ONLY_PIPELINES = "only-pipelines"; private static final String AUDIT_FILTER_TEXT = "filter-text"; private static final String AUDIT_TIME_FILTER = "time-filter"; + private static final String AUDIT_ACTIVE_TAB = "active-tab"; + private static final String TAB_KEY_DELIMITER = "\t"; public static final String SNAP_ID_EIL_REFRESH = "EILRefresh"; @Getter private static ExecutionPerspective instance; @@ -184,6 +201,19 @@ public class ExecutionPerspective implements IHopPerspective, TabClosable { private final List viewers = new ArrayList<>(); + /** + * When true, {@link #addViewer(IExecutionViewer)} does not activate this perspective. Used while + * restoring open tabs on project switch so the data-orchestration perspective keeps focus. + */ + private boolean restoringTabs; + + /** + * When true, {@link #closeTab(CTabFolderEvent, CTabItem)} does not rewrite audit state. Used by + * {@link #closeAllTabs()} so a project switch that already called {@link #saveState()} is not + * overwritten with an empty open-tabs list. + */ + private boolean closingAllTabs; + /** * Gets locationMap * @@ -327,6 +357,7 @@ protected void createTree(Composite parent) { tree = new Tree(composite, SWT.SINGLE | SWT.H_SCROLL | SWT.V_SCROLL); tree.setHeaderVisible(false); + tree.addListener(SWT.Selection, event -> updateSelection()); tree.addListener( SWT.DefaultSelection, event -> { @@ -336,6 +367,9 @@ protected void createTree(Composite parent) { } }); + // Copy-filename starts disabled until an execution with a filename is selected. + updateSelection(); + PropsUi.setLook(tree); FormData treeFormData = new FormData(); @@ -403,9 +437,11 @@ public void addViewer(IExecutionViewer viewer) { viewers.add(viewer); - // Activate the perspective + // Activate the perspective unless we are restoring tabs after a project switch // - this.activate(); + if (!restoringTabs) { + this.activate(); + } // Switch to the tab // @@ -467,7 +503,9 @@ public void onNewViewer() { TreeItem treeItem = tree.getSelection()[0]; if (treeItem != null) { - if (treeItem.getData() instanceof Execution execution) { + if (Boolean.TRUE.equals(treeItem.getData(CONFIGURE_LOCATIONS))) { + createNewExecutionInfoLocation(); + } else if (treeItem.getData() instanceof Execution execution) { ExecutionInfoLocation location = (ExecutionInfoLocation) treeItem.getParentItem().getData(); ExecutionState executionState = @@ -483,6 +521,29 @@ public void onNewViewer() { } } + /** + * Opens a new Execution Information Location editor in the metadata perspective. Used when the + * user double-clicks the empty-state placeholder in the execution tree. + */ + private void createNewExecutionInfoLocation() { + try { + MetadataManager manager = + new MetadataManager<>( + hopGui.getVariables(), + hopGui.getMetadataProvider(), + ExecutionInfoLocation.class, + hopGui.getShell()); + manager.newMetadataWithEditor(""); + hopGui.getEventsHandler().fire(HopGuiEvents.MetadataCreated.name()); + } catch (Exception e) { + new ErrorDialog( + getShell(), + CONST_ERROR1, + BaseMessages.getString(PKG, "ExecutionPerspective.CreateLocation.Error.Message"), + e); + } + } + public void createExecutionViewer( String locationName, Execution execution, ExecutionState executionState) throws Exception { Cursor busyCursor = getBusyCursor(); @@ -731,115 +792,131 @@ public void refresh() { List locations = serializer.loadAll(); locations.sort(Comparator.comparing(HopMetadataBase::getName)); - ILogChannel log = hopGui.getLog(); - Metrics startLocationRefresh = - new Metrics(MetricsSnapshotType.START, SNAP_ID_EIL_REFRESH, "Refresh EIL tree"); - Metrics endLocationRefresh = - new Metrics(MetricsSnapshotType.STOP, SNAP_ID_EIL_REFRESH, "Refresh EIL tree end"); - log.setGatheringMetrics(true); - - for (ExecutionInfoLocation location : locations) { - log.snap(startLocationRefresh); - IExecutionInfoLocation iLocation = location.getExecutionInfoLocation(); - - try { - // Initialize the location first... - // - iLocation.initialize(hopGui.getVariables(), hopGui.getMetadataProvider()); + // When no locations are configured, show a helpful placeholder the user can double-click + // to create a new Execution Information Location in the metadata perspective. + // + if (locations.isEmpty()) { + TreeItem configureItem = new TreeItem(tree, SWT.NONE); + configureItem.setText( + 0, BaseMessages.getString(PKG, "ExecutionPerspective.Tree.ConfigureLocations")); + configureItem.setImage(GuiResource.getInstance().getImageLocation()); + configureItem.setData(CONFIGURE_LOCATIONS, true); + tree.setToolTipText( + BaseMessages.getString(PKG, "ExecutionPerspective.Tree.ConfigureLocations.Tooltip")); + } else { + ILogChannel log = hopGui.getLog(); + Metrics startLocationRefresh = + new Metrics(MetricsSnapshotType.START, SNAP_ID_EIL_REFRESH, "Refresh EIL tree"); + Metrics endLocationRefresh = + new Metrics(MetricsSnapshotType.STOP, SNAP_ID_EIL_REFRESH, "Refresh EIL tree end"); - // Keep the location around to close at the next refresh. - // - locationMap.put(location.getName(), location); + log.setGatheringMetrics(true); + tree.setToolTipText(null); - TreeItem locationItem = new TreeItem(tree, SWT.NONE); - locationItem.setText(0, Const.NVL(location.getName(), "")); - locationItem.setImage(GuiResource.getInstance().getImageLocation()); - TreeMemory.getInstance().storeExpanded(EXECUTION_PERSPECTIVE_TREE, locationItem, true); - locationItem.setData(location); + for (ExecutionInfoLocation location : locations) { + log.snap(startLocationRefresh); + IExecutionInfoLocation iLocation = location.getExecutionInfoLocation(); try { - // Get the data in the location. The plugins are supposed to prune as much of the IDs - // upfront. - // Below we'll run the isSelected() condition again to make sure. - // - IExecutionSelector executionSelector = - new DefaultExecutionSelector( - onlyShowingParents, - onlyShowingFailed, - onlyShowingRunning, - onlyShowingFinished, - onlyShowingWorkflows, - onlyShowingPipelines, - filterText, - timeFilter); - List ids = iLocation.findExecutionIDs(executionSelector); - - // Display the executions + // Initialize the location first... // - for (String id : ids) { - try { - Execution execution = iLocation.getExecution(id); - if (execution != null) { - // Apply an extra filter to make sure - // - if (!executionSelector.isSelected(execution)) { - continue; - } - // We only need to consider the state after the previous filtering - // - ExecutionState state = iLocation.getExecutionState(id); - if (!executionSelector.isSelected(state)) { - continue; - } + iLocation.initialize(hopGui.getVariables(), hopGui.getMetadataProvider()); - TreeItem executionItem = new TreeItem(locationItem, SWT.NONE); - switch (execution.getExecutionType()) { - case Pipeline: - decoratePipelineTreeItem(executionItem, location, execution, state); - break; - case Workflow: - decorateWorkflowTreeItem(executionItem, location, execution, state); - break; - default: - break; + // Keep the location around to close at the next refresh. + // + locationMap.put(location.getName(), location); + + TreeItem locationItem = new TreeItem(tree, SWT.NONE); + locationItem.setText(0, Const.NVL(location.getName(), "")); + locationItem.setImage(GuiResource.getInstance().getImageLocation()); + TreeMemory.getInstance().storeExpanded(EXECUTION_PERSPECTIVE_TREE, locationItem, true); + locationItem.setData(location); + + try { + // Get the data in the location. The plugins are supposed to prune as much of the IDs + // upfront. + // Below we'll run the isSelected() condition again to make sure. + // + IExecutionSelector executionSelector = + new DefaultExecutionSelector( + onlyShowingParents, + onlyShowingFailed, + onlyShowingRunning, + onlyShowingFinished, + onlyShowingWorkflows, + onlyShowingPipelines, + filterText, + timeFilter); + List ids = iLocation.findExecutionIDs(executionSelector); + + // Display the executions + // + for (String id : ids) { + try { + Execution execution = iLocation.getExecution(id); + if (execution != null) { + // Apply an extra filter to make sure + // + if (!executionSelector.isSelected(execution)) { + continue; + } + // We only need to consider the state after the previous filtering + // + ExecutionState state = iLocation.getExecutionState(id); + if (!executionSelector.isSelected(state)) { + continue; + } + + TreeItem executionItem = new TreeItem(locationItem, SWT.NONE); + switch (execution.getExecutionType()) { + case Pipeline: + decoratePipelineTreeItem(executionItem, location, execution, state); + break; + case Workflow: + decorateWorkflowTreeItem(executionItem, location, execution, state); + break; + default: + break; + } } + } catch (Exception e) { + TreeItem errorItem = new TreeItem(locationItem, SWT.NONE); + errorItem.setText("Error reading " + id + " (double click for details)"); + errorItem.setForeground(GuiResource.getInstance().getColorRed()); + errorItem.setData(CONST_ERROR, e); + errorItem.setImage(GuiResource.getInstance().getImageError()); } - } catch (Exception e) { - TreeItem errorItem = new TreeItem(locationItem, SWT.NONE); - errorItem.setText("Error reading " + id + " (double click for details)"); - errorItem.setForeground(GuiResource.getInstance().getColorRed()); - errorItem.setData(CONST_ERROR, e); - errorItem.setImage(GuiResource.getInstance().getImageError()); } + } catch (Exception e) { + // Error contacting location + // + TreeItem errorItem = new TreeItem(locationItem, SWT.NONE); + errorItem.setText("Not reachable (double click for details)"); + errorItem.setForeground(GuiResource.getInstance().getColorRed()); + errorItem.setData(CONST_ERROR, e); + errorItem.setImage(GuiResource.getInstance().getImageError()); } } catch (Exception e) { - // Error contacting location + // We couldn't initialize a location // - TreeItem errorItem = new TreeItem(locationItem, SWT.NONE); - errorItem.setText("Not reachable (double click for details)"); - errorItem.setForeground(GuiResource.getInstance().getColorRed()); - errorItem.setData(CONST_ERROR, e); - errorItem.setImage(GuiResource.getInstance().getImageError()); + TreeItem locationItem = new TreeItem(tree, SWT.NONE); + locationItem.setText( + 0, Const.NVL(location.getName(), "") + " (error: double click for details)"); + locationItem.setForeground(GuiResource.getInstance().getColorRed()); + locationItem.setImage(GuiResource.getInstance().getImageLocation()); + locationItem.setData(CONST_ERROR, e); } - } catch (Exception e) { - // We couldn't initialize a location - // - TreeItem locationItem = new TreeItem(tree, SWT.NONE); - locationItem.setText( - 0, Const.NVL(location.getName(), "") + " (error: double click for details)"); - locationItem.setForeground(GuiResource.getInstance().getColorRed()); - locationItem.setImage(GuiResource.getInstance().getImageLocation()); - locationItem.setData(CONST_ERROR, e); + log.snap(endLocationRefresh); } - log.snap(endLocationRefresh); + log.setGatheringMetrics(false); } - log.setGatheringMetrics(false); TreeUtil.setOptimalWidthOnColumns(tree); TreeMemory.setExpandedFromMemory(tree, EXECUTION_PERSPECTIVE_TREE); tree.setRedraw(true); + updateSelection(); } catch (Exception e) { getShell().setCursor(null); new ErrorDialog( @@ -1077,6 +1154,223 @@ public boolean remove(IHopFileTypeHandler typeHandler) { return false; } + /** + * Enables toolbar items that depend on the current tree selection (copy is enabled when an + * execution is selected so UUID and other options can be offered). + */ + private void updateSelection() { + if (toolBarWidgets == null) { + return; + } + toolBarWidgets.enableToolbarItem(TOOLBAR_ITEM_COPY_FILENAME, getSelectedExecution() != null); + } + + /** + * @return the selected execution tree item data, or {@code null} if none / not an execution + */ + private Execution getSelectedExecution() { + if (tree == null || tree.isDisposed() || tree.getSelectionCount() != 1) { + return null; + } + Object data = tree.getSelection()[0].getData(); + if (data instanceof Execution execution) { + return execution; + } + return null; + } + + /** + * @return the parent {@link ExecutionInfoLocation} for the selected execution, or {@code null} + */ + private ExecutionInfoLocation getSelectedExecutionLocation() { + if (tree == null || tree.isDisposed() || tree.getSelectionCount() != 1) { + return null; + } + TreeItem item = tree.getSelection()[0]; + if (!(item.getData() instanceof Execution)) { + return null; + } + TreeItem parentItem = item.getParentItem(); + if (parentItem != null && parentItem.getData() instanceof ExecutionInfoLocation location) { + return location; + } + return null; + } + + /** + * Shows a popup menu under the copy toolbar button so the user can choose what to copy: full + * filename path, project-relative filename, on-disk storage path (file locations only), or UUID. + */ + @GuiToolbarElement( + root = GUI_PLUGIN_TOOLBAR_PARENT_ID, + id = TOOLBAR_ITEM_COPY_FILENAME, + toolTip = "i18n::ExecutionPerspective.ToolbarElement.Copy.Tooltip", + image = "ui/images/copy.svg") + public void copyToClipboard() { + Execution execution = getSelectedExecution(); + if (execution == null) { + return; + } + + String filename = execution.getFilename(); + String relativeFilename = toProjectRelativePath(filename); + String storageRoot = resolveStorageRootPath(getSelectedExecutionLocation()); + String storagePath = resolveStoragePath(getSelectedExecutionLocation(), execution); + String uuid = execution.getId(); + + Menu menu = new Menu(getShell(), SWT.POP_UP); + + addCopyMenuItem( + menu, + BaseMessages.getString(PKG, "ExecutionPerspective.CopyMenu.Filename"), + filename, + StringUtils.isNotEmpty(filename)); + addCopyMenuItem( + menu, + BaseMessages.getString(PKG, "ExecutionPerspective.CopyMenu.RelativeFilename"), + relativeFilename, + StringUtils.isNotEmpty(relativeFilename)); + addCopyMenuItem( + menu, + BaseMessages.getString(PKG, "ExecutionPerspective.CopyMenu.StorageRoot"), + storageRoot, + StringUtils.isNotEmpty(storageRoot)); + addCopyMenuItem( + menu, + BaseMessages.getString(PKG, "ExecutionPerspective.CopyMenu.StoragePath"), + storagePath, + StringUtils.isNotEmpty(storagePath)); + addCopyMenuItem( + menu, + BaseMessages.getString(PKG, "ExecutionPerspective.CopyMenu.Uuid"), + uuid, + StringUtils.isNotEmpty(uuid)); + + positionCopyMenu(menu); + menu.addListener(SWT.Hide, event -> menu.getDisplay().asyncExec(menu::dispose)); + menu.setVisible(true); + } + + private void addCopyMenuItem(Menu menu, String label, String value, boolean enabled) { + MenuItem item = new MenuItem(menu, SWT.PUSH); + item.setText(label); + item.setEnabled(enabled); + if (enabled && value != null) { + item.addListener(SWT.Selection, e -> GuiResource.getInstance().toClipboard(value)); + } + } + + private void positionCopyMenu(Menu menu) { + ToolItem toolItem = + toolBarWidgets != null ? toolBarWidgets.findToolItem(TOOLBAR_ITEM_COPY_FILENAME) : null; + if (toolItem != null && !toolItem.isDisposed() && toolItem.getParent() != null) { + ToolBar bar = toolItem.getParent(); + Rectangle bounds = toolItem.getBounds(); + menu.setLocation(bar.toDisplay(bounds.x, bounds.y + bounds.height)); + return; + } + Control control = + toolBarWidgets != null + ? toolBarWidgets.getControlForMenu(TOOLBAR_ITEM_COPY_FILENAME) + : null; + if (control != null && !control.isDisposed()) { + Rectangle bounds = control.getBounds(); + Point location = control.getParent().toDisplay(bounds.x, bounds.y + bounds.height); + menu.setLocation(location); + return; + } + menu.setLocation(getShell().getDisplay().getCursorLocation()); + } + + /** + * Expresses {@code path} relative to {@code PROJECT_HOME} as {@code ${PROJECT_HOME}/…} when + * possible. Returns {@code null} when the path cannot be relativized. + */ + private String toProjectRelativePath(String path) { + if (StringUtils.isEmpty(path)) { + return null; + } + // Already project-relative + if (path.startsWith(Const.VAR_PROJECT_HOME) || path.startsWith("${PROJECT_HOME}")) { + return path; + } + String projectHome = hopGui.getVariables().resolve(Const.VAR_PROJECT_HOME); + if (StringUtils.isEmpty(projectHome) || Const.VAR_PROJECT_HOME.equals(projectHome)) { + return null; + } + // Normalize trailing separators on project home for prefix matching + String home = projectHome; + while (home.endsWith("/") || home.endsWith("\\")) { + home = home.substring(0, home.length() - 1); + } + if (path.startsWith(home + "/") || path.startsWith(home + "\\") || path.equals(home)) { + String rel = path.substring(home.length()); + return Const.VAR_PROJECT_HOME + + (rel.isEmpty() || rel.startsWith("/") || rel.startsWith("\\") + ? rel.replace('\\', '/') + : "/" + rel.replace('\\', '/')); + } + return null; + } + + /** + * Builds the on-disk storage path for file-based execution locations, or {@code null} when the + * location type does not store execution information as files/folders. + */ + private String resolveStorageRootPath(ExecutionInfoLocation location) { + if (location == null) { + return null; + } + IExecutionInfoLocation iLocation = location.getExecutionInfoLocation(); + if (iLocation instanceof FileExecutionInfoLocation fileLocation) { + String root = hopGui.getVariables().resolve(fileLocation.getRootFolder()); + if (StringUtils.isEmpty(root)) { + return null; + } + return root; + } + if (iLocation instanceof CachingFileExecutionInfoLocation cachingLocation) { + String root = cachingLocation.getActualRootFolder(); + if (StringUtils.isEmpty(root)) { + root = hopGui.getVariables().resolve(cachingLocation.getRootFolder()); + } + if (StringUtils.isEmpty(root)) { + return null; + } + return root; + } + return null; + } + + /** + * Builds the on-disk storage path for file-based execution locations, or {@code null} when the + * location type does not store execution information as files/folders. + */ + private String resolveStoragePath(ExecutionInfoLocation location, Execution execution) { + String root = resolveStorageRootPath(location); + if (StringUtils.isEmpty(root)) { + return null; + } + IExecutionInfoLocation iLocation = location.getExecutionInfoLocation(); + if (iLocation instanceof FileExecutionInfoLocation fileLocation) { + if (root.endsWith("/") || root.endsWith("\\")) { + return root + execution.getId(); + } + return root + "/" + execution.getId(); + } + if (iLocation instanceof CachingFileExecutionInfoLocation cachingLocation) { + // Match CacheEntry.calculateFilename: root + id + ".json" + if (root.endsWith("/") || root.endsWith("\\")) { + return root + execution.getId() + ".json"; + } + if (root.contains("://") || root.startsWith("file:")) { + return root + "/" + execution.getId() + ".json"; + } + return root + Const.FILE_SEPARATOR + execution.getId() + ".json"; + } + return null; + } + @GuiToolbarElement( root = GUI_PLUGIN_TOOLBAR_PARENT_ID, id = TOOLBAR_ITEM_DELETE, @@ -1181,6 +1475,12 @@ public void closeTab(CTabFolderEvent event, CTabItem tabItem) { if (isRemoved) { hopGui.auditDelegate.writeLastOpenFiles(); + // Keep the per-project open-tabs audit in sync when the user closes a single tab. + // Skip during closeAllTabs so a prior saveState() for project switch is not wiped. + // + if (!closingAllTabs) { + saveState(); + } } if (!isRemoved && event != null) { event.doit = false; @@ -1193,6 +1493,33 @@ public void closeTab(CTabFolderEvent event, CTabItem tabItem) { } } + /** + * Close every open execution viewer tab. Does not write audit state — callers that need to + * remember tabs (project switch) must call {@link #saveState()} first. + */ + public void closeAllTabs() { + if (!isInitialized() || tabFolder == null || tabFolder.isDisposed()) { + return; + } + + closingAllTabs = true; + try { + // Copy the list — closeTab disposes items and mutates the folder + // + List items = new ArrayList<>(); + for (CTabItem item : tabFolder.getItems()) { + items.add(item); + } + for (CTabItem item : items) { + if (!item.isDisposed()) { + closeTab(null, item); + } + } + } finally { + closingAllTabs = false; + } + } + @Override public CTabFolder getTabFolder() { return tabFolder; @@ -1206,30 +1533,52 @@ public void saveState() { return; } try { + String namespace = HopNamespace.getNamespace(); + String activeKey = ""; + IExecutionViewer activeViewer = getActiveViewer(); + List openTabs = new ArrayList<>(); + + if (tabFolder != null && !tabFolder.isDisposed()) { + for (CTabItem item : tabFolder.getItems()) { + if (item.isDisposed()) { + continue; + } + Object data = item.getData(); + if (data instanceof BaseExecutionViewer baseViewer) { + String locationName = Const.NVL(baseViewer.getLocationName(), ""); + String executionId = + baseViewer.getExecution() != null + ? Const.NVL(baseViewer.getExecution().getId(), "") + : ""; + if (StringUtils.isEmpty(locationName) || StringUtils.isEmpty(executionId)) { + continue; + } + String tabKey = toTabKey(locationName, executionId); + openTabs.add(tabKey); + if (baseViewer == activeViewer) { + activeKey = tabKey; + } + } + } + } + + Map toolbarProps = new HashMap<>(); + toolbarProps.put(AUDIT_ONLY_PARENTS, onlyShowingParents); + toolbarProps.put(AUDIT_ONLY_FAILED, onlyShowingFailed); + toolbarProps.put(AUDIT_ONLY_RUNNING, onlyShowingRunning); + toolbarProps.put(AUDIT_ONLY_FINISHED, onlyShowingFinished); + toolbarProps.put(AUDIT_ONLY_WORKFLOWS, onlyShowingWorkflows); + toolbarProps.put(AUDIT_ONLY_PIPELINES, onlyShowingPipelines); + toolbarProps.put(AUDIT_FILTER_TEXT, Const.NVL(filterText, "")); + toolbarProps.put(AUDIT_TIME_FILTER, timeFilter.getCode()); + toolbarProps.put(AUDIT_ACTIVE_TAB, activeKey); + AuditStateMap stateMap = new AuditStateMap(); - stateMap.add( - new AuditState( - AUDIT_EXECUTION_TOOLBAR, - Map.of( - AUDIT_ONLY_PARENTS, - onlyShowingParents, - AUDIT_ONLY_FAILED, - onlyShowingFailed, - AUDIT_ONLY_RUNNING, - onlyShowingRunning, - AUDIT_ONLY_FINISHED, - onlyShowingFinished, - AUDIT_ONLY_WORKFLOWS, - onlyShowingWorkflows, - AUDIT_ONLY_PIPELINES, - onlyShowingPipelines, - AUDIT_FILTER_TEXT, - Const.NVL(filterText, ""), - AUDIT_TIME_FILTER, - timeFilter.getCode()))); + stateMap.add(new AuditState(AUDIT_EXECUTION_TOOLBAR, toolbarProps)); + AuditManager.getActive().saveAuditStateMap(namespace, EXECUTION_AUDIT_TYPE, stateMap); AuditManager.getActive() - .saveAuditStateMap(HopNamespace.getNamespace(), EXECUTION_AUDIT_TYPE, stateMap); + .storeList(namespace, EXECUTION_TABS_AUDIT_TYPE, new AuditList(openTabs)); } catch (Exception e) { hopGui.getLog().logError("Error saving execution perspective state", e); } @@ -1243,9 +1592,9 @@ public void restoreState() { return; } try { + String namespace = HopNamespace.getNamespace(); AuditStateMap stateMap = - AuditManager.getActive() - .loadAuditStateMap(HopNamespace.getNamespace(), EXECUTION_AUDIT_TYPE); + AuditManager.getActive().loadAuditStateMap(namespace, EXECUTION_AUDIT_TYPE); AuditState toolbarState = stateMap.get(AUDIT_EXECUTION_TOOLBAR); if (toolbarState == null) { toolbarState = new AuditState(); @@ -1259,14 +1608,152 @@ public void restoreState() { filterText = toolbarState.extractString(AUDIT_FILTER_TEXT, ""); String timeFilterName = toolbarState.extractString(AUDIT_TIME_FILTER, ""); timeFilter = IEnumHasCode.lookupCode(LastPeriod.class, timeFilterName, LastPeriod.ONE_HOUR); + String activeKey = toolbarState.extractString(AUDIT_ACTIVE_TAB, ""); updateGui(); refresh(); + + // Re-open execution viewer tabs that were open last time this project was used. + // Missing locations/executions fail silently — they are not important enough to block + // project activation. + // + restoreOpenTabs(namespace, activeKey); + } catch (Exception e) { + hopGui.getLog().logError("Error restoring execution perspective state", e); + } + } + + /** + * Restore previously open execution tabs for the given project namespace. Failures for individual + * tabs are logged and skipped. + */ + private void restoreOpenTabs(String namespace, String activeKey) { + if (tabFolder == null || tabFolder.isDisposed()) { + return; + } + + AuditList auditList; + try { + auditList = AuditManager.getActive().retrieveList(namespace, EXECUTION_TABS_AUDIT_TYPE); + } catch (Exception e) { + hopGui.getLog().logError("Error loading open execution tabs for project restore", e); + return; + } + if (auditList == null || auditList.getNames() == null || auditList.getNames().isEmpty()) { + return; + } + + restoringTabs = true; + IExecutionViewer activeViewer = null; + try { + for (String tabKey : auditList.getNames()) { + if (StringUtils.isEmpty(tabKey)) { + continue; + } + try { + String[] parts = tabKey.split(TAB_KEY_DELIMITER, 2); + if (parts.length < 2 || StringUtils.isEmpty(parts[0]) || StringUtils.isEmpty(parts[1])) { + hopGui + .getLog() + .logDebug("Skipping invalid execution tab key during restore: " + tabKey); + continue; + } + String locationName = parts[0]; + String executionId = parts[1]; + + ExecutionInfoLocation location = ensureLocationInitialized(locationName); + if (location == null) { + hopGui + .getLog() + .logDebug( + "Skipping restore of execution tab: location '" + + locationName + + "' not available"); + continue; + } + + IExecutionInfoLocation iLocation = location.getExecutionInfoLocation(); + Execution execution = iLocation.getExecution(executionId); + if (execution == null) { + hopGui + .getLog() + .logDebug( + "Skipping restore of execution tab: execution '" + + executionId + + "' not found in location '" + + locationName + + "'"); + continue; + } + + ExecutionState executionState = iLocation.getExecutionState(executionId); + createExecutionViewer(locationName, execution, executionState); + + if (tabKey.equals(activeKey)) { + activeViewer = findViewer(execution.getId(), execution.getName()); + } + } catch (Exception e) { + // Missing or corrupt execution information is not important enough to surface. + // + hopGui + .getLog() + .logDebug("Skipping restore of execution tab '" + tabKey + "': " + e.getMessage()); + } + } + + if (activeViewer != null) { + setActiveViewer(activeViewer); + } + } finally { + restoringTabs = false; + } + } + + /** + * Ensure the named execution information location is initialized and present in {@link + * #locationMap}. Does not require the perspective to be active (unlike {@link #refresh()}). + * + * @param locationName metadata name of the location + * @return the location, or null if it cannot be loaded/initialized + */ + private ExecutionInfoLocation ensureLocationInitialized(String locationName) { + if (StringUtils.isEmpty(locationName) || locationMap == null) { + return null; + } + + ExecutionInfoLocation location = locationMap.get(locationName); + if (location != null) { + return location; + } + + try { + IHopMetadataSerializer serializer = + hopGui.getMetadataProvider().getSerializer(ExecutionInfoLocation.class); + if (!serializer.exists(locationName)) { + return null; + } + location = serializer.load(locationName); + if (location == null || location.getExecutionInfoLocation() == null) { + return null; + } + location + .getExecutionInfoLocation() + .initialize(hopGui.getVariables(), hopGui.getMetadataProvider()); + locationMap.put(locationName, location); + return location; } catch (Exception e) { - hopGui.getLog().logError("Error restoring explorer perspective state", e); + hopGui + .getLog() + .logDebug( + "Could not initialize execution location '" + locationName + "' for tab restore", e); + return null; } } + private static String toTabKey(String locationName, String executionId) { + return locationName + TAB_KEY_DELIMITER + executionId; + } + public ExecutionInfoLocation lookupLocation(String locationName) { ExecutionInfoLocation location = locationMap.get(locationName); if (location == null) { diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/execution/PipelineExecutionViewer.java b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/execution/PipelineExecutionViewer.java index 14e8289aa3c..4c6a64cecd9 100644 --- a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/execution/PipelineExecutionViewer.java +++ b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/execution/PipelineExecutionViewer.java @@ -54,6 +54,7 @@ import org.apache.hop.core.row.IValueMeta; import org.apache.hop.core.row.RowBuffer; import org.apache.hop.core.row.RowMetaBuilder; +import org.apache.hop.core.util.Utils; import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.variables.Variables; import org.apache.hop.core.xml.XmlHandler; @@ -63,6 +64,7 @@ import org.apache.hop.execution.ExecutionDataSetMeta; import org.apache.hop.execution.ExecutionInfoLocation; import org.apache.hop.execution.ExecutionState; +import org.apache.hop.execution.ExecutionStateBuilder; import org.apache.hop.execution.ExecutionStateComponentMetrics; import org.apache.hop.execution.ExecutionType; import org.apache.hop.execution.IExecutionInfoLocation; @@ -331,6 +333,7 @@ private void refreshStatus() { infoView.add("Registration", formatDate(execution.getRegistrationDate())); infoView.add("Start", formatDate(execution.getExecutionStartDate())); infoView.add("End", formatDate(executionState.getExecutionEndDate())); + infoView.add("Duration", formatExecutionDuration(execution, executionState)); infoView.add("Type", executionState.getExecutionType().name()); infoView.add("Status", statusDescription); infoView.add("Status Last updated", formatDate(executionState.getUpdateTime())); @@ -472,7 +475,7 @@ private void refreshMetrics() { // We display everything that makes sense: // name, copy, inits, input, read, written, output, updated, rejected, errors, buffer in, - // buffer out, data volume, data volume in, data volume out + // buffer out, data volume, data volume in, data volume out, duration, speed // List columns = new ArrayList<>(); columns.add(new ColumnInfo("Name", ColumnInfo.COLUMN_TYPE_TEXT, false, true)); @@ -491,6 +494,9 @@ private void refreshMetrics() { addColumn(columns, indexMap, metricNames, Pipeline.METRIC_DATA_VOLUME); addColumn(columns, indexMap, metricNames, Pipeline.METRIC_DATA_VOLUME_IN); addColumn(columns, indexMap, metricNames, Pipeline.METRIC_DATA_VOLUME_OUT); + // Always reserve Duration / Speed columns (blank when not recorded for a component). + addNamedColumn(columns, indexMap, ExecutionStateBuilder.METRIC_HEADER_DURATION, "Duration"); + addNamedColumn(columns, indexMap, ExecutionStateBuilder.METRIC_HEADER_SPEED, "Speed (r/s)"); metricsView = new TableView( @@ -512,7 +518,11 @@ private void refreshMetrics() { Integer index = indexMap.get(metricHeader); Long value = metrics.getMetrics().get(metricHeader); if (value != null && index != null && index > 1 && index <= columns.size()) { - item.setText(index, value.toString()); + if (ExecutionStateBuilder.METRIC_HEADER_DURATION.equals(metricHeader)) { + item.setText(index, Utils.getDurationHMS(value / 1000.0)); + } else { + item.setText(index, value.toString()); + } } } } @@ -540,6 +550,12 @@ private void addColumn( } } + private void addNamedColumn( + List columns, Map indexMap, String key, String displayHeader) { + columns.add(new ColumnInfo(displayHeader, ColumnInfo.COLUMN_TYPE_TEXT, true, true)); + indexMap.put(key, columns.size()); + } + /** An entry is selected in the data list. Show the corresponding rows. */ private void showDataRows() { int[] weights = dataSash.getWeights(); diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/execution/WorkflowExecutionViewer.java b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/execution/WorkflowExecutionViewer.java index 428a2ba7067..7cef9e27f7e 100644 --- a/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/execution/WorkflowExecutionViewer.java +++ b/ui/src/main/java/org/apache/hop/ui/hopgui/perspective/execution/WorkflowExecutionViewer.java @@ -318,6 +318,8 @@ private void refreshStatus() { infoView.add("Parent ID", execution.getParentId()); infoView.add("Registration", formatDate(execution.getRegistrationDate())); infoView.add("Start", formatDate(execution.getExecutionStartDate())); + infoView.add("End", formatDate(executionState.getExecutionEndDate())); + infoView.add("Duration", formatExecutionDuration(execution, executionState)); infoView.add("Type", executionState.getExecutionType().name()); infoView.add("Status", statusDescription); infoView.add("Status Last updated", formatDate(executionState.getUpdateTime())); diff --git a/ui/src/main/java/org/apache/hop/ui/hopgui/shared/BaseExecutionViewer.java b/ui/src/main/java/org/apache/hop/ui/hopgui/shared/BaseExecutionViewer.java index e71411e76f6..316ce7327fa 100644 --- a/ui/src/main/java/org/apache/hop/ui/hopgui/shared/BaseExecutionViewer.java +++ b/ui/src/main/java/org/apache/hop/ui/hopgui/shared/BaseExecutionViewer.java @@ -32,6 +32,7 @@ import org.apache.hop.core.gui.DPoint; import org.apache.hop.core.gui.Point; import org.apache.hop.core.metadata.SerializableMetadataProvider; +import org.apache.hop.core.util.Utils; import org.apache.hop.core.variables.IVariables; import org.apache.hop.core.variables.Variables; import org.apache.hop.execution.Execution; @@ -141,6 +142,33 @@ protected String formatDate(Date date) { return new SimpleDateFormat("yyyy/MM/dd HH:mm:ss").format(date); } + /** + * Formats pipeline/workflow execution duration from start to end (or last update while still + * running). + * + * @return human-readable duration, or empty string when start is unknown + */ + protected String formatExecutionDuration(Execution execution, ExecutionState executionState) { + if (execution == null || execution.getExecutionStartDate() == null) { + return ""; + } + Date end = null; + if (executionState != null) { + end = executionState.getExecutionEndDate(); + if (end == null) { + end = executionState.getUpdateTime(); + } + } + if (end == null) { + return ""; + } + long durationMs = end.getTime() - execution.getExecutionStartDate().getTime(); + if (durationMs < 0) { + return ""; + } + return Utils.getDurationHMS(durationMs / 1000.0); + } + public abstract void drillDownOnLocation(Point location); @Override diff --git a/ui/src/main/resources/org/apache/hop/ui/core/database/messages/messages_en_US.properties b/ui/src/main/resources/org/apache/hop/ui/core/database/messages/messages_en_US.properties index 563576b233b..36a0fc6b5e9 100644 --- a/ui/src/main/resources/org/apache/hop/ui/core/database/messages/messages_en_US.properties +++ b/ui/src/main/resources/org/apache/hop/ui/core/database/messages/messages_en_US.properties @@ -18,6 +18,24 @@ DatabaseDialog.AdvancedTab.title=Advanced DatabaseDialog.button.DownloadDriver=Download driver DatabaseDialog.button.Explore=Explore +DatabaseDialog.button.GenerateVariables=Generate variables +DatabaseDialog.GenerateVariables.NameRequired.Title=Connection name required +DatabaseDialog.GenerateVariables.NameRequired.Message=Please enter a connection name before generating variables. The name is used as a prefix (for example EDW_HOSTNAME). +DatabaseDialog.GenerateVariables.DialogMessage=Proposed environment variables for database connection ''{0}''. Review names, values and descriptions. Values can be left empty for a DTAP template checked into version control. +DatabaseDialog.GenerateVariables.CreateConfig.Title=Environment configuration file +DatabaseDialog.GenerateVariables.CreateConfig.Message=Create or update an environment configuration JSON file with these variables?\n\nThis is useful as a template for DTAP environments (checked into version control; environment-specific values are typically deployed separately). +DatabaseDialog.GenerateVariables.FileFilter.Json=Config JSON files +DatabaseDialog.GenerateVariables.FileFilter.All=All files +DatabaseDialog.GenerateVariables.ConfigSaved.Title=Configuration file saved +DatabaseDialog.GenerateVariables.ConfigSaved.Message=Variables were written to:\n{0}\n\nAdd this file to a Hop environment''s configuration files list so the variables are loaded at runtime. +DatabaseDialog.GenerateVariables.ConfigError.Title=Error +DatabaseDialog.GenerateVariables.ConfigError.Message=Unable to save the environment configuration file +DatabaseDialog.GenerateVariables.Description.Hostname=Hostname for database connection {0} +DatabaseDialog.GenerateVariables.Description.Port=Port for database connection {0} +DatabaseDialog.GenerateVariables.Description.Database=Database name for database connection {0} +DatabaseDialog.GenerateVariables.Description.Username=Username for database connection {0} +DatabaseDialog.GenerateVariables.Description.Password=Password for database connection {0} +DatabaseDialog.GenerateVariables.Description.Url=Manual JDBC URL for database connection {0} JdbcDriverDownloadDialog.Shell.Title=Download JDBC driver JdbcDriverDownloadDialog.Intro.Restricted=Download the {0} JDBC driver from Maven Central. Accept the vendor license below to continue. JdbcDriverDownloadDialog.Intro.Open=Download the {0} JDBC driver from Maven Central. diff --git a/ui/src/main/resources/org/apache/hop/ui/core/widget/messages/messages_en_US.properties b/ui/src/main/resources/org/apache/hop/ui/core/widget/messages/messages_en_US.properties index 323a8ec2ba4..d8ac9eb19b0 100644 --- a/ui/src/main/resources/org/apache/hop/ui/core/widget/messages/messages_en_US.properties +++ b/ui/src/main/resources/org/apache/hop/ui/core/widget/messages/messages_en_US.properties @@ -95,6 +95,7 @@ TableView.ToolBarWidget.SetFilter.ToolTip=Select rows using a filter TableView.ToolBarWidget.UndoAction.ToolTip=Undo the last action TextVar.InternalVariable.Message=This is an internal variable. TextVar.tooltip.InsertVariable=Enter CTRL-SPACE to select a variable to insert +TextVar.tooltip.ExpandedInteger=This field accepts expanded integer values:\n- Grouping: 100,000,000 or 100.000.000 or 100_000_000\n- Suffixes: 100k, 100m, 1g, 1b (\u00d71,000 / \u00d71,000,000 / \u00d71,000,000,000)\n- Fractional: 1.5m\n- Scientific: 1e8, 1x10^8, 10^8\nVariables ($'{'...'}') are resolved before parsing. TextVar.VariableValue.Message=The value of variable ''{0}'' is \: \n\n\t{1}\n\n WidgetDialog.Styled.Copy=Copy\tCtrl+C WidgetDialog.Styled.Cut=Cut\tCtrl+X diff --git a/ui/src/main/resources/org/apache/hop/ui/hopgui/perspective/execution/messages/messages_en_US.properties b/ui/src/main/resources/org/apache/hop/ui/hopgui/perspective/execution/messages/messages_en_US.properties index c27c4de9393..faecfc7af58 100644 --- a/ui/src/main/resources/org/apache/hop/ui/hopgui/perspective/execution/messages/messages_en_US.properties +++ b/ui/src/main/resources/org/apache/hop/ui/hopgui/perspective/execution/messages/messages_en_US.properties @@ -17,8 +17,17 @@ ExecutionPerspective.Description=Shows information about executing pipelines and workflows ExecutionPerspective.Name=Execution information +ExecutionPerspective.Tree.ConfigureLocations=Please configure one or more locations. +ExecutionPerspective.Tree.ConfigureLocations.Tooltip=Double click to create an execution information location.\nConfigure a pipeline or workflow run configuration with it to see information appear here. +ExecutionPerspective.CreateLocation.Error.Message=Error creating a new Execution Information Location ExecutionPerspective.Refresh.Error.Header=Error ExecutionPerspective.Refresh.Error.Message=There was an error refreshing execution information: +ExecutionPerspective.ToolbarElement.Copy.Tooltip=Copy path or ID to clipboard +ExecutionPerspective.CopyMenu.Filename=Copy filename path +ExecutionPerspective.CopyMenu.RelativeFilename=Copy relative filename path +ExecutionPerspective.CopyMenu.StorageRoot=Copy storage root path on disk +ExecutionPerspective.CopyMenu.StoragePath=Copy storage path on disk +ExecutionPerspective.CopyMenu.Uuid=Copy ID ExecutionPerspective.ToolbarElement.Delete.Tooltip=Delete ExecutionPerspective.ToolbarElement.Refresh.Tooltip=Refresh information ExecutionPerspective.ToolbarElement.ForceRefresh.Tooltip=Force refresh information (clear caches) diff --git a/ui/src/main/resources/ui/images/hash.svg b/ui/src/main/resources/ui/images/hash.svg new file mode 100644 index 00000000000..f16ab252a4a --- /dev/null +++ b/ui/src/main/resources/ui/images/hash.svg @@ -0,0 +1,8 @@ + + + + + + + diff --git a/ui/src/test/java/org/apache/hop/ui/core/database/DatabaseConnectionVariablesHelperTest.java b/ui/src/test/java/org/apache/hop/ui/core/database/DatabaseConnectionVariablesHelperTest.java new file mode 100644 index 00000000000..2229b9eda49 --- /dev/null +++ b/ui/src/test/java/org/apache/hop/ui/core/database/DatabaseConnectionVariablesHelperTest.java @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.ui.core.database; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.hop.core.variables.DescribedVariable; +import org.junit.jupiter.api.Test; + +class DatabaseConnectionVariablesHelperTest { + + @Test + void sanitizeConnectionNamePrefix() { + assertEquals("EDW", DatabaseConnectionVariablesHelper.sanitizeConnectionNamePrefix("EDW")); + assertEquals("MY_DB", DatabaseConnectionVariablesHelper.sanitizeConnectionNamePrefix("My DB")); + assertEquals( + "MY_DB", DatabaseConnectionVariablesHelper.sanitizeConnectionNamePrefix(" my-db ")); + assertEquals("", DatabaseConnectionVariablesHelper.sanitizeConnectionNamePrefix("")); + assertEquals("", DatabaseConnectionVariablesHelper.sanitizeConnectionNamePrefix(" ")); + assertEquals( + "A_B_C", DatabaseConnectionVariablesHelper.sanitizeConnectionNamePrefix("a__b---c")); + } + + @Test + void expressionFor() { + assertEquals( + "${EDW_HOSTNAME}", DatabaseConnectionVariablesHelper.expressionFor("EDW_HOSTNAME")); + } + + @Test + void buildProposedVariablesIncludesStandardFields() { + Map descriptions = new HashMap<>(); + descriptions.put(DatabaseConnectionVariablesHelper.SUFFIX_HOSTNAME, "Host"); + descriptions.put(DatabaseConnectionVariablesHelper.SUFFIX_PORT, "Port"); + descriptions.put(DatabaseConnectionVariablesHelper.SUFFIX_DATABASE, "Database"); + descriptions.put(DatabaseConnectionVariablesHelper.SUFFIX_USERNAME, "User"); + descriptions.put(DatabaseConnectionVariablesHelper.SUFFIX_PASSWORD, "Password"); + + List variables = + DatabaseConnectionVariablesHelper.buildProposedVariables( + "EDW", + "localhost", + "5432", + "warehouse", + "alice", + "secret", + null, + true, + true, + descriptions); + + assertEquals(5, variables.size()); + assertEquals("EDW_HOSTNAME", variables.get(0).getName()); + assertEquals("localhost", variables.get(0).getValue()); + assertEquals("Host", variables.get(0).getDescription()); + assertEquals("EDW_PORT", variables.get(1).getName()); + assertEquals("5432", variables.get(1).getValue()); + assertEquals("EDW_DATABASE", variables.get(2).getName()); + assertEquals("warehouse", variables.get(2).getValue()); + assertEquals("EDW_USERNAME", variables.get(3).getName()); + assertEquals("alice", variables.get(3).getValue()); + assertEquals("EDW_PASSWORD", variables.get(4).getName()); + assertEquals("secret", variables.get(4).getValue()); + } + + @Test + void buildProposedVariablesIncludesUrlOnlyWhenFilled() { + List withoutUrl = + DatabaseConnectionVariablesHelper.buildProposedVariables( + "EDW", "h", "1", "d", "u", "p", " ", true, true, null); + assertFalse( + withoutUrl.stream() + .anyMatch(v -> v.getName().endsWith(DatabaseConnectionVariablesHelper.SUFFIX_URL))); + + List withUrl = + DatabaseConnectionVariablesHelper.buildProposedVariables( + "EDW", "h", "1", "d", "u", "p", "jdbc:test://x", true, true, null); + assertTrue( + withUrl.stream() + .anyMatch(v -> "EDW_URL".equals(v.getName()) && "jdbc:test://x".equals(v.getValue()))); + } + + @Test + void buildProposedVariablesSkipsUsernamePasswordWhenExcluded() { + List variables = + DatabaseConnectionVariablesHelper.buildProposedVariables( + "EDW", "h", "1", "d", "u", "p", null, false, false, null); + assertEquals(3, variables.size()); + assertFalse( + variables.stream() + .anyMatch( + v -> v.getName().endsWith(DatabaseConnectionVariablesHelper.SUFFIX_USERNAME))); + assertFalse( + variables.stream() + .anyMatch( + v -> v.getName().endsWith(DatabaseConnectionVariablesHelper.SUFFIX_PASSWORD))); + } + + @Test + void findVariableNamesBySuffix() { + List variables = + List.of( + new DescribedVariable("EDW_HOSTNAME", "a", ""), + new DescribedVariable("CUSTOM_PORT", "b", ""), + new DescribedVariable("other", "c", ""), + new DescribedVariable("EDW_HOSTNAME", "ignored", "")); + + Map bySuffix = + DatabaseConnectionVariablesHelper.findVariableNamesBySuffix(variables); + assertEquals("EDW_HOSTNAME", bySuffix.get(DatabaseConnectionVariablesHelper.SUFFIX_HOSTNAME)); + assertEquals("CUSTOM_PORT", bySuffix.get(DatabaseConnectionVariablesHelper.SUFFIX_PORT)); + assertFalse(bySuffix.containsKey(DatabaseConnectionVariablesHelper.SUFFIX_DATABASE)); + } + + @Test + void defaultConfigFilename() { + assertEquals("EDW-config.json", DatabaseConnectionVariablesHelper.defaultConfigFilename("EDW")); + assertEquals( + "database-config.json", DatabaseConnectionVariablesHelper.defaultConfigFilename("")); + } +} diff --git a/ui/src/test/java/org/apache/hop/ui/hopgui/perspective/execution/ExecutionPerspectiveDisabledTest.java b/ui/src/test/java/org/apache/hop/ui/hopgui/perspective/execution/ExecutionPerspectiveDisabledTest.java index acda548165b..bc80b89ca5d 100644 --- a/ui/src/test/java/org/apache/hop/ui/hopgui/perspective/execution/ExecutionPerspectiveDisabledTest.java +++ b/ui/src/test/java/org/apache/hop/ui/hopgui/perspective/execution/ExecutionPerspectiveDisabledTest.java @@ -73,6 +73,12 @@ void closingAProjectDoesNotFail() { assertDoesNotThrow(() -> disabledPerspective.saveState()); } + @Test + void closingAllTabsDoesNotFail() { + // HopGuiFileDelegate#closeAllFiles closes execution tabs after file tabs. + assertDoesNotThrow(() -> disabledPerspective.closeAllTabs()); + } + @Test void theKeyboardShortcutDoesNotFail() { // The Ctrl-Shift-I shortcut on activate() is registered even though the perspective is not.