From f54fb0466dae71e8a26c8b8edd1e4f2b15c01e82 Mon Sep 17 00:00:00 2001 From: Umesh Patil Date: Mon, 20 Jul 2026 23:40:24 +0530 Subject: [PATCH 1/3] ATLAS-5313: Implement glossary-scoped download APIs also added GlossaryExportParameters, search models, collector/filter/writer pipeline etc. --- hbase-shaded-client-fixed/pom.xml | 22 +- .../org/apache/atlas/AtlasConfiguration.java | 2 + .../glossary/GlossaryExportParameters.java | 151 +++++++++ .../model/glossary/GlossaryExportRow.java | 295 ++++++++++++++++ .../glossary/GlossarySearchParameters.java | 151 +++++++++ .../model/glossary/GlossarySearchResult.java | 290 ++++++++++++++++ pom.xml | 14 + repository/pom.xml | 27 +- .../glossary/GlossaryExportDataCollector.java | 156 +++++++++ .../glossary/GlossaryExportDownloadTask.java | 141 ++++++++ .../GlossaryExportDownloadTaskFactory.java | 149 +++++++++ .../atlas/glossary/GlossaryExportFilter.java | 81 +++++ .../GlossaryExportParameterMapper.java | 122 +++++++ .../atlas/glossary/GlossaryExportUtils.java | 209 ++++++++++++ .../atlas/glossary/GlossaryExportWriter.java | 179 ++++++++++ .../atlas/glossary/GlossarySearchBuilder.java | 316 ++++++++++++++++++ .../atlas/glossary/GlossaryService.java | 117 ++++++- .../repository/graph/AtlasGraphProvider.java | 2 +- .../glossary/GlossaryExportFilterTest.java | 139 ++++++++ .../glossary/GlossaryExportUtilsTest.java | 84 +++++ .../glossary/GlossaryExportWriterTest.java | 78 +++++ .../glossary/GlossarySearchBuilderTest.java | 129 +++++++ .../atlas/glossary/GlossaryServiceTest.java | 2 +- server-api/pom.xml | 11 +- webapp/pom.xml | 6 + .../apache/atlas/web/rest/GlossaryREST.java | 147 ++++++++ .../atlas/web/rest/GlossaryRESTTest.java | 70 ++++ 27 files changed, 3058 insertions(+), 32 deletions(-) create mode 100644 intg/src/main/java/org/apache/atlas/model/glossary/GlossaryExportParameters.java create mode 100644 intg/src/main/java/org/apache/atlas/model/glossary/GlossaryExportRow.java create mode 100644 intg/src/main/java/org/apache/atlas/model/glossary/GlossarySearchParameters.java create mode 100644 intg/src/main/java/org/apache/atlas/model/glossary/GlossarySearchResult.java create mode 100644 repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDataCollector.java create mode 100644 repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDownloadTask.java create mode 100644 repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDownloadTaskFactory.java create mode 100644 repository/src/main/java/org/apache/atlas/glossary/GlossaryExportFilter.java create mode 100644 repository/src/main/java/org/apache/atlas/glossary/GlossaryExportParameterMapper.java create mode 100644 repository/src/main/java/org/apache/atlas/glossary/GlossaryExportUtils.java create mode 100644 repository/src/main/java/org/apache/atlas/glossary/GlossaryExportWriter.java create mode 100644 repository/src/main/java/org/apache/atlas/glossary/GlossarySearchBuilder.java create mode 100644 repository/src/test/java/org/apache/atlas/glossary/GlossaryExportFilterTest.java create mode 100644 repository/src/test/java/org/apache/atlas/glossary/GlossaryExportUtilsTest.java create mode 100644 repository/src/test/java/org/apache/atlas/glossary/GlossaryExportWriterTest.java create mode 100644 repository/src/test/java/org/apache/atlas/glossary/GlossarySearchBuilderTest.java diff --git a/hbase-shaded-client-fixed/pom.xml b/hbase-shaded-client-fixed/pom.xml index 09e75bae4aa..6ffe4a84e50 100644 --- a/hbase-shaded-client-fixed/pom.xml +++ b/hbase-shaded-client-fixed/pom.xml @@ -73,28 +73,14 @@ - + org.apache.maven.plugins maven-jar-plugin 2.4 - - - default-jar - none - - - repack-fixed-hbase-shaded-client - - jar - - package - - ${project.build.directory}/unpacked - hbase-shaded-client-${hbase.version}-hadoop3-fixed - - - + + ${project.build.directory}/unpacked + diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 6ce7d8eb6df..27610285c69 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -68,6 +68,8 @@ public enum AtlasConfiguration { SEARCH_MAX_LIMIT("atlas.search.maxlimit", 10000), SEARCH_DEFAULT_LIMIT("atlas.search.defaultlimit", 100), + GLOSSARY_EXPORT_MAX_ROWS("atlas.glossary.export.max.rows", 5000), + CUSTOM_ATTRIBUTE_KEY_MAX_LENGTH("atlas.custom.attribute.key.max.length", 50), CUSTOM_ATTRIBUTE_VALUE_MAX_LENGTH("atlas.custom.attribute.value.max.length", 500), CUSTOM_ATTRIBUTE_KEY_SPECIAL_PREFIX("atlas.custom.attribute.special.prefix", ""), diff --git a/intg/src/main/java/org/apache/atlas/model/glossary/GlossaryExportParameters.java b/intg/src/main/java/org/apache/atlas/model/glossary/GlossaryExportParameters.java new file mode 100644 index 00000000000..09fd2d41c05 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/glossary/GlossaryExportParameters.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.glossary; + +import org.apache.atlas.SortOrder; +import org.apache.atlas.model.annotation.AtlasJSON; + +import java.io.Serializable; +import java.util.List; + +@AtlasJSON +public class GlossaryExportParameters implements Serializable { + private static final long serialVersionUID = 1L; + + public enum ExportFormat { + CSV, XLSX + } + + public enum ExportMode { + AUDIT, IMPORT_COMPATIBLE + } + + public enum RecordType { + ALL, TERM, CATEGORY + } + + private ExportFormat format; + private ExportMode mode; + private RecordType recordType; + private String glossaryGuid; + private List glossaryGuids; + private String statusContains; + private String classificationContains; + private String searchQuery; + private boolean excludeDeleted = true; + private String sortBy = "name"; + private SortOrder sortOrder = SortOrder.ASCENDING; + /** Optional shared timestamp token so CSV and XLSX exports correlate (yyyy-MM-dd_HH-mm-ss). */ + private String exportTimestamp; + + public ExportFormat getFormat() { + return format; + } + + public void setFormat(ExportFormat format) { + this.format = format; + } + + public ExportMode getMode() { + return mode != null ? mode : ExportMode.AUDIT; + } + + public void setMode(ExportMode mode) { + this.mode = mode; + } + + public RecordType getRecordType() { + return recordType != null ? recordType : RecordType.ALL; + } + + public void setRecordType(RecordType recordType) { + this.recordType = recordType; + } + + public String getGlossaryGuid() { + return glossaryGuid; + } + + public void setGlossaryGuid(String glossaryGuid) { + this.glossaryGuid = glossaryGuid; + } + + public List getGlossaryGuids() { + return glossaryGuids; + } + + public void setGlossaryGuids(List glossaryGuids) { + this.glossaryGuids = glossaryGuids; + } + + public String getStatusContains() { + return statusContains; + } + + public void setStatusContains(String statusContains) { + this.statusContains = statusContains; + } + + public String getClassificationContains() { + return classificationContains; + } + + public void setClassificationContains(String classificationContains) { + this.classificationContains = classificationContains; + } + + public String getSearchQuery() { + return searchQuery; + } + + public void setSearchQuery(String searchQuery) { + this.searchQuery = searchQuery; + } + + public boolean getExcludeDeleted() { + return excludeDeleted; + } + + public void setExcludeDeleted(boolean excludeDeleted) { + this.excludeDeleted = excludeDeleted; + } + + public String getSortBy() { + return sortBy; + } + + public void setSortBy(String sortBy) { + this.sortBy = sortBy; + } + + public SortOrder getSortOrder() { + return sortOrder; + } + + public void setSortOrder(SortOrder sortOrder) { + this.sortOrder = sortOrder; + } + + public String getExportTimestamp() { + return exportTimestamp; + } + + public void setExportTimestamp(String exportTimestamp) { + this.exportTimestamp = exportTimestamp; + } +} diff --git a/intg/src/main/java/org/apache/atlas/model/glossary/GlossaryExportRow.java b/intg/src/main/java/org/apache/atlas/model/glossary/GlossaryExportRow.java new file mode 100644 index 00000000000..88d3b3cb270 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/glossary/GlossaryExportRow.java @@ -0,0 +1,295 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.glossary; + +import org.apache.atlas.model.annotation.AtlasJSON; + +import java.io.Serializable; + +@AtlasJSON +public class GlossaryExportRow implements Serializable { + private static final long serialVersionUID = 1L; + + public enum RecordTypeKind { + TERM, CATEGORY + } + + private RecordTypeKind recordType; + private String guid; + private String glossaryGuid; + private String name; + private String glossaryName; + private String shortDescription; + private String longDescription; + private String status; + private String classifications; + private String customAttributes; + private String relatedCategoriesOrParent; + private String qualifiedName; + private String entityStatus; + + // Import-compatible relationship columns (terms only) + private String examples; + private String abbreviation; + private String usage; + private String translationTerms; + private String validValuesFor; + private String synonyms; + private String replacedBy; + private String validValues; + private String replacementTerms; + private String seeAlso; + private String translatedTerms; + private String isA; + private String antonyms; + private String classifies; + private String preferredToTerms; + private String preferredTerms; + + public RecordTypeKind getRecordType() { + return recordType; + } + + public void setRecordType(RecordTypeKind recordType) { + this.recordType = recordType; + } + + public String getGuid() { + return guid; + } + + public void setGuid(String guid) { + this.guid = guid; + } + + public String getGlossaryGuid() { + return glossaryGuid; + } + + public void setGlossaryGuid(String glossaryGuid) { + this.glossaryGuid = glossaryGuid; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getGlossaryName() { + return glossaryName; + } + + public void setGlossaryName(String glossaryName) { + this.glossaryName = glossaryName; + } + + public String getShortDescription() { + return shortDescription; + } + + public void setShortDescription(String shortDescription) { + this.shortDescription = shortDescription; + } + + public String getLongDescription() { + return longDescription; + } + + public void setLongDescription(String longDescription) { + this.longDescription = longDescription; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getClassifications() { + return classifications; + } + + public void setClassifications(String classifications) { + this.classifications = classifications; + } + + public String getCustomAttributes() { + return customAttributes; + } + + public void setCustomAttributes(String customAttributes) { + this.customAttributes = customAttributes; + } + + public String getRelatedCategoriesOrParent() { + return relatedCategoriesOrParent; + } + + public void setRelatedCategoriesOrParent(String relatedCategoriesOrParent) { + this.relatedCategoriesOrParent = relatedCategoriesOrParent; + } + + public String getQualifiedName() { + return qualifiedName; + } + + public void setQualifiedName(String qualifiedName) { + this.qualifiedName = qualifiedName; + } + + public String getEntityStatus() { + return entityStatus; + } + + public void setEntityStatus(String entityStatus) { + this.entityStatus = entityStatus; + } + + public String getExamples() { + return examples; + } + + public void setExamples(String examples) { + this.examples = examples; + } + + public String getAbbreviation() { + return abbreviation; + } + + public void setAbbreviation(String abbreviation) { + this.abbreviation = abbreviation; + } + + public String getUsage() { + return usage; + } + + public void setUsage(String usage) { + this.usage = usage; + } + + public String getTranslationTerms() { + return translationTerms; + } + + public void setTranslationTerms(String translationTerms) { + this.translationTerms = translationTerms; + } + + public String getValidValuesFor() { + return validValuesFor; + } + + public void setValidValuesFor(String validValuesFor) { + this.validValuesFor = validValuesFor; + } + + public String getSynonyms() { + return synonyms; + } + + public void setSynonyms(String synonyms) { + this.synonyms = synonyms; + } + + public String getReplacedBy() { + return replacedBy; + } + + public void setReplacedBy(String replacedBy) { + this.replacedBy = replacedBy; + } + + public String getValidValues() { + return validValues; + } + + public void setValidValues(String validValues) { + this.validValues = validValues; + } + + public String getReplacementTerms() { + return replacementTerms; + } + + public void setReplacementTerms(String replacementTerms) { + this.replacementTerms = replacementTerms; + } + + public String getSeeAlso() { + return seeAlso; + } + + public void setSeeAlso(String seeAlso) { + this.seeAlso = seeAlso; + } + + public String getTranslatedTerms() { + return translatedTerms; + } + + public void setTranslatedTerms(String translatedTerms) { + this.translatedTerms = translatedTerms; + } + + public String getIsA() { + return isA; + } + + public void setIsA(String isA) { + this.isA = isA; + } + + public String getAntonyms() { + return antonyms; + } + + public void setAntonyms(String antonyms) { + this.antonyms = antonyms; + } + + public String getClassifies() { + return classifies; + } + + public void setClassifies(String classifies) { + this.classifies = classifies; + } + + public String getPreferredToTerms() { + return preferredToTerms; + } + + public void setPreferredToTerms(String preferredToTerms) { + this.preferredToTerms = preferredToTerms; + } + + public String getPreferredTerms() { + return preferredTerms; + } + + public void setPreferredTerms(String preferredTerms) { + this.preferredTerms = preferredTerms; + } +} diff --git a/intg/src/main/java/org/apache/atlas/model/glossary/GlossarySearchParameters.java b/intg/src/main/java/org/apache/atlas/model/glossary/GlossarySearchParameters.java new file mode 100644 index 00000000000..7638f801cf4 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/glossary/GlossarySearchParameters.java @@ -0,0 +1,151 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.glossary; + +import org.apache.atlas.SortOrder; +import org.apache.atlas.model.annotation.AtlasJSON; + +import java.io.Serializable; + +/** + * Request parameters for glossary terms/categories search (display API). + * Mirrors Basic Search {@code SearchParameters} pagination and filter pattern for the UI table. + */ +@AtlasJSON +public class GlossarySearchParameters implements Serializable { + private static final long serialVersionUID = 1L; + + public enum GlossaryType { + ALL, TERM, CATEGORY + } + + @AtlasJSON + public static class GlossaryFilter implements Serializable { + private static final long serialVersionUID = 1L; + + private String name; + private String guid; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getGuid() { + return guid; + } + + public void setGuid(String guid) { + this.guid = guid; + } + } + + private int limit = 25; + private int offset; + private String sortBy = "name"; + private SortOrder sortOrder = SortOrder.ASCENDING; + private GlossaryFilter glossary; + private GlossaryType glossaryType; + private String status; + private String classificationContains; + private String searchQuery; + private boolean excludeDeleted = true; + + public int getLimit() { + return limit; + } + + public void setLimit(int limit) { + this.limit = limit; + } + + public int getOffset() { + return offset; + } + + public void setOffset(int offset) { + this.offset = offset; + } + + public String getSortBy() { + return sortBy; + } + + public void setSortBy(String sortBy) { + this.sortBy = sortBy; + } + + public SortOrder getSortOrder() { + return sortOrder != null ? sortOrder : SortOrder.ASCENDING; + } + + public void setSortOrder(SortOrder sortOrder) { + this.sortOrder = sortOrder; + } + + public GlossaryFilter getGlossary() { + return glossary; + } + + public void setGlossary(GlossaryFilter glossary) { + this.glossary = glossary; + } + + public GlossaryType getGlossaryType() { + return glossaryType != null ? glossaryType : GlossaryType.ALL; + } + + public void setGlossaryType(GlossaryType glossaryType) { + this.glossaryType = glossaryType; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getClassificationContains() { + return classificationContains; + } + + public void setClassificationContains(String classificationContains) { + this.classificationContains = classificationContains; + } + + public String getSearchQuery() { + return searchQuery; + } + + public void setSearchQuery(String searchQuery) { + this.searchQuery = searchQuery; + } + + public boolean getExcludeDeleted() { + return excludeDeleted; + } + + public void setExcludeDeleted(boolean excludeDeleted) { + this.excludeDeleted = excludeDeleted; + } +} diff --git a/intg/src/main/java/org/apache/atlas/model/glossary/GlossarySearchResult.java b/intg/src/main/java/org/apache/atlas/model/glossary/GlossarySearchResult.java new file mode 100644 index 00000000000..579132fbc79 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/glossary/GlossarySearchResult.java @@ -0,0 +1,290 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.glossary; + +import org.apache.atlas.model.annotation.AtlasJSON; +import org.apache.atlas.model.instance.AtlasClassification; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * Response for glossary search (display API). Nested glossary → terms/categories structure for the UI table. + */ +@AtlasJSON +public class GlossarySearchResult implements Serializable { + private static final long serialVersionUID = 1L; + + private List glossary = new ArrayList<>(); + private long approximateCount; + + public List getGlossary() { + if (glossary == null) { + glossary = new ArrayList<>(); + } + + return glossary; + } + + public void setGlossary(List glossary) { + this.glossary = glossary != null ? glossary : new ArrayList<>(); + } + + public long getApproximateCount() { + return approximateCount; + } + + public void setApproximateCount(long approximateCount) { + this.approximateCount = approximateCount; + } + + @AtlasJSON + public static class GlossaryDetail implements Serializable { + private static final long serialVersionUID = 1L; + + private String name; + private String guid; + private String shortDescription; + private String longDescription; + private String status; + private List terms = new ArrayList<>(); + private List categories = new ArrayList<>(); + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getGuid() { + return guid; + } + + public void setGuid(String guid) { + this.guid = guid; + } + + public String getShortDescription() { + return shortDescription; + } + + public void setShortDescription(String shortDescription) { + this.shortDescription = shortDescription; + } + + public String getLongDescription() { + return longDescription; + } + + public void setLongDescription(String longDescription) { + this.longDescription = longDescription; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public List getTerms() { + if (terms == null) { + terms = new ArrayList<>(); + } + + return terms; + } + + public void setTerms(List terms) { + this.terms = terms != null ? terms : new ArrayList<>(); + } + + public List getCategories() { + if (categories == null) { + categories = new ArrayList<>(); + } + + return categories; + } + + public void setCategories(List categories) { + this.categories = categories != null ? categories : new ArrayList<>(); + } + } + + @AtlasJSON + public static class GlossaryTermDetail implements Serializable { + private static final long serialVersionUID = 1L; + + private String name; + private String qualifiedName; + private String guid; + private String shortDescription; + private String longDescription; + private String status; + private List classifications = new ArrayList<>(); + private Map customAttributes; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getQualifiedName() { + return qualifiedName; + } + + public void setQualifiedName(String qualifiedName) { + this.qualifiedName = qualifiedName; + } + + public String getGuid() { + return guid; + } + + public void setGuid(String guid) { + this.guid = guid; + } + + public String getShortDescription() { + return shortDescription; + } + + public void setShortDescription(String shortDescription) { + this.shortDescription = shortDescription; + } + + public String getLongDescription() { + return longDescription; + } + + public void setLongDescription(String longDescription) { + this.longDescription = longDescription; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public List getClassifications() { + return classifications; + } + + public void setClassifications(List classifications) { + this.classifications = classifications; + } + + public Map getCustomAttributes() { + return customAttributes; + } + + public void setCustomAttributes(Map customAttributes) { + this.customAttributes = customAttributes; + } + } + + @AtlasJSON + public static class GlossaryCategoryDetail implements Serializable { + private static final long serialVersionUID = 1L; + + private String name; + private String qualifiedName; + private String guid; + private String shortDescription; + private String longDescription; + private String status; + private List classifications = new ArrayList<>(); + private Map customAttributes; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getQualifiedName() { + return qualifiedName; + } + + public void setQualifiedName(String qualifiedName) { + this.qualifiedName = qualifiedName; + } + + public String getGuid() { + return guid; + } + + public void setGuid(String guid) { + this.guid = guid; + } + + public String getShortDescription() { + return shortDescription; + } + + public void setShortDescription(String shortDescription) { + this.shortDescription = shortDescription; + } + + public String getLongDescription() { + return longDescription; + } + + public void setLongDescription(String longDescription) { + this.longDescription = longDescription; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public List getClassifications() { + return classifications; + } + + public void setClassifications(List classifications) { + this.classifications = classifications; + } + + public Map getCustomAttributes() { + return customAttributes; + } + + public void setCustomAttributes(Map customAttributes) { + this.customAttributes = customAttributes; + } + } +} diff --git a/pom.xml b/pom.xml index 350783560f2..9b4cb0f7508 100644 --- a/pom.xml +++ b/pom.xml @@ -99,6 +99,7 @@ 1.14 3.2.2 4.4 + 1.10 2.10.1 1.0 2.17.0 @@ -152,6 +153,7 @@ 2.13 3.9.1 6.0.1 + 2.17.1 1.3.15 8.11.3 3.11.0 @@ -357,6 +359,12 @@ ${commons-collections.version} + + commons-configuration + commons-configuration + ${commons-conf.version} + + commons-el commons-el @@ -824,6 +832,12 @@ + + org.apache.logging.log4j + log4j-api + ${log4j-api.version} + + org.apache.zookeeper zookeeper diff --git a/repository/pom.xml b/repository/pom.xml index 701baf9f4a7..319ab27bd75 100644 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -80,6 +80,12 @@ ${commons-codec.version} + + commons-lang + commons-lang + 2.6 + + joda-time joda-time @@ -168,6 +174,15 @@ hbase-client + + + org.apache.logging.log4j + log4j-api + ${log4j-api.version} + + org.apache.poi poi @@ -215,12 +230,6 @@ ${javax.persistence.version} - - org.mockito - mockito-core - ${mockito.version} - - org.postgresql postgresql @@ -301,6 +310,12 @@ + + org.mockito + mockito-core + test + + diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDataCollector.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDataCollector.java new file mode 100644 index 00000000000..cb71a44c5c3 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDataCollector.java @@ -0,0 +1,156 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.glossary; + +import org.apache.atlas.SortOrder; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.glossary.AtlasGlossary; +import org.apache.atlas.model.glossary.AtlasGlossaryCategory; +import org.apache.atlas.model.glossary.AtlasGlossaryTerm; +import org.apache.atlas.model.glossary.GlossaryExportParameters; +import org.apache.atlas.model.glossary.GlossaryExportRow; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +public class GlossaryExportDataCollector { + private static final Logger LOG = LoggerFactory.getLogger(GlossaryExportDataCollector.class); + + private final GlossaryService glossaryService; + + public GlossaryExportDataCollector(GlossaryService glossaryService) { + this.glossaryService = glossaryService; + } + + public List collect(GlossaryExportParameters parameters) throws AtlasBaseException { + List ret = new ArrayList<>(); + List glossaryGuids = resolveGlossaryGuids(parameters); + + LOG.debug("Collecting glossary export rows for {} glossaries", glossaryGuids.size()); + + for (String glossaryGuid : glossaryGuids) { + AtlasGlossary.AtlasGlossaryExtInfo glossaryExt = glossaryService.getDetailedGlossary(glossaryGuid); + + if (glossaryExt == null) { + continue; + } + + String glossaryName = glossaryExt.getName(); + + if (MapUtils.isNotEmpty(glossaryExt.getTermInfo())) { + for (AtlasGlossaryTerm term : glossaryExt.getTermInfo().values()) { + ret.add(buildTermRow(term, glossaryGuid, glossaryName)); + } + } + + if (MapUtils.isNotEmpty(glossaryExt.getCategoryInfo())) { + for (AtlasGlossaryCategory category : glossaryExt.getCategoryInfo().values()) { + ret.add(buildCategoryRow(category, glossaryGuid, glossaryName)); + } + } + } + + LOG.debug("Collected {} glossary export rows before filtering", ret.size()); + + return ret; + } + + private List resolveGlossaryGuids(GlossaryExportParameters parameters) throws AtlasBaseException { + if (StringUtils.isNotEmpty(parameters.getGlossaryGuid())) { + return Collections.singletonList(parameters.getGlossaryGuid()); + } + + if (CollectionUtils.isNotEmpty(parameters.getGlossaryGuids())) { + return parameters.getGlossaryGuids(); + } + + List glossaries = glossaryService.getGlossaries(-1, 0, SortOrder.ASCENDING); + List ret = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(glossaries)) { + for (AtlasGlossary glossary : glossaries) { + ret.add(glossary.getGuid()); + } + } + + return ret; + } + + private GlossaryExportRow buildTermRow(AtlasGlossaryTerm term, String glossaryGuid, String glossaryName) { + GlossaryExportRow row = new GlossaryExportRow(); + + row.setRecordType(GlossaryExportRow.RecordTypeKind.TERM); + row.setGuid(term.getGuid()); + row.setGlossaryGuid(glossaryGuid); + row.setGlossaryName(glossaryName); + row.setName(term.getName()); + row.setShortDescription(term.getShortDescription()); + row.setLongDescription(term.getLongDescription()); + row.setQualifiedName(term.getQualifiedName()); + row.setStatus(GlossaryExportUtils.deriveTermStatus(term)); + row.setClassifications(GlossaryExportUtils.formatClassifications(term.getClassifications())); + row.setCustomAttributes(GlossaryExportUtils.formatAdditionalAttributes(term.getAdditionalAttributes())); + row.setRelatedCategoriesOrParent(GlossaryExportUtils.formatTermCategories(term.getCategories())); + row.setEntityStatus(AtlasEntity.Status.ACTIVE.name()); + row.setExamples(GlossaryExportUtils.formatExamples(term.getExamples())); + row.setAbbreviation(term.getAbbreviation()); + row.setUsage(term.getUsage()); + row.setTranslationTerms(GlossaryExportUtils.formatRelatedTerms(term.getTranslationTerms(), glossaryName)); + row.setValidValuesFor(GlossaryExportUtils.formatRelatedTerms(term.getValidValuesFor(), glossaryName)); + row.setSynonyms(GlossaryExportUtils.formatRelatedTerms(term.getSynonyms(), glossaryName)); + row.setReplacedBy(GlossaryExportUtils.formatRelatedTerms(term.getReplacedBy(), glossaryName)); + row.setValidValues(GlossaryExportUtils.formatRelatedTerms(term.getValidValues(), glossaryName)); + row.setReplacementTerms(GlossaryExportUtils.formatRelatedTerms(term.getReplacementTerms(), glossaryName)); + row.setSeeAlso(GlossaryExportUtils.formatRelatedTerms(term.getSeeAlso(), glossaryName)); + row.setTranslatedTerms(GlossaryExportUtils.formatRelatedTerms(term.getTranslatedTerms(), glossaryName)); + row.setIsA(GlossaryExportUtils.formatRelatedTerms(term.getIsA(), glossaryName)); + row.setAntonyms(GlossaryExportUtils.formatRelatedTerms(term.getAntonyms(), glossaryName)); + row.setClassifies(GlossaryExportUtils.formatRelatedTerms(term.getClassifies(), glossaryName)); + row.setPreferredToTerms(GlossaryExportUtils.formatRelatedTerms(term.getPreferredToTerms(), glossaryName)); + row.setPreferredTerms(GlossaryExportUtils.formatRelatedTerms(term.getPreferredTerms(), glossaryName)); + + return row; + } + + private GlossaryExportRow buildCategoryRow(AtlasGlossaryCategory category, String glossaryGuid, String glossaryName) { + GlossaryExportRow row = new GlossaryExportRow(); + + row.setRecordType(GlossaryExportRow.RecordTypeKind.CATEGORY); + row.setGuid(category.getGuid()); + row.setGlossaryGuid(glossaryGuid); + row.setGlossaryName(glossaryName); + row.setName(category.getName()); + row.setShortDescription(category.getShortDescription()); + row.setLongDescription(category.getLongDescription()); + row.setQualifiedName(category.getQualifiedName()); + row.setStatus(GlossaryExportUtils.deriveCategoryStatus(category)); + row.setClassifications(GlossaryExportUtils.formatClassifications(category.getClassifications())); + row.setCustomAttributes(GlossaryExportUtils.formatAdditionalAttributes(category.getAdditionalAttributes())); + row.setRelatedCategoriesOrParent(GlossaryExportUtils.formatCategoryParent(category)); + row.setEntityStatus(AtlasEntity.Status.ACTIVE.name()); + + return row; + } +} diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDownloadTask.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDownloadTask.java new file mode 100644 index 00000000000..7d84850e035 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDownloadTask.java @@ -0,0 +1,141 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.glossary; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.glossary.GlossaryExportParameters; +import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.tasks.AbstractTask; +import org.apache.atlas.utils.AtlasJson; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.util.Map; + +import static org.apache.atlas.model.tasks.AtlasTask.Status.COMPLETE; +import static org.apache.atlas.model.tasks.AtlasTask.Status.FAILED; + +/** + * Background task that writes a glossary export file (CSV or XLSX) to the user's download directory. + * Mirrors {@link org.apache.atlas.repository.store.graph.v2.tasks.searchdownload.SearchResultDownloadTask}. + */ +public class GlossaryExportDownloadTask extends AbstractTask { + private static final Logger LOG = LoggerFactory.getLogger(GlossaryExportDownloadTask.class); + + public static final String EXPORT_PARAMETERS_JSON_KEY = "export_parameters_json"; + public static final String FILE_NAME_KEY = "file_name"; + + private static final String DOWNLOAD_DIR_PATH_KEY = "atlas.download.glossary.dir.path"; + private static final String DOWNLOAD_DIR_PATH_DEFAULT = System.getProperty("user.dir"); + private static final String GLOSSARY_DOWNLOAD_DIR = "glossary_export_downloads"; + + public static final String DOWNLOAD_DIR_PATH; + + private final GlossaryService glossaryService; + + public GlossaryExportDownloadTask(AtlasTask task, GlossaryService glossaryService) { + super(task); + this.glossaryService = glossaryService; + } + + @Override + public AtlasTask.Status perform() throws Exception { + RequestContext.clear(); + + Map params = getTaskDef().getParameters(); + + if (MapUtils.isEmpty(params)) { + LOG.warn("Task: {}: Unable to process task: Parameters is not readable!", getTaskGuid()); + return FAILED; + } + + String userName = getTaskDef().getCreatedBy(); + + if (StringUtils.isEmpty(userName)) { + LOG.warn("Task: {}: Unable to process task as user name is empty!", getTaskGuid()); + return FAILED; + } + + RequestContext.get().setUser(userName, null); + + try { + run(params); + setStatus(COMPLETE); + } catch (Exception e) { + LOG.error("Task: {}: Error performing glossary export download task!", getTaskGuid(), e); + setStatus(FAILED); + throw e; + } finally { + RequestContext.clear(); + } + + return getStatus(); + } + + protected void run(Map parameters) throws AtlasBaseException, IOException { + String exportParametersJson = (String) parameters.get(EXPORT_PARAMETERS_JSON_KEY); + String fileName = (String) parameters.get(FILE_NAME_KEY); + + GlossaryExportParameters exportParameters = AtlasJson.fromJson(exportParametersJson, GlossaryExportParameters.class); + + if (exportParameters == null || exportParameters.getFormat() == null) { + throw new AtlasBaseException(org.apache.atlas.AtlasErrorCode.BAD_REQUEST, "Export format (CSV or XLSX) is required"); + } + + File dir = new File(DOWNLOAD_DIR_PATH, RequestContext.getCurrentUser()); + + if (!dir.exists()) { + dir.mkdirs(); + } + + File exportFile = new File(dir, fileName); + + try (OutputStream outputStream = new FileOutputStream(exportFile)) { + glossaryService.exportGlossary(exportParameters, outputStream); + } + + LOG.info("Glossary export file generated: {}", exportFile.getAbsolutePath()); + } + + static { + Configuration configuration = null; + + try { + configuration = ApplicationProperties.get(); + } catch (AtlasException e) { + LOG.error("Error fetching application properties", e); + } + + if (configuration != null) { + DOWNLOAD_DIR_PATH = configuration.getString(DOWNLOAD_DIR_PATH_KEY, DOWNLOAD_DIR_PATH_DEFAULT) + + File.separator + GLOSSARY_DOWNLOAD_DIR; + } else { + DOWNLOAD_DIR_PATH = DOWNLOAD_DIR_PATH_DEFAULT + File.separator + GLOSSARY_DOWNLOAD_DIR; + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDownloadTaskFactory.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDownloadTaskFactory.java new file mode 100644 index 00000000000..c5c7f548f4b --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDownloadTaskFactory.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.glossary; + +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.model.tasks.AtlasTask; +import org.apache.atlas.tasks.AbstractTask; +import org.apache.atlas.tasks.TaskFactory; +import org.apache.commons.configuration2.Configuration; +import org.apache.commons.lang3.ArrayUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import javax.inject.Singleton; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +@Singleton +@Service +@EnableScheduling +public class GlossaryExportDownloadTaskFactory implements TaskFactory { + private static final Logger LOG = LoggerFactory.getLogger(GlossaryExportDownloadTaskFactory.class); + + public static final String GLOSSARY_EXPORT_DOWNLOAD = "GLOSSARY_EXPORT_DOWNLOAD"; + + private static final int MAX_PENDING_TASKS_ALLOWED_DEFAULT = 50; + private static final String MAX_PENDING_TASKS_ALLOWED_KEY = "atlas.download.glossary.max.pending.tasks"; + private static final String FILES_CLEANUP_INTERVAL = "0 0/1 * * * *"; + private static final long FILE_EXP_DURATION_IN_MILLIS_DEFAULT = 24 * 60 * 60 * 1000; + private static final String FILE_EXP_DURATION_IN_MILLIS_KEY = "atlas.download.glossary.file.expiry.millis"; + private static final List SUPPORTED_TYPES = new ArrayList<>(Collections.singletonList(GLOSSARY_EXPORT_DOWNLOAD)); + + public static final int MAX_PENDING_TASKS_ALLOWED; + private static final long FILE_EXP_DURATION_IN_MILLIS; + + private final GlossaryService glossaryService; + + @Inject + public GlossaryExportDownloadTaskFactory(GlossaryService glossaryService) { + this.glossaryService = glossaryService; + } + + @Override + public AbstractTask create(AtlasTask task) { + String taskType = task.getType(); + String taskGuid = task.getGuid(); + + switch (taskType) { + case GLOSSARY_EXPORT_DOWNLOAD: + return new GlossaryExportDownloadTask(task, glossaryService); + + default: + LOG.warn("Type: {} - {} not found!. The task will be ignored.", taskType, taskGuid); + return null; + } + } + + @Override + public List getSupportedTypes() { + return SUPPORTED_TYPES; + } + + @Scheduled(cron = "#{getGlossaryExportCronExpressionForCleanup}") + public void cleanupExpiredFiles() { + File downloadDir = new File(GlossaryExportDownloadTask.DOWNLOAD_DIR_PATH); + + deleteFiles(downloadDir); + } + + @Bean + private String getGlossaryExportCronExpressionForCleanup() { + return FILES_CLEANUP_INTERVAL; + } + + private void deleteFiles(File downloadDir) { + File[] subDirs = downloadDir.listFiles(); + + if (ArrayUtils.isNotEmpty(subDirs)) { + for (File subDir : subDirs) { + File[] exportFiles = subDir.listFiles(); + + if (ArrayUtils.isNotEmpty(exportFiles)) { + for (File exportFile : exportFiles) { + BasicFileAttributes attr; + + try { + attr = Files.readAttributes(exportFile.toPath(), BasicFileAttributes.class); + } catch (IOException e) { + throw new RuntimeException(e); + } + + Date createdTime = new Date(attr.creationTime().toMillis()); + + if (System.currentTimeMillis() - createdTime.getTime() > FILE_EXP_DURATION_IN_MILLIS) { + if (!exportFile.delete()) { + LOG.warn("Failed to delete expired glossary export file: {}", exportFile.getAbsolutePath()); + } + } + } + } + } + } + } + + static { + Configuration configuration = null; + + try { + configuration = ApplicationProperties.get(); + } catch (Exception e) { + LOG.error("Error fetching application properties", e); + } + + if (configuration != null) { + MAX_PENDING_TASKS_ALLOWED = configuration.getInt(MAX_PENDING_TASKS_ALLOWED_KEY, MAX_PENDING_TASKS_ALLOWED_DEFAULT); + FILE_EXP_DURATION_IN_MILLIS = configuration.getLong(FILE_EXP_DURATION_IN_MILLIS_KEY, FILE_EXP_DURATION_IN_MILLIS_DEFAULT); + } else { + MAX_PENDING_TASKS_ALLOWED = MAX_PENDING_TASKS_ALLOWED_DEFAULT; + FILE_EXP_DURATION_IN_MILLIS = FILE_EXP_DURATION_IN_MILLIS_DEFAULT; + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportFilter.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportFilter.java new file mode 100644 index 00000000000..e7c0814cc9d --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportFilter.java @@ -0,0 +1,81 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.glossary; + +import org.apache.atlas.model.glossary.GlossaryExportParameters; +import org.apache.atlas.model.glossary.GlossaryExportRow; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +public class GlossaryExportFilter { + public List apply(List rows, GlossaryExportParameters parameters) { + if (rows == null) { + return new ArrayList<>(); + } + + return rows.stream() + .filter(row -> matchesRecordType(row, parameters.getRecordType())) + .filter(row -> matchesGlossary(row, parameters)) + .filter(row -> GlossaryExportUtils.containsIgnoreCase(row.getStatus(), parameters.getStatusContains())) + .filter(row -> GlossaryExportUtils.containsIgnoreCase(row.getClassifications(), parameters.getClassificationContains())) + .filter(row -> GlossaryExportUtils.matchesSearch(row, parameters.getSearchQuery(), isGlossaryScoped(parameters))) + .filter(row -> matchesDeleted(row, parameters.getExcludeDeleted())) + .collect(Collectors.toList()); + } + + private boolean matchesRecordType(GlossaryExportRow row, GlossaryExportParameters.RecordType recordType) { + if (recordType == null || recordType == GlossaryExportParameters.RecordType.ALL) { + return true; + } + + if (recordType == GlossaryExportParameters.RecordType.TERM) { + return row.getRecordType() == GlossaryExportRow.RecordTypeKind.TERM; + } + + return row.getRecordType() == GlossaryExportRow.RecordTypeKind.CATEGORY; + } + + private boolean matchesGlossary(GlossaryExportRow row, GlossaryExportParameters parameters) { + if (StringUtils.isNotEmpty(parameters.getGlossaryGuid())) { + return StringUtils.equals(row.getGlossaryGuid(), parameters.getGlossaryGuid()); + } + + if (parameters.getGlossaryGuids() != null && !parameters.getGlossaryGuids().isEmpty()) { + return parameters.getGlossaryGuids().contains(row.getGlossaryGuid()); + } + + return true; + } + + private boolean isGlossaryScoped(GlossaryExportParameters parameters) { + return StringUtils.isNotEmpty(parameters.getGlossaryGuid()) + || (parameters.getGlossaryGuids() != null && !parameters.getGlossaryGuids().isEmpty()); + } + + private boolean matchesDeleted(GlossaryExportRow row, boolean excludeDeleted) { + if (!excludeDeleted) { + return true; + } + + return !StringUtils.equalsIgnoreCase(row.getEntityStatus(), AtlasEntity.Status.DELETED.name()); + } +} diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportParameterMapper.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportParameterMapper.java new file mode 100644 index 00000000000..d45fdccaab4 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportParameterMapper.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.glossary; + +import org.apache.atlas.model.glossary.GlossaryExportParameters; +import org.apache.atlas.model.glossary.GlossarySearchParameters; +import org.apache.atlas.utils.AtlasJson; +import org.apache.commons.lang3.StringUtils; + +import java.util.Map; + +/** + * Maps UI / REST request bodies to {@link GlossaryExportParameters}. + * Supports Basic Search-style {@code exportParameters} wrapper + * ({@link GlossarySearchParameters} fields sent directly to {@code /glossary/create_file}). + */ +public final class GlossaryExportParameterMapper { + private GlossaryExportParameterMapper() { + } + + public static GlossaryExportParameters fromCreateFileRequest(Map parameterMap) { + if (parameterMap == null || parameterMap.isEmpty()) { + return null; + } + + Object exportParameters = parameterMap.get("exportParameters"); + + if (exportParameters != null) { + return AtlasJson.fromLinkedHashMap(exportParameters, GlossaryExportParameters.class); + } + + if (isSearchStyleCreateFileRequest(parameterMap)) { + GlossarySearchParameters searchParameters = AtlasJson.fromLinkedHashMap(parameterMap, GlossarySearchParameters.class); + + return fromSearchParameters(searchParameters, resolveFormat(parameterMap)); + } + + if (parameterMap.containsKey("format") || parameterMap.containsKey("mode") || parameterMap.containsKey("recordType")) { + return AtlasJson.fromLinkedHashMap(parameterMap, GlossaryExportParameters.class); + } + + GlossarySearchParameters searchParameters = AtlasJson.fromLinkedHashMap(parameterMap, GlossarySearchParameters.class); + + return fromSearchParameters(searchParameters, resolveFormat(parameterMap)); + } + + /** + * Detect UI / search-style payloads so {@code format} does not force direct + * {@link GlossaryExportParameters} + */ + static boolean isSearchStyleCreateFileRequest(Map parameterMap) { + return parameterMap.containsKey("glossaryType") + || parameterMap.containsKey("glossary") + || parameterMap.containsKey("limit") + || parameterMap.containsKey("offset") + || parameterMap.containsKey("status"); + } + + public static GlossaryExportParameters fromSearchParameters(GlossarySearchParameters searchParameters, + GlossaryExportParameters.ExportFormat format) { + GlossaryExportParameters exportParameters = new GlossaryExportParameters(); + + exportParameters.setFormat(format != null ? format : GlossaryExportParameters.ExportFormat.CSV); + exportParameters.setMode(GlossaryExportParameters.ExportMode.AUDIT); + exportParameters.setExcludeDeleted(searchParameters != null && searchParameters.getExcludeDeleted()); + + if (searchParameters == null) { + return exportParameters; + } + + exportParameters.setSearchQuery(searchParameters.getSearchQuery()); + exportParameters.setClassificationContains(searchParameters.getClassificationContains()); + exportParameters.setStatusContains(searchParameters.getStatus()); + exportParameters.setSortBy(searchParameters.getSortBy()); + exportParameters.setSortOrder(searchParameters.getSortOrder()); + + if (searchParameters.getGlossaryType() == GlossarySearchParameters.GlossaryType.TERM) { + exportParameters.setRecordType(GlossaryExportParameters.RecordType.TERM); + } else if (searchParameters.getGlossaryType() == GlossarySearchParameters.GlossaryType.CATEGORY) { + exportParameters.setRecordType(GlossaryExportParameters.RecordType.CATEGORY); + } else { + exportParameters.setRecordType(GlossaryExportParameters.RecordType.ALL); + } + + if (searchParameters.getGlossary() != null) { + if (StringUtils.isNotEmpty(searchParameters.getGlossary().getGuid())) { + exportParameters.setGlossaryGuid(searchParameters.getGlossary().getGuid()); + } + } + + return exportParameters; + } + + private static GlossaryExportParameters.ExportFormat resolveFormat(Map parameterMap) { + Object format = parameterMap.get("format"); + + if (format == null) { + return GlossaryExportParameters.ExportFormat.CSV; + } + + try { + return GlossaryExportParameters.ExportFormat.valueOf(String.valueOf(format).trim().toUpperCase()); + } catch (IllegalArgumentException ex) { + return GlossaryExportParameters.ExportFormat.CSV; + } + } +} diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportUtils.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportUtils.java new file mode 100644 index 00000000000..82eb7842691 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportUtils.java @@ -0,0 +1,209 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.glossary; + +import org.apache.atlas.SortOrder; +import org.apache.atlas.model.glossary.AtlasGlossaryCategory; +import org.apache.atlas.model.glossary.AtlasGlossaryTerm; +import org.apache.atlas.model.glossary.GlossaryExportRow; +import org.apache.atlas.model.glossary.enums.AtlasTermRelationshipStatus; +import org.apache.atlas.model.glossary.relations.AtlasRelatedTermHeader; +import org.apache.atlas.model.glossary.relations.AtlasTermCategorizationHeader; +import org.apache.atlas.model.instance.AtlasClassification; +import org.apache.atlas.util.FileUtils; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +public final class GlossaryExportUtils { + public static final String DEFAULT_STATUS = "ACTIVE"; + public static final String STATUS_ATTR = "status"; + + private GlossaryExportUtils() { + } + + public static String deriveTermStatus(AtlasGlossaryTerm term) { + if (term == null) { + return DEFAULT_STATUS; + } + + if (CollectionUtils.isNotEmpty(term.getCategories())) { + for (AtlasTermCategorizationHeader category : term.getCategories()) { + if (category.getStatus() != null) { + return category.getStatus().name(); + } + } + } + + Map additionalAttributes = term.getAdditionalAttributes(); + + if (MapUtils.isNotEmpty(additionalAttributes) && additionalAttributes.get(STATUS_ATTR) != null) { + return String.valueOf(additionalAttributes.get(STATUS_ATTR)); + } + + return DEFAULT_STATUS; + } + + public static String deriveCategoryStatus(AtlasGlossaryCategory category) { + if (category == null) { + return DEFAULT_STATUS; + } + + Map additionalAttributes = category.getAdditionalAttributes(); + + if (MapUtils.isNotEmpty(additionalAttributes) && additionalAttributes.get(STATUS_ATTR) != null) { + return String.valueOf(additionalAttributes.get(STATUS_ATTR)); + } + + return DEFAULT_STATUS; + } + + public static String formatClassifications(List classifications) { + if (CollectionUtils.isEmpty(classifications)) { + return ""; + } + + return classifications.stream() + .map(AtlasClassification::getTypeName) + .filter(StringUtils::isNotEmpty) + .sorted() + .collect(Collectors.joining(", ")); + } + + public static String formatAdditionalAttributes(Map additionalAttributes) { + if (MapUtils.isEmpty(additionalAttributes)) { + return ""; + } + + List parts = new ArrayList<>(); + + for (Map.Entry entry : additionalAttributes.entrySet()) { + if (entry.getValue() != null) { + parts.add(entry.getKey() + "=" + entry.getValue()); + } + } + + return String.join("; ", parts); + } + + public static String formatTermCategories(Set categories) { + if (CollectionUtils.isEmpty(categories)) { + return ""; + } + + return categories.stream() + .map(AtlasTermCategorizationHeader::getDisplayText) + .filter(StringUtils::isNotEmpty) + .sorted() + .collect(Collectors.joining(", ")); + } + + public static String formatCategoryParent(AtlasGlossaryCategory category) { + if (category == null || category.getParentCategory() == null) { + return ""; + } + + return StringUtils.defaultString(category.getParentCategory().getDisplayText()); + } + + public static String formatExamples(List examples) { + if (CollectionUtils.isEmpty(examples)) { + return ""; + } + + return String.join("|", examples); + } + + public static String formatRelatedTerms(Set relatedTerms, String glossaryName) { + if (CollectionUtils.isEmpty(relatedTerms)) { + return ""; + } + + List parts = new ArrayList<>(); + + for (AtlasRelatedTermHeader header : relatedTerms) { + String termName = StringUtils.defaultIfEmpty(header.getDisplayText(), header.getTermGuid()); + String status = header.getStatus() != null ? header.getStatus().name() : AtlasTermRelationshipStatus.ACTIVE.name(); + + parts.add(termName + FileUtils.COLON_CHARACTER + status); + } + + return String.join("|", parts); + } + + public static boolean containsIgnoreCase(String value, String search) { + if (StringUtils.isEmpty(search)) { + return true; + } + + return StringUtils.containsIgnoreCase(StringUtils.defaultString(value), search); + } + + public static boolean matchesSearch(GlossaryExportRow row, String searchQuery) { + return matchesSearch(row, searchQuery, false); + } + + public static boolean matchesSearch(GlossaryExportRow row, String searchQuery, boolean glossaryScoped) { + if (StringUtils.isEmpty(searchQuery)) { + return true; + } + + if (containsIgnoreCase(row.getName(), searchQuery) + || containsIgnoreCase(row.getShortDescription(), searchQuery) + || containsIgnoreCase(row.getLongDescription(), searchQuery) + || containsIgnoreCase(row.getStatus(), searchQuery) + || containsIgnoreCase(row.getClassifications(), searchQuery) + || containsIgnoreCase(row.getCustomAttributes(), searchQuery) + || containsIgnoreCase(row.getRelatedCategoriesOrParent(), searchQuery)) { + return true; + } + + return !glossaryScoped && containsIgnoreCase(row.getGlossaryName(), searchQuery); + } + + public static void sortRows(List rows, String sortBy, SortOrder sortOrder) { + if (CollectionUtils.isEmpty(rows)) { + return; + } + + Comparator comparator = comparatorForSortBy(sortBy); + + if (sortOrder == SortOrder.DESCENDING) { + comparator = comparator.reversed(); + } + + rows.sort(comparator); + } + + private static Comparator comparatorForSortBy(String sortBy) { + if (StringUtils.equalsIgnoreCase(sortBy, "glossaryName")) { + return Comparator.comparing( + row -> StringUtils.defaultString(row.getGlossaryName()), String.CASE_INSENSITIVE_ORDER) + .thenComparing(row -> StringUtils.defaultString(row.getName()), String.CASE_INSENSITIVE_ORDER); + } + + return Comparator.comparing(row -> StringUtils.defaultString(row.getName()), String.CASE_INSENSITIVE_ORDER); + } +} diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportWriter.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportWriter.java new file mode 100644 index 00000000000..8c81e87c1e3 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportWriter.java @@ -0,0 +1,179 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.glossary; + +import com.opencsv.CSVWriter; +import org.apache.atlas.model.glossary.GlossaryExportParameters; +import org.apache.atlas.model.glossary.GlossaryExportRow; +import org.apache.commons.collections.CollectionUtils; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.List; + +public class GlossaryExportWriter { + private static final int MAX_CELL_LENGTH = 32767; + + private static final String[] AUDIT_HEADERS = { + "Record Type", "Name", "Glossary Name", "Short Description", "Long Description", + "Status", "Classifications", "Custom Attributes", "Related Categories / Parent", + "Qualified Name", "GUID" + }; + + public void write(OutputStream outputStream, List rows, + GlossaryExportParameters.ExportFormat format, + GlossaryExportParameters.ExportMode mode) throws IOException { + if (format == GlossaryExportParameters.ExportFormat.XLSX) { + writeXlsx(outputStream, rows, mode); + } else { + writeCsv(outputStream, rows, mode); + } + } + + public void writeCsv(OutputStream outputStream, List rows, + GlossaryExportParameters.ExportMode mode) throws IOException { + try (OutputStreamWriter osw = new OutputStreamWriter(outputStream, StandardCharsets.UTF_8); + CSVWriter writer = new CSVWriter(osw)) { + osw.write('\ufeff'); + + if (mode == GlossaryExportParameters.ExportMode.IMPORT_COMPATIBLE) { + writer.writeNext(getImportHeaders()); + if (CollectionUtils.isNotEmpty(rows)) { + for (GlossaryExportRow row : rows) { + if (row.getRecordType() == GlossaryExportRow.RecordTypeKind.TERM) { + writer.writeNext(toImportRow(row)); + } + } + } + } else { + writer.writeNext(AUDIT_HEADERS); + if (CollectionUtils.isNotEmpty(rows)) { + for (GlossaryExportRow row : rows) { + writer.writeNext(toAuditRow(row)); + } + } + } + } + } + + public void writeXlsx(OutputStream outputStream, List rows, + GlossaryExportParameters.ExportMode mode) throws IOException { + try (XSSFWorkbook workbook = new XSSFWorkbook()) { + Sheet sheet = workbook.createSheet("Glossary Export"); + String[] headers; + List dataRows = new ArrayList<>(); + + if (mode == GlossaryExportParameters.ExportMode.IMPORT_COMPATIBLE) { + headers = getImportHeaders(); + if (CollectionUtils.isNotEmpty(rows)) { + for (GlossaryExportRow row : rows) { + if (row.getRecordType() == GlossaryExportRow.RecordTypeKind.TERM) { + dataRows.add(toImportRow(row)); + } + } + } + } else { + headers = AUDIT_HEADERS; + if (CollectionUtils.isNotEmpty(rows)) { + for (GlossaryExportRow row : rows) { + dataRows.add(toAuditRow(row)); + } + } + } + + Row headerRow = sheet.createRow(0); + for (int i = 0; i < headers.length; i++) { + headerRow.createCell(i).setCellValue(headers[i]); + } + + int rowIdx = 1; + for (String[] data : dataRows) { + Row excelRow = sheet.createRow(rowIdx++); + for (int i = 0; i < data.length; i++) { + excelRow.createCell(i).setCellValue(truncate(data[i])); + } + } + + workbook.write(outputStream); + } + } + + private String[] getImportHeaders() { + return GlossaryTermUtils.getGlossaryTermHeaders().split(", "); + } + + private String[] toAuditRow(GlossaryExportRow row) { + return new String[] { + row.getRecordType() != null ? row.getRecordType().name() : "", + safe(row.getName()), + safe(row.getGlossaryName()), + safe(row.getShortDescription()), + safe(row.getLongDescription()), + safe(row.getStatus()), + safe(row.getClassifications()), + safe(row.getCustomAttributes()), + safe(row.getRelatedCategoriesOrParent()), + safe(row.getQualifiedName()), + safe(row.getGuid()) + }; + } + + private String[] toImportRow(GlossaryExportRow row) { + return new String[] { + safe(row.getGlossaryName()), + safe(row.getName()), + safe(row.getShortDescription()), + safe(row.getLongDescription()), + safe(row.getExamples()), + safe(row.getAbbreviation()), + safe(row.getUsage()), + safe(row.getCustomAttributes()), + safe(row.getTranslationTerms()), + safe(row.getValidValuesFor()), + safe(row.getSynonyms()), + safe(row.getReplacedBy()), + safe(row.getValidValues()), + safe(row.getReplacementTerms()), + safe(row.getSeeAlso()), + safe(row.getTranslatedTerms()), + safe(row.getIsA()), + safe(row.getAntonyms()), + safe(row.getClassifies()), + safe(row.getPreferredToTerms()), + safe(row.getPreferredTerms()) + }; + } + + private String safe(String value) { + return value != null ? value : ""; + } + + private String truncate(String value) { + if (value == null) { + return ""; + } + + return value.length() > MAX_CELL_LENGTH ? value.substring(0, MAX_CELL_LENGTH) : value; + } +} diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossarySearchBuilder.java b/repository/src/main/java/org/apache/atlas/glossary/GlossarySearchBuilder.java new file mode 100644 index 00000000000..4f1ce91d77a --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossarySearchBuilder.java @@ -0,0 +1,316 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.glossary; + +import org.apache.atlas.SortOrder; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.glossary.AtlasGlossary; +import org.apache.atlas.model.glossary.AtlasGlossaryCategory; +import org.apache.atlas.model.glossary.AtlasGlossaryTerm; +import org.apache.atlas.model.glossary.GlossaryExportParameters; +import org.apache.atlas.model.glossary.GlossaryExportRow; +import org.apache.atlas.model.glossary.GlossarySearchParameters; +import org.apache.atlas.model.glossary.GlossarySearchResult; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +/** + * Builds paginated, nested JSON for glossary terms-list display. + * Reuses {@link GlossaryExportDataCollector} and {@link GlossaryExportFilter} so search filters match export filters. + */ +public class GlossarySearchBuilder { + private static final Logger LOG = LoggerFactory.getLogger(GlossarySearchBuilder.class); + + private final GlossaryService glossaryService; + + public GlossarySearchBuilder(GlossaryService glossaryService) { + this.glossaryService = glossaryService; + } + + public GlossarySearchResult search(GlossarySearchParameters parameters) throws AtlasBaseException { + GlossaryExportParameters exportParameters = toExportParameters(parameters); + GlossaryExportDataCollector collector = new GlossaryExportDataCollector(glossaryService); + GlossaryExportFilter filter = new GlossaryExportFilter(); + + List rows = filter.apply(collector.collect(exportParameters), exportParameters); + rows = filterByGlossaryName(rows, parameters); + sort(rows, parameters); + + long totalCount = rows.size(); + List pageRows = paginate(rows, parameters); + + LOG.debug("Glossary search matched {} rows, returning page of {}", totalCount, pageRows.size()); + + return buildResult(pageRows, totalCount, parameters); + } + + private GlossaryExportParameters toExportParameters(GlossarySearchParameters parameters) { + GlossaryExportParameters exportParameters = new GlossaryExportParameters(); + + exportParameters.setRecordType(toRecordType(parameters.getGlossaryType())); + exportParameters.setStatusContains(parameters.getStatus()); + exportParameters.setClassificationContains(parameters.getClassificationContains()); + exportParameters.setSearchQuery(parameters.getSearchQuery()); + exportParameters.setExcludeDeleted(parameters.getExcludeDeleted()); + + if (parameters.getGlossary() != null && StringUtils.isNotEmpty(parameters.getGlossary().getGuid())) { + exportParameters.setGlossaryGuid(parameters.getGlossary().getGuid()); + } + + return exportParameters; + } + + private GlossaryExportParameters.RecordType toRecordType(GlossarySearchParameters.GlossaryType glossaryType) { + if (glossaryType == null || glossaryType == GlossarySearchParameters.GlossaryType.ALL) { + return GlossaryExportParameters.RecordType.ALL; + } + + if (glossaryType == GlossarySearchParameters.GlossaryType.TERM) { + return GlossaryExportParameters.RecordType.TERM; + } + + return GlossaryExportParameters.RecordType.CATEGORY; + } + + private List filterByGlossaryName(List rows, GlossarySearchParameters parameters) { + if (parameters.getGlossary() == null || StringUtils.isEmpty(parameters.getGlossary().getName())) { + return rows; + } + + String glossaryName = parameters.getGlossary().getName().toLowerCase(Locale.ROOT); + List ret = new ArrayList<>(); + + for (GlossaryExportRow row : rows) { + if (row.getGlossaryName() != null && row.getGlossaryName().toLowerCase(Locale.ROOT).contains(glossaryName)) { + ret.add(row); + } + } + + return ret; + } + + private void sort(List rows, GlossarySearchParameters parameters) { + Comparator comparator = resolveComparator(parameters.getSortBy()); + boolean descending = parameters.getSortOrder() == SortOrder.DESCENDING; + + if (descending) { + comparator = comparator.reversed(); + } + + rows.sort(comparator); + } + + private Comparator resolveComparator(String sortBy) { + if (StringUtils.isBlank(sortBy)) { + return Comparator.comparing(row -> StringUtils.defaultString(row.getName()), String.CASE_INSENSITIVE_ORDER); + } + + switch (sortBy.toLowerCase(Locale.ROOT)) { + case "glossaryname": + case "glossary": + return Comparator.comparing(row -> StringUtils.defaultString(row.getGlossaryName()), String.CASE_INSENSITIVE_ORDER); + case "status": + return Comparator.comparing(row -> StringUtils.defaultString(row.getStatus()), String.CASE_INSENSITIVE_ORDER); + case "qualifiedname": + return Comparator.comparing(row -> StringUtils.defaultString(row.getQualifiedName()), String.CASE_INSENSITIVE_ORDER); + case "recordtype": + case "type": + case "glossarytype": + return Comparator.comparing(row -> row.getRecordType() != null ? row.getRecordType().name() : "", String.CASE_INSENSITIVE_ORDER); + case "name": + default: + return Comparator.comparing(row -> StringUtils.defaultString(row.getName()), String.CASE_INSENSITIVE_ORDER); + } + } + + private List paginate(List rows, GlossarySearchParameters parameters) { + int offset = Math.max(0, parameters.getOffset()); + int limit = parameters.getLimit(); + + if (offset >= rows.size()) { + return Collections.emptyList(); + } + + if (limit <= 0) { + return new ArrayList<>(rows.subList(offset, rows.size())); + } + + int end = Math.min(offset + limit, rows.size()); + + return new ArrayList<>(rows.subList(offset, end)); + } + + private GlossarySearchResult buildResult(List pageRows, long totalCount, GlossarySearchParameters parameters) + throws AtlasBaseException { + GlossarySearchResult result = new GlossarySearchResult(); + result.setApproximateCount(totalCount); + + if (CollectionUtils.isEmpty(pageRows)) { + result.setGlossary(Collections.emptyList()); + return result; + } + + Map glossaryByGuid = new LinkedHashMap<>(); + Map extByGuid = new LinkedHashMap<>(); + + boolean includeTerms = parameters.getGlossaryType() != GlossarySearchParameters.GlossaryType.CATEGORY; + boolean includeCategories = parameters.getGlossaryType() != GlossarySearchParameters.GlossaryType.TERM; + + for (GlossaryExportRow row : pageRows) { + AtlasGlossary.AtlasGlossaryExtInfo ext = extByGuid.computeIfAbsent(row.getGlossaryGuid(), this::loadExtInfoSafely); + + if (ext == null) { + continue; + } + + GlossarySearchResult.GlossaryDetail glossaryDetail = glossaryByGuid.computeIfAbsent( + row.getGlossaryGuid(), guid -> buildGlossaryDetail(ext)); + + if (row.getRecordType() == GlossaryExportRow.RecordTypeKind.TERM && includeTerms) { + AtlasGlossaryTerm term = findTerm(ext, row.getGuid()); + + if (term != null) { + glossaryDetail.getTerms().add(buildTermDetail(term)); + } + } else if (row.getRecordType() == GlossaryExportRow.RecordTypeKind.CATEGORY && includeCategories) { + AtlasGlossaryCategory category = findCategory(ext, row.getGuid()); + + if (category != null) { + glossaryDetail.getCategories().add(buildCategoryDetail(category)); + } + } + } + + result.setGlossary(new ArrayList<>(glossaryByGuid.values())); + return result; + } + + private AtlasGlossary.AtlasGlossaryExtInfo loadExtInfoSafely(String glossaryGuid) { + try { + return glossaryService.getDetailedGlossary(glossaryGuid); + } catch (AtlasBaseException e) { + LOG.warn("Failed to load detailed glossary for guid {}", glossaryGuid, e); + return null; + } + } + + private GlossarySearchResult.GlossaryDetail buildGlossaryDetail(AtlasGlossary.AtlasGlossaryExtInfo ext) { + GlossarySearchResult.GlossaryDetail detail = new GlossarySearchResult.GlossaryDetail(); + + detail.setName(ext.getName()); + detail.setGuid(ext.getGuid()); + detail.setShortDescription(ext.getShortDescription()); + detail.setLongDescription(ext.getLongDescription()); + detail.setStatus(GlossaryExportUtils.DEFAULT_STATUS); + + if (MapUtils.isNotEmpty(ext.getAdditionalAttributes()) && ext.getAdditionalAttributes().get(GlossaryExportUtils.STATUS_ATTR) != null) { + detail.setStatus(String.valueOf(ext.getAdditionalAttributes().get(GlossaryExportUtils.STATUS_ATTR))); + } + + return detail; + } + + private AtlasGlossaryTerm findTerm(AtlasGlossary.AtlasGlossaryExtInfo ext, String guid) { + if (MapUtils.isEmpty(ext.getTermInfo())) { + return null; + } + + AtlasGlossaryTerm term = ext.getTermInfo().get(guid); + + if (term != null) { + return term; + } + + for (AtlasGlossaryTerm candidate : ext.getTermInfo().values()) { + if (StringUtils.equals(candidate.getGuid(), guid)) { + return candidate; + } + } + + return null; + } + + private AtlasGlossaryCategory findCategory(AtlasGlossary.AtlasGlossaryExtInfo ext, String guid) { + if (MapUtils.isEmpty(ext.getCategoryInfo())) { + return null; + } + + AtlasGlossaryCategory category = ext.getCategoryInfo().get(guid); + + if (category != null) { + return category; + } + + for (AtlasGlossaryCategory candidate : ext.getCategoryInfo().values()) { + if (StringUtils.equals(candidate.getGuid(), guid)) { + return candidate; + } + } + + return null; + } + + private GlossarySearchResult.GlossaryTermDetail buildTermDetail(AtlasGlossaryTerm term) { + GlossarySearchResult.GlossaryTermDetail detail = new GlossarySearchResult.GlossaryTermDetail(); + + detail.setName(term.getName()); + detail.setQualifiedName(term.getQualifiedName()); + detail.setGuid(term.getGuid()); + detail.setShortDescription(term.getShortDescription()); + detail.setLongDescription(term.getLongDescription()); + detail.setStatus(GlossaryExportUtils.deriveTermStatus(term)); + + if (CollectionUtils.isNotEmpty(term.getClassifications())) { + detail.setClassifications(new ArrayList<>(term.getClassifications())); + } + + detail.setCustomAttributes(term.getAdditionalAttributes()); + + return detail; + } + + private GlossarySearchResult.GlossaryCategoryDetail buildCategoryDetail(AtlasGlossaryCategory category) { + GlossarySearchResult.GlossaryCategoryDetail detail = new GlossarySearchResult.GlossaryCategoryDetail(); + + detail.setName(category.getName()); + detail.setQualifiedName(category.getQualifiedName()); + detail.setGuid(category.getGuid()); + detail.setShortDescription(category.getShortDescription()); + detail.setLongDescription(category.getLongDescription()); + detail.setStatus(GlossaryExportUtils.deriveCategoryStatus(category)); + + if (CollectionUtils.isNotEmpty(category.getClassifications())) { + detail.setClassifications(new ArrayList<>(category.getClassifications())); + } + + detail.setCustomAttributes(category.getAdditionalAttributes()); + + return detail; + } +} diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java index f28b9cb625a..8323a17e739 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryService.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.glossary; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.SortOrder; @@ -24,14 +25,20 @@ import org.apache.atlas.bulkimport.BulkImportResponse; import org.apache.atlas.bulkimport.BulkImportResponse.ImportInfo; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.AtlasSearchResultDownloadStatus; +import org.apache.atlas.model.discovery.AtlasSearchResultDownloadStatus.AtlasSearchDownloadRecord; import org.apache.atlas.model.glossary.AtlasGlossary; import org.apache.atlas.model.glossary.AtlasGlossaryCategory; import org.apache.atlas.model.glossary.AtlasGlossaryTerm; +import org.apache.atlas.model.glossary.GlossaryExportParameters; +import org.apache.atlas.model.glossary.GlossarySearchParameters; +import org.apache.atlas.model.glossary.GlossarySearchResult; import org.apache.atlas.model.glossary.relations.AtlasRelatedCategoryHeader; import org.apache.atlas.model.glossary.relations.AtlasRelatedTermHeader; import org.apache.atlas.model.glossary.relations.AtlasTermCategorizationHeader; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasRelatedObjectId; +import org.apache.atlas.model.tasks.AtlasTask; import org.apache.atlas.repository.graphdb.AtlasEdge; import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -40,6 +47,7 @@ import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.v2.AtlasEntityChangeNotifier; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.tasks.TaskManagement; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.util.FileUtils; import org.apache.atlas.utils.AtlasJson; @@ -51,10 +59,16 @@ import javax.inject.Inject; +import java.io.File; +import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -64,6 +78,7 @@ import static java.util.Objects.isNull; import static java.util.Objects.nonNull; import static java.util.Objects.requireNonNull; +import static org.apache.atlas.AtlasErrorCode.PENDING_TASKS_ALREADY_IN_PROGRESS; import static org.apache.atlas.bulkimport.BulkImportResponse.ImportStatus.FAILED; import static org.apache.atlas.bulkimport.BulkImportResponse.ImportStatus.SUCCESS; import static org.apache.atlas.glossary.GlossaryUtils.getAtlasGlossaryCategorySkeleton; @@ -93,15 +108,18 @@ public class GlossaryService { private final AtlasTypeRegistry atlasTypeRegistry; private final AtlasEntityChangeNotifier entityChangeNotifier; private final AtlasGlossaryDTO glossaryDTO; + private final TaskManagement taskManagement; @Inject - public GlossaryService(DataAccess dataAccess, final AtlasRelationshipStore relationshipStore, final AtlasTypeRegistry typeRegistry, AtlasEntityChangeNotifier entityChangeNotifier, final AtlasGlossaryDTO glossaryDTO) { + public GlossaryService(DataAccess dataAccess, final AtlasRelationshipStore relationshipStore, final AtlasTypeRegistry typeRegistry, + AtlasEntityChangeNotifier entityChangeNotifier, final AtlasGlossaryDTO glossaryDTO, TaskManagement taskManagement) { this.dataAccess = dataAccess; atlasTypeRegistry = typeRegistry; glossaryTermUtils = new GlossaryTermUtils(relationshipStore, typeRegistry, dataAccess); glossaryCategoryUtils = new GlossaryCategoryUtils(relationshipStore, typeRegistry, dataAccess); this.entityChangeNotifier = entityChangeNotifier; this.glossaryDTO = glossaryDTO; + this.taskManagement = taskManagement; } public static boolean isNameInvalid(String name) { @@ -952,6 +970,103 @@ public BulkImportResponse importGlossaryData(InputStream inputStream, String fil return ret; } + @GraphTransaction + public void exportGlossary(GlossaryExportParameters parameters, OutputStream outputStream) throws AtlasBaseException { + LOG.debug("==> GlossaryService.exportGlossary({})", parameters); + + if (parameters == null || parameters.getFormat() == null) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Export format (CSV or XLSX) is required"); + } + + GlossaryExportDataCollector collector = new GlossaryExportDataCollector(this); + GlossaryExportFilter filter = new GlossaryExportFilter(); + GlossaryExportWriter writer = new GlossaryExportWriter(); + + List rows = filter.apply(collector.collect(parameters), parameters); + + GlossaryExportUtils.sortRows(rows, parameters.getSortBy(), parameters.getSortOrder()); + + int maxRows = AtlasConfiguration.GLOSSARY_EXPORT_MAX_ROWS.getInt(); + + if (rows.size() > maxRows) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, + "Export row count (" + rows.size() + ") exceeds maximum allowed (" + maxRows + ")"); + } + + try { + writer.write(outputStream, rows, parameters.getFormat(), parameters.getMode()); + } catch (IOException e) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Failed to write glossary export: " + e.getMessage()); + } + + LOG.debug("<== GlossaryService.exportGlossary(): {} rows exported", rows.size()); + } + + @GraphTransaction + public GlossarySearchResult searchGlossary(GlossarySearchParameters parameters) throws AtlasBaseException { + LOG.debug("==> GlossaryService.searchGlossary({})", parameters); + + GlossarySearchBuilder builder = new GlossarySearchBuilder(this); + GlossarySearchResult result = builder.search(parameters != null ? parameters : new GlossarySearchParameters()); + + LOG.debug("<== GlossaryService.searchGlossary(): {} glossaries, {} total rows", result.getGlossary().size(), result.getApproximateCount()); + + return result; + } + + @GraphTransaction + public void createAndQueueGlossaryExportDownloadTask(Map taskParams) throws AtlasBaseException { + List pendingTasks = taskManagement.getPendingTasksByType(GlossaryExportDownloadTaskFactory.GLOSSARY_EXPORT_DOWNLOAD); + + if (CollectionUtils.isNotEmpty(pendingTasks) && pendingTasks.size() > GlossaryExportDownloadTaskFactory.MAX_PENDING_TASKS_ALLOWED) { + throw new AtlasBaseException(PENDING_TASKS_ALREADY_IN_PROGRESS, String.valueOf(pendingTasks.size())); + } + + AtlasTask task = taskManagement.createTask(GlossaryExportDownloadTaskFactory.GLOSSARY_EXPORT_DOWNLOAD, + RequestContext.getCurrentUser(), taskParams); + + RequestContext.get().queueTask(task); + } + + public AtlasSearchResultDownloadStatus getGlossaryExportDownloadStatus() throws IOException { + List pendingTasks = taskManagement.getPendingTasksByType(GlossaryExportDownloadTaskFactory.GLOSSARY_EXPORT_DOWNLOAD); + List currentUserPendingTasks = pendingTasks.stream() + .filter(task -> task.getCreatedBy().equals(RequestContext.getCurrentUser())) + .collect(Collectors.toList()); + List downloadRecords = new ArrayList<>(); + + for (AtlasTask pendingTask : currentUserPendingTasks) { + String fileName = (String) pendingTask.getParameters().get(GlossaryExportDownloadTask.FILE_NAME_KEY); + AtlasSearchDownloadRecord record = new AtlasSearchDownloadRecord(pendingTask.getStatus(), fileName, + pendingTask.getCreatedBy(), pendingTask.getCreatedTime(), pendingTask.getStartTime()); + + downloadRecords.add(record); + } + + File fileDir = new File(GlossaryExportDownloadTask.DOWNLOAD_DIR_PATH, RequestContext.getCurrentUser()); + + if (fileDir.exists()) { + File[] currentUserFiles = fileDir.listFiles(); + + if (currentUserFiles != null) { + for (File file : currentUserFiles) { + BasicFileAttributes attr = Files.readAttributes(file.toPath(), BasicFileAttributes.class); + Date createdTime = new Date(attr.creationTime().toMillis()); + AtlasSearchDownloadRecord record = new AtlasSearchDownloadRecord(AtlasTask.Status.COMPLETE, + file.getName(), RequestContext.getCurrentUser(), createdTime); + + downloadRecords.add(record); + } + } + } + + AtlasSearchResultDownloadStatus result = new AtlasSearchResultDownloadStatus(); + + result.setSearchDownloadRecords(downloadRecords); + + return result; + } + private boolean glossaryExists(AtlasGlossary atlasGlossary) { Map uniqAttr = new HashMap<>(); diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java b/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java index 51ae8b3c8a3..4b5baf61912 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java @@ -144,7 +144,7 @@ private static Integer getMaxRetryCount() { private static Long getRetrySleepTime() { initApplicationProperties(); - return (applicationProperties == null) ? 30000 : applicationProperties.getLong(GRAPH_REPOSITORY_RETRY_SLEEPTIME, 30000); + return (applicationProperties == null) ? 30000L : applicationProperties.getLong(GRAPH_REPOSITORY_RETRY_SLEEPTIME, 30000L); } private static void initApplicationProperties() { diff --git a/repository/src/test/java/org/apache/atlas/glossary/GlossaryExportFilterTest.java b/repository/src/test/java/org/apache/atlas/glossary/GlossaryExportFilterTest.java new file mode 100644 index 00000000000..391a889da1f --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/glossary/GlossaryExportFilterTest.java @@ -0,0 +1,139 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.glossary; + +import org.apache.atlas.model.glossary.GlossaryExportParameters; +import org.apache.atlas.model.glossary.GlossaryExportRow; +import org.apache.atlas.model.instance.AtlasEntity; +import org.testng.annotations.Test; + +import java.util.Arrays; +import java.util.List; + +import static org.testng.Assert.assertEquals; + +public class GlossaryExportFilterTest { + private final GlossaryExportFilter filter = new GlossaryExportFilter(); + + @Test + public void testFilterByRecordTypeTerm() { + GlossaryExportParameters parameters = new GlossaryExportParameters(); + parameters.setRecordType(GlossaryExportParameters.RecordType.TERM); + + List rows = filter.apply(sampleRows(), parameters); + + assertEquals(rows.size(), 1); + assertEquals(rows.get(0).getName(), "BankBranch"); + } + + @Test + public void testFilterByGlossaryGuid() { + GlossaryExportParameters parameters = new GlossaryExportParameters(); + parameters.setGlossaryGuid("glossary-guid-1"); + + List rows = filter.apply(sampleRows(), parameters); + + assertEquals(rows.size(), 1); + assertEquals(rows.get(0).getGlossaryGuid(), "glossary-guid-1"); + } + + @Test + public void testFilterByStatusContains() { + GlossaryExportParameters parameters = new GlossaryExportParameters(); + parameters.setStatusContains("draft"); + + List rows = filter.apply(sampleRows(), parameters); + + assertEquals(rows.size(), 1); + assertEquals(rows.get(0).getStatus(), "DRAFT"); + } + + @Test + public void testFilterBySearchQuery() { + GlossaryExportParameters parameters = new GlossaryExportParameters(); + parameters.setSearchQuery("capital"); + + List rows = filter.apply(sampleRows(), parameters); + + assertEquals(rows.size(), 1); + assertEquals(rows.get(0).getName(), "CapitalTerm"); + } + + @Test + public void testExcludeDeleted() { + GlossaryExportParameters parameters = new GlossaryExportParameters(); + parameters.setExcludeDeleted(true); + + List rows = filter.apply(sampleRows(), parameters); + + assertEquals(rows.size(), 2); + } + + @Test + public void testFilterByCombinedGlossaryGuidAndSearchQuery() { + GlossaryExportParameters parameters = new GlossaryExportParameters(); + parameters.setGlossaryGuid("glossary-guid-1"); + parameters.setRecordType(GlossaryExportParameters.RecordType.TERM); + parameters.setSearchQuery("Bank"); + + GlossaryExportRow capitalTerm = new GlossaryExportRow(); + capitalTerm.setRecordType(GlossaryExportRow.RecordTypeKind.TERM); + capitalTerm.setGuid("term-2"); + capitalTerm.setGlossaryGuid("glossary-guid-1"); + capitalTerm.setGlossaryName("testBankingGlossary"); + capitalTerm.setName("CapitalTerm075"); + capitalTerm.setStatus("DRAFT"); + capitalTerm.setEntityStatus(AtlasEntity.Status.ACTIVE.name()); + + List rows = filter.apply(Arrays.asList(sampleRows().get(0), capitalTerm), parameters); + + assertEquals(rows.size(), 1); + assertEquals(rows.get(0).getName(), "BankBranch"); + } + + private List sampleRows() { + GlossaryExportRow term = new GlossaryExportRow(); + term.setRecordType(GlossaryExportRow.RecordTypeKind.TERM); + term.setGuid("term-1"); + term.setGlossaryGuid("glossary-guid-1"); + term.setGlossaryName("testBankingGlossary"); + term.setName("BankBranch"); + term.setStatus("ACTIVE"); + term.setEntityStatus(AtlasEntity.Status.ACTIVE.name()); + + GlossaryExportRow category = new GlossaryExportRow(); + category.setRecordType(GlossaryExportRow.RecordTypeKind.CATEGORY); + category.setGuid("cat-1"); + category.setGlossaryGuid("glossary-guid-2"); + category.setGlossaryName("testFinanceGlossary"); + category.setName("CapitalTerm"); + category.setStatus("DRAFT"); + category.setEntityStatus(AtlasEntity.Status.ACTIVE.name()); + + GlossaryExportRow deleted = new GlossaryExportRow(); + deleted.setRecordType(GlossaryExportRow.RecordTypeKind.TERM); + deleted.setGuid("term-2"); + deleted.setGlossaryGuid("glossary-guid-1"); + deleted.setGlossaryName("testBankingGlossary"); + deleted.setName("DeletedTerm"); + deleted.setStatus("ACTIVE"); + deleted.setEntityStatus(AtlasEntity.Status.DELETED.name()); + + return Arrays.asList(term, category, deleted); + } +} diff --git a/repository/src/test/java/org/apache/atlas/glossary/GlossaryExportUtilsTest.java b/repository/src/test/java/org/apache/atlas/glossary/GlossaryExportUtilsTest.java new file mode 100644 index 00000000000..00879ba5f87 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/glossary/GlossaryExportUtilsTest.java @@ -0,0 +1,84 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.glossary; + +import org.apache.atlas.model.glossary.AtlasGlossaryTerm; +import org.apache.atlas.model.glossary.enums.AtlasTermRelationshipStatus; +import org.apache.atlas.model.glossary.relations.AtlasTermCategorizationHeader; +import org.apache.atlas.model.instance.AtlasClassification; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class GlossaryExportUtilsTest { + @Test + public void testDeriveTermStatusFromCategory() { + AtlasGlossaryTerm term = new AtlasGlossaryTerm(); + + AtlasTermCategorizationHeader header = new AtlasTermCategorizationHeader(); + header.setStatus(AtlasTermRelationshipStatus.DRAFT); + + term.setCategories(new HashSet<>(Collections.singletonList(header))); + + assertEquals(GlossaryExportUtils.deriveTermStatus(term), "DRAFT"); + } + + @Test + public void testDeriveTermStatusFromAdditionalAttributes() { + AtlasGlossaryTerm term = new AtlasGlossaryTerm(); + Map attrs = new HashMap<>(); + attrs.put("status", "Deprecated"); + term.setAdditionalAttributes(attrs); + + assertEquals(GlossaryExportUtils.deriveTermStatus(term), "Deprecated"); + } + + @Test + public void testFormatClassifications() { + AtlasClassification classification = new AtlasClassification(); + classification.setTypeName("PII"); + + String result = GlossaryExportUtils.formatClassifications(Collections.singletonList(classification)); + assertEquals(result, "PII"); + } + + @Test + public void testMatchesSearch() { + org.apache.atlas.model.glossary.GlossaryExportRow row = new org.apache.atlas.model.glossary.GlossaryExportRow(); + row.setName("BankBranch"); + row.setGlossaryName("testBankingGlossary"); + + assertTrue(GlossaryExportUtils.matchesSearch(row, "bank")); + } + + @Test + public void testMatchesSearchWhenGlossaryScopedSkipsGlossaryName() { + org.apache.atlas.model.glossary.GlossaryExportRow row = new org.apache.atlas.model.glossary.GlossaryExportRow(); + row.setName("CapitalTerm075"); + row.setGlossaryName("testBankingGlossary"); + + assertTrue(GlossaryExportUtils.matchesSearch(row, "bank")); + assertTrue(!GlossaryExportUtils.matchesSearch(row, "bank", true)); + } +} diff --git a/repository/src/test/java/org/apache/atlas/glossary/GlossaryExportWriterTest.java b/repository/src/test/java/org/apache/atlas/glossary/GlossaryExportWriterTest.java new file mode 100644 index 00000000000..e1cebfe521e --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/glossary/GlossaryExportWriterTest.java @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.glossary; + +import org.apache.atlas.model.glossary.GlossaryExportParameters; +import org.apache.atlas.model.glossary.GlossaryExportRow; +import org.apache.poi.ss.usermodel.Row; +import org.apache.poi.ss.usermodel.Sheet; +import org.apache.poi.xssf.usermodel.XSSFWorkbook; +import org.testng.annotations.Test; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.Collections; + +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertTrue; + +public class GlossaryExportWriterTest { + private final GlossaryExportWriter writer = new GlossaryExportWriter(); + + @Test + public void testWriteAuditCsv() throws Exception { + GlossaryExportRow row = new GlossaryExportRow(); + row.setRecordType(GlossaryExportRow.RecordTypeKind.TERM); + row.setName("BankBranch"); + row.setGlossaryName("testBankingGlossary"); + row.setShortDescription("SD4"); + row.setLongDescription("LD4"); + row.setStatus("ACTIVE"); + row.setGuid("term-guid-1"); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + writer.writeCsv(output, Collections.singletonList(row), GlossaryExportParameters.ExportMode.AUDIT); + + String csv = output.toString(StandardCharsets.UTF_8.name()); + assertTrue(csv.contains("BankBranch")); + assertTrue(csv.contains("testBankingGlossary")); + assertTrue(csv.contains("Record Type")); + } + + @Test + public void testWriteAuditXlsx() throws Exception { + GlossaryExportRow row = new GlossaryExportRow(); + row.setRecordType(GlossaryExportRow.RecordTypeKind.TERM); + row.setName("BankBranch"); + row.setGlossaryName("testBankingGlossary"); + + ByteArrayOutputStream output = new ByteArrayOutputStream(); + writer.writeXlsx(output, Collections.singletonList(row), GlossaryExportParameters.ExportMode.AUDIT); + + try (XSSFWorkbook workbook = new XSSFWorkbook(new ByteArrayInputStream(output.toByteArray()))) { + Sheet sheet = workbook.getSheetAt(0); + Row header = sheet.getRow(0); + Row data = sheet.getRow(1); + + assertEquals(header.getCell(0).getStringCellValue(), "Record Type"); + assertEquals(data.getCell(1).getStringCellValue(), "BankBranch"); + assertEquals(data.getCell(2).getStringCellValue(), "testBankingGlossary"); + } + } +} diff --git a/repository/src/test/java/org/apache/atlas/glossary/GlossarySearchBuilderTest.java b/repository/src/test/java/org/apache/atlas/glossary/GlossarySearchBuilderTest.java new file mode 100644 index 00000000000..66da6f05241 --- /dev/null +++ b/repository/src/test/java/org/apache/atlas/glossary/GlossarySearchBuilderTest.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.glossary; + +import org.apache.atlas.SortOrder; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.glossary.AtlasGlossary; +import org.apache.atlas.model.glossary.AtlasGlossaryCategory; +import org.apache.atlas.model.glossary.AtlasGlossaryTerm; +import org.apache.atlas.model.glossary.GlossarySearchParameters; +import org.apache.atlas.model.glossary.GlossarySearchResult; +import org.mockito.Mockito; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.when; +import static org.testng.Assert.assertEquals; +import static org.testng.Assert.assertNotNull; +import static org.testng.Assert.assertTrue; + +public class GlossarySearchBuilderTest { + private GlossaryService mockGlossaryService; + private GlossarySearchBuilder builder; + + @BeforeMethod + public void setUp() throws AtlasBaseException { + mockGlossaryService = Mockito.mock(GlossaryService.class); + builder = new GlossarySearchBuilder(mockGlossaryService); + + AtlasGlossary glossary = new AtlasGlossary(); + glossary.setGuid("glossary-guid-1"); + glossary.setName("BankGlossary"); + glossary.setShortDescription("Bank glossary"); + glossary.setLongDescription("Long description"); + + AtlasGlossaryTerm term = new AtlasGlossaryTerm(); + term.setGuid("term-guid-1"); + term.setName("BankBranch"); + term.setQualifiedName("BankBranch@BankGlossary"); + term.setShortDescription("Branch term"); + + AtlasGlossaryCategory category = new AtlasGlossaryCategory(); + category.setGuid("category-guid-1"); + category.setName("BankCategory"); + category.setQualifiedName("BankCategory@BankGlossary"); + category.setShortDescription("Category desc"); + + Map termInfo = new HashMap<>(); + termInfo.put(term.getGuid(), term); + + Map categoryInfo = new HashMap<>(); + categoryInfo.put(category.getGuid(), category); + + AtlasGlossary.AtlasGlossaryExtInfo extInfo = new AtlasGlossary.AtlasGlossaryExtInfo(); + extInfo.setGuid(glossary.getGuid()); + extInfo.setName(glossary.getName()); + extInfo.setShortDescription(glossary.getShortDescription()); + extInfo.setLongDescription(glossary.getLongDescription()); + extInfo.setTermInfo(termInfo); + extInfo.setCategoryInfo(categoryInfo); + + when(mockGlossaryService.getGlossaries(-1, 0, SortOrder.ASCENDING)).thenReturn(Collections.singletonList(glossary)); + when(mockGlossaryService.getDetailedGlossary(anyString())).thenReturn(extInfo); + } + + @Test + public void testSearchReturnsNestedGlossaryStructure() throws AtlasBaseException { + GlossarySearchParameters parameters = new GlossarySearchParameters(); + parameters.setLimit(25); + parameters.setOffset(0); + + GlossarySearchResult result = builder.search(parameters); + + assertNotNull(result); + assertEquals(result.getApproximateCount(), 2); + assertEquals(result.getGlossary().size(), 1); + assertEquals(result.getGlossary().get(0).getName(), "BankGlossary"); + assertEquals(result.getGlossary().get(0).getTerms().size(), 1); + assertEquals(result.getGlossary().get(0).getCategories().size(), 1); + assertEquals(result.getGlossary().get(0).getTerms().get(0).getName(), "BankBranch"); + } + + @Test + public void testSearchFiltersByGlossaryTypeTerm() throws AtlasBaseException { + GlossarySearchParameters parameters = new GlossarySearchParameters(); + parameters.setGlossaryType(GlossarySearchParameters.GlossaryType.TERM); + + GlossarySearchResult result = builder.search(parameters); + + assertEquals(result.getApproximateCount(), 1); + assertTrue(result.getGlossary().get(0).getCategories().isEmpty()); + assertEquals(result.getGlossary().get(0).getTerms().size(), 1); + } + + @Test + public void testSearchPaginatesRows() throws AtlasBaseException { + GlossarySearchParameters parameters = new GlossarySearchParameters(); + parameters.setLimit(1); + parameters.setOffset(0); + parameters.setSortBy("name"); + parameters.setSortOrder(SortOrder.ASCENDING); + + GlossarySearchResult result = builder.search(parameters); + + assertEquals(result.getApproximateCount(), 2); + assertEquals(result.getGlossary().size(), 1); + assertEquals(result.getGlossary().get(0).getTerms().size() + result.getGlossary().get(0).getCategories().size(), 1); + } +} diff --git a/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java b/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java index f8cac1c3cdf..d62aff6b9e5 100644 --- a/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java +++ b/repository/src/test/java/org/apache/atlas/glossary/GlossaryServiceTest.java @@ -267,7 +267,7 @@ public void setupSampleGlossary() { // Create a real instance of GlossaryService with mocked dependencies mockedDataAccess = mock(DataAccess.class); mockedGlossaryDTO = mock(AtlasGlossaryDTO.class); - mockedGlossaryService = new GlossaryService(mockedDataAccess, null, null, null, mockedGlossaryDTO); + mockedGlossaryService = new GlossaryService(mockedDataAccess, null, null, null, mockedGlossaryDTO, null); } @Test(groups = "Glossary.CREATE") diff --git a/server-api/pom.xml b/server-api/pom.xml index 0f6caf3471e..5d8e7bb0cb3 100644 --- a/server-api/pom.xml +++ b/server-api/pom.xml @@ -37,6 +37,7 @@ + org.apache.atlas atlas-client-v1 @@ -64,15 +65,15 @@ + + org.testng + testng + org.mockito mockito-core - ${mockito.version} - - - org.testng - testng + test diff --git a/webapp/pom.xml b/webapp/pom.xml index 968fef62019..372bc6337ed 100755 --- a/webapp/pom.xml +++ b/webapp/pom.xml @@ -339,6 +339,12 @@ httpclient + + + org.apache.logging.log4j + log4j-api + + org.apache.zookeeper zookeeper diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java index a9b7196674e..b01bd740014 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java @@ -20,21 +20,30 @@ import com.sun.jersey.core.header.FormDataContentDisposition; import com.sun.jersey.multipart.FormDataParam; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; import org.apache.atlas.SortOrder; import org.apache.atlas.annotation.Timed; import org.apache.atlas.bulkimport.BulkImportResponse; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.glossary.GlossaryExportDownloadTask; +import org.apache.atlas.glossary.GlossaryExportParameterMapper; import org.apache.atlas.glossary.GlossaryService; import org.apache.atlas.glossary.GlossaryTermUtils; +import org.apache.atlas.model.discovery.AtlasSearchResultDownloadStatus; import org.apache.atlas.model.glossary.AtlasGlossary; import org.apache.atlas.model.glossary.AtlasGlossaryCategory; import org.apache.atlas.model.glossary.AtlasGlossaryTerm; +import org.apache.atlas.model.glossary.GlossaryExportParameters; +import org.apache.atlas.model.glossary.GlossarySearchParameters; +import org.apache.atlas.model.glossary.GlossarySearchResult; import org.apache.atlas.model.glossary.relations.AtlasRelatedCategoryHeader; import org.apache.atlas.model.glossary.relations.AtlasRelatedTermHeader; import org.apache.atlas.model.instance.AtlasRelatedObjectId; +import org.apache.atlas.utils.AtlasJson; import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.atlas.web.util.Servlets; import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.springframework.stereotype.Service; @@ -50,9 +59,15 @@ import javax.ws.rs.Produces; import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; +import java.io.File; +import java.io.IOException; import java.io.InputStream; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Objects; @@ -1017,6 +1032,93 @@ public StreamingOutput produceTemplate() { return outputStream -> outputStream.write(GlossaryTermUtils.getGlossaryTermHeaders().getBytes()); } + /** + * Search glossary terms and categories for UI display (JSON). + * Uses the same server-side filter semantics as async glossary export ({@link GlossaryExportParameters}). + * Pagination and sort apply to the flat term/category row set, then results are grouped by glossary. + * + * @param parameters limit, offset, sort, and filter criteria + * @return nested glossary structure with terms and/or categories + * @throws AtlasBaseException + * @HTTP 200 On successful search + */ + @POST + @Path("/search") + @Timed + public GlossarySearchResult searchGlossary(GlossarySearchParameters parameters) throws AtlasBaseException { + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "GlossaryREST.searchGlossary()"); + } + + return glossaryService.searchGlossary(parameters); + } finally { + AtlasPerfTracer.log(perf); + } + } + + /** + * Queue a glossary export file (CSV or XLSX), mirroring Basic Search + * {@code POST /api/atlas/v2/search/basic/download/create_file}. + * Accepts either {@code exportParameters} or the same JSON body as {@link #searchGlossary(GlossarySearchParameters)} + */ + @POST + @Path("/download/create_file") + @Timed + public void glossaryExportCreateFile(Map parameterMap) throws AtlasBaseException { + glossaryExportCreateFileInternal(parameterMap); + } + + /** + * Alias for {@link #glossaryExportCreateFile(Map)} — supports UI paths that call {@code /glossary/create_file}. + */ + @POST + @Path("/create_file") + @Timed + public void glossaryExportCreateFileAlias(Map parameterMap) throws AtlasBaseException { + glossaryExportCreateFileInternal(parameterMap); + } + + /** + * Returns pending and completed glossary export download records for the current user. + * Response shape matches Basic Search {@code GET /api/atlas/v2/search/download/status}. + */ + @GET + @Path("/download/status") + @Timed + public AtlasSearchResultDownloadStatus getGlossaryExportDownloadStatus() throws IOException { + return glossaryService.getGlossaryExportDownloadStatus(); + } + + /** + * Download a completed glossary export file for the current user. + */ + @GET + @Path("/download/{filename}") + @Produces(MediaType.APPLICATION_OCTET_STREAM) + @Timed + public Response downloadGlossaryExportFile(@PathParam("filename") String fileName) { + File dir = new File(GlossaryExportDownloadTask.DOWNLOAD_DIR_PATH, RequestContext.getCurrentUser()); + File exportFile = new File(dir, fileName); + + if (!exportFile.exists()) { + return Response.noContent().build(); + } + + return Response.ok(exportFile) + .header("Content-Disposition", "attachment; filename=\"" + fileName + "\"") + .build(); + } + + @POST + @Path("/export") + @Timed + public Response exportGlossaryRemoved() { + return Response.status(Response.Status.NOT_FOUND).build(); + } + /** * Upload glossary file for creating AtlasGlossaryTerms in bulk * @@ -1063,4 +1165,49 @@ private SortOrder toSortOrder(final String sort) { return ret; } + + private String buildExportFileName(GlossaryExportParameters parameters) { + GlossaryExportParameters.ExportFormat format = parameters.getFormat(); + String extension = format == GlossaryExportParameters.ExportFormat.XLSX ? "xlsx" : "csv"; + String timestamp = StringUtils.isNotEmpty(parameters.getExportTimestamp()) + ? parameters.getExportTimestamp() + : new SimpleDateFormat("yyyy-MM-dd_HH-mm-ss").format(new Date()); + String user = RequestContext.getCurrentUser(); + + return user + "_GLOSSARY_EXPORT_" + timestamp + "." + extension; + } + + private void glossaryExportCreateFileInternal(Map parameterMap) throws AtlasBaseException { + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "GlossaryREST.glossaryExportCreateFile(" + parameterMap + ")"); + } + + GlossaryExportParameters parameters = GlossaryExportParameterMapper.fromCreateFileRequest(parameterMap); + validateExportParameters(parameters); + + Map taskParams = populateGlossaryExportTaskParams(parameters); + glossaryService.createAndQueueGlossaryExportDownloadTask(taskParams); + } finally { + AtlasPerfTracer.log(perf); + } + } + + private void validateExportParameters(GlossaryExportParameters parameters) throws AtlasBaseException { + if (parameters == null || parameters.getFormat() == null) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Export format (CSV or XLSX) is required"); + } + } + + private Map populateGlossaryExportTaskParams(GlossaryExportParameters parameters) { + Map taskParams = new HashMap<>(); + String fileName = buildExportFileName(parameters); + + taskParams.put(GlossaryExportDownloadTask.EXPORT_PARAMETERS_JSON_KEY, AtlasJson.toJson(parameters)); + taskParams.put(GlossaryExportDownloadTask.FILE_NAME_KEY, fileName); + + return taskParams; + } } diff --git a/webapp/src/test/java/org/apache/atlas/web/rest/GlossaryRESTTest.java b/webapp/src/test/java/org/apache/atlas/web/rest/GlossaryRESTTest.java index 00233cf5d78..3836b189b44 100644 --- a/webapp/src/test/java/org/apache/atlas/web/rest/GlossaryRESTTest.java +++ b/webapp/src/test/java/org/apache/atlas/web/rest/GlossaryRESTTest.java @@ -21,9 +21,13 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.glossary.GlossaryService; import org.apache.atlas.glossary.GlossaryTermUtils; +import org.apache.atlas.model.discovery.AtlasSearchResultDownloadStatus; import org.apache.atlas.model.glossary.AtlasGlossary; import org.apache.atlas.model.glossary.AtlasGlossaryCategory; import org.apache.atlas.model.glossary.AtlasGlossaryTerm; +import org.apache.atlas.model.glossary.GlossaryExportParameters; +import org.apache.atlas.model.glossary.GlossarySearchParameters; +import org.apache.atlas.model.glossary.GlossarySearchResult; import org.apache.atlas.model.glossary.relations.AtlasGlossaryHeader; import org.apache.atlas.model.glossary.relations.AtlasRelatedCategoryHeader; import org.apache.atlas.model.glossary.relations.AtlasRelatedTermHeader; @@ -893,4 +897,70 @@ public void testImportGlossaryData() throws Exception { assertEquals(expectedResponse, result); verify(mockGlossaryService).importGlossaryData(inputStream, testFileName); } + + @Test + public void testGlossaryExportCreateFile_Success() throws Exception { + Map body = new HashMap<>(); + body.put("limit", 0); + body.put("offset", 0); + body.put("sortBy", "glossaryName"); + body.put("sortOrder", "DESCENDING"); + body.put("excludeDeleted", true); + + glossaryREST.glossaryExportCreateFile(body); + + verify(mockGlossaryService).createAndQueueGlossaryExportDownloadTask(any(Map.class)); + } + + @Test + public void testGlossaryExportCreateFileAlias_Success() throws Exception { + GlossaryExportParameters parameters = new GlossaryExportParameters(); + parameters.setFormat(GlossaryExportParameters.ExportFormat.XLSX); + + Map body = new HashMap<>(); + body.put("exportParameters", parameters); + + glossaryREST.glossaryExportCreateFileAlias(body); + + verify(mockGlossaryService).createAndQueueGlossaryExportDownloadTask(any(Map.class)); + } + + @Test(expectedExceptions = AtlasBaseException.class) + public void testGlossaryExportCreateFile_MissingFormat() throws Exception { + Map body = new HashMap<>(); + body.put("exportParameters", new HashMap<>()); + + glossaryREST.glossaryExportCreateFile(body); + } + + @Test + public void testGetGlossaryExportDownloadStatus_Success() throws Exception { + AtlasSearchResultDownloadStatus expected = new AtlasSearchResultDownloadStatus(); + when(mockGlossaryService.getGlossaryExportDownloadStatus()).thenReturn(expected); + + AtlasSearchResultDownloadStatus result = glossaryREST.getGlossaryExportDownloadStatus(); + + assertNotNull(result); + verify(mockGlossaryService).getGlossaryExportDownloadStatus(); + } + + @Test + public void testSearchGlossary_Success() throws Exception { + GlossarySearchParameters parameters = new GlossarySearchParameters(); + parameters.setLimit(25); + parameters.setOffset(0); + parameters.setSortBy("name"); + parameters.setSortOrder(SortOrder.ASCENDING); + + GlossarySearchResult expected = new GlossarySearchResult(); + expected.setApproximateCount(1); + + when(mockGlossaryService.searchGlossary(parameters)).thenReturn(expected); + + GlossarySearchResult result = glossaryREST.searchGlossary(parameters); + + assertNotNull(result); + assertEquals(result.getApproximateCount(), 1); + verify(mockGlossaryService).searchGlossary(parameters); + } } From a1606eb50cb1f11c8bd68cae6fea63fb7513f94a Mon Sep 17 00:00:00 2001 From: Umesh Patil Date: Sun, 26 Jul 2026 11:39:29 +0530 Subject: [PATCH 2/3] ATLAS-5313: Remove unused POST /glossary/export stub. --- .../main/java/org/apache/atlas/web/rest/GlossaryREST.java | 7 ------- 1 file changed, 7 deletions(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java index 7e80683a82b..7ac06a0293b 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/GlossaryREST.java @@ -1112,13 +1112,6 @@ public Response downloadGlossaryExportFile(@PathParam("filename") String fileNam .build(); } - @POST - @Path("/export") - @Timed - public Response exportGlossaryRemoved() { - return Response.status(Response.Status.NOT_FOUND).build(); - } - /** * Upload glossary file for creating AtlasGlossaryTerms in bulk * From b78aeb16d6eb55c2b4338965d8d32985c52ddba3 Mon Sep 17 00:00:00 2001 From: Umesh Patil Date: Thu, 6 Aug 2026 12:15:27 +0530 Subject: [PATCH 3/3] ATLAS-5313: Address review feedback by reverting unrelated changes and deduplicating glossary search load --- hbase-shaded-client-fixed/pom.xml | 22 ++++++++++--- repository/pom.xml | 6 ---- .../glossary/GlossaryExportDataCollector.java | 33 +++++++++++++++++-- .../atlas/glossary/GlossarySearchBuilder.java | 21 ++++-------- .../repository/graph/AtlasGraphProvider.java | 2 +- .../glossary/GlossarySearchBuilderTest.java | 13 ++++++++ 6 files changed, 68 insertions(+), 29 deletions(-) diff --git a/hbase-shaded-client-fixed/pom.xml b/hbase-shaded-client-fixed/pom.xml index 6ffe4a84e50..09e75bae4aa 100644 --- a/hbase-shaded-client-fixed/pom.xml +++ b/hbase-shaded-client-fixed/pom.xml @@ -73,14 +73,28 @@ - + org.apache.maven.plugins maven-jar-plugin 2.4 - - ${project.build.directory}/unpacked - + + + default-jar + none + + + repack-fixed-hbase-shaded-client + + jar + + package + + ${project.build.directory}/unpacked + hbase-shaded-client-${hbase.version}-hadoop3-fixed + + + diff --git a/repository/pom.xml b/repository/pom.xml index 319ab27bd75..8f00deb1a43 100644 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -80,12 +80,6 @@ ${commons-codec.version} - - commons-lang - commons-lang - 2.6 - - joda-time joda-time diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDataCollector.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDataCollector.java index cb71a44c5c3..8c8fd4675d4 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDataCollector.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryExportDataCollector.java @@ -33,7 +33,9 @@ import java.util.ArrayList; import java.util.Collections; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; public class GlossaryExportDataCollector { private static final Logger LOG = LoggerFactory.getLogger(GlossaryExportDataCollector.class); @@ -44,9 +46,32 @@ public GlossaryExportDataCollector(GlossaryService glossaryService) { this.glossaryService = glossaryService; } + public static class CollectResult { + private final List rows; + private final Map extInfoByGuid; + + public CollectResult(List rows, Map extInfoByGuid) { + this.rows = rows; + this.extInfoByGuid = extInfoByGuid; + } + + public List getRows() { + return rows; + } + + public Map getExtInfoByGuid() { + return extInfoByGuid; + } + } + public List collect(GlossaryExportParameters parameters) throws AtlasBaseException { - List ret = new ArrayList<>(); - List glossaryGuids = resolveGlossaryGuids(parameters); + return collectWithExtInfo(parameters).getRows(); + } + + public CollectResult collectWithExtInfo(GlossaryExportParameters parameters) throws AtlasBaseException { + List ret = new ArrayList<>(); + Map extInfoByGuid = new LinkedHashMap<>(); + List glossaryGuids = resolveGlossaryGuids(parameters); LOG.debug("Collecting glossary export rows for {} glossaries", glossaryGuids.size()); @@ -57,6 +82,8 @@ public List collect(GlossaryExportParameters parameters) thro continue; } + extInfoByGuid.put(glossaryGuid, glossaryExt); + String glossaryName = glossaryExt.getName(); if (MapUtils.isNotEmpty(glossaryExt.getTermInfo())) { @@ -74,7 +101,7 @@ public List collect(GlossaryExportParameters parameters) thro LOG.debug("Collected {} glossary export rows before filtering", ret.size()); - return ret; + return new CollectResult(ret, extInfoByGuid); } private List resolveGlossaryGuids(GlossaryExportParameters parameters) throws AtlasBaseException { diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossarySearchBuilder.java b/repository/src/main/java/org/apache/atlas/glossary/GlossarySearchBuilder.java index 4f1ce91d77a..b816a3b5c39 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossarySearchBuilder.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossarySearchBuilder.java @@ -58,7 +58,8 @@ public GlossarySearchResult search(GlossarySearchParameters parameters) throws A GlossaryExportDataCollector collector = new GlossaryExportDataCollector(glossaryService); GlossaryExportFilter filter = new GlossaryExportFilter(); - List rows = filter.apply(collector.collect(exportParameters), exportParameters); + GlossaryExportDataCollector.CollectResult collected = collector.collectWithExtInfo(exportParameters); + List rows = filter.apply(collected.getRows(), exportParameters); rows = filterByGlossaryName(rows, parameters); sort(rows, parameters); @@ -67,7 +68,7 @@ public GlossarySearchResult search(GlossarySearchParameters parameters) throws A LOG.debug("Glossary search matched {} rows, returning page of {}", totalCount, pageRows.size()); - return buildResult(pageRows, totalCount, parameters); + return buildResult(pageRows, totalCount, parameters, collected.getExtInfoByGuid()); } private GlossaryExportParameters toExportParameters(GlossarySearchParameters parameters) { @@ -166,8 +167,8 @@ private List paginate(List rows, GlossaryS return new ArrayList<>(rows.subList(offset, end)); } - private GlossarySearchResult buildResult(List pageRows, long totalCount, GlossarySearchParameters parameters) - throws AtlasBaseException { + private GlossarySearchResult buildResult(List pageRows, long totalCount, GlossarySearchParameters parameters, + Map extInfoByGuid) throws AtlasBaseException { GlossarySearchResult result = new GlossarySearchResult(); result.setApproximateCount(totalCount); @@ -177,13 +178,12 @@ private GlossarySearchResult buildResult(List pageRows, long } Map glossaryByGuid = new LinkedHashMap<>(); - Map extByGuid = new LinkedHashMap<>(); boolean includeTerms = parameters.getGlossaryType() != GlossarySearchParameters.GlossaryType.CATEGORY; boolean includeCategories = parameters.getGlossaryType() != GlossarySearchParameters.GlossaryType.TERM; for (GlossaryExportRow row : pageRows) { - AtlasGlossary.AtlasGlossaryExtInfo ext = extByGuid.computeIfAbsent(row.getGlossaryGuid(), this::loadExtInfoSafely); + AtlasGlossary.AtlasGlossaryExtInfo ext = extInfoByGuid.get(row.getGlossaryGuid()); if (ext == null) { continue; @@ -211,15 +211,6 @@ private GlossarySearchResult buildResult(List pageRows, long return result; } - private AtlasGlossary.AtlasGlossaryExtInfo loadExtInfoSafely(String glossaryGuid) { - try { - return glossaryService.getDetailedGlossary(glossaryGuid); - } catch (AtlasBaseException e) { - LOG.warn("Failed to load detailed glossary for guid {}", glossaryGuid, e); - return null; - } - } - private GlossarySearchResult.GlossaryDetail buildGlossaryDetail(AtlasGlossary.AtlasGlossaryExtInfo ext) { GlossarySearchResult.GlossaryDetail detail = new GlossarySearchResult.GlossaryDetail(); diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java b/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java index 4b5baf61912..51ae8b3c8a3 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/AtlasGraphProvider.java @@ -144,7 +144,7 @@ private static Integer getMaxRetryCount() { private static Long getRetrySleepTime() { initApplicationProperties(); - return (applicationProperties == null) ? 30000L : applicationProperties.getLong(GRAPH_REPOSITORY_RETRY_SLEEPTIME, 30000L); + return (applicationProperties == null) ? 30000 : applicationProperties.getLong(GRAPH_REPOSITORY_RETRY_SLEEPTIME, 30000); } private static void initApplicationProperties() { diff --git a/repository/src/test/java/org/apache/atlas/glossary/GlossarySearchBuilderTest.java b/repository/src/test/java/org/apache/atlas/glossary/GlossarySearchBuilderTest.java index 66da6f05241..b144bdb25ec 100644 --- a/repository/src/test/java/org/apache/atlas/glossary/GlossarySearchBuilderTest.java +++ b/repository/src/test/java/org/apache/atlas/glossary/GlossarySearchBuilderTest.java @@ -33,6 +33,8 @@ import java.util.Map; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; import static org.mockito.Mockito.when; import static org.testng.Assert.assertEquals; import static org.testng.Assert.assertNotNull; @@ -126,4 +128,15 @@ public void testSearchPaginatesRows() throws AtlasBaseException { assertEquals(result.getGlossary().size(), 1); assertEquals(result.getGlossary().get(0).getTerms().size() + result.getGlossary().get(0).getCategories().size(), 1); } + + @Test + public void testSearchLoadsDetailedGlossaryOncePerRequest() throws AtlasBaseException { + GlossarySearchParameters parameters = new GlossarySearchParameters(); + parameters.setLimit(25); + parameters.setOffset(0); + + builder.search(parameters); + + verify(mockGlossaryService, times(1)).getDetailedGlossary(anyString()); + } }