Skip to content

ATLAS-4988: BusinessMetadata with attribute of data type Array takes time to Delete. #490

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
UmeshPatil-1 wants to merge 3 commits into
base: master
Choose a base branch
from
+95 −20
Open

ATLAS-4988: BusinessMetadata with attribute of data type Array takes time to Delete. #490

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fbf8033
ATLAS_4988: Resolve slow deletion issue of Business Metadata with arr…
UmeshPatil-1 Dec 27, 2025
5bf649f
ATLAS-4988 : Replaced manual iteration with graph query, added error …
UmeshPatil-1 Dec 30, 2025
8478f2b
ATLAS-4988 : Removed entity-type filter from graph query,Ensured pare…
UmeshPatil-1 Jan 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
113 changes: 94 additions & 19 deletions ...main/java/org/apache/atlas/repository/store/graph/v2/AtlasBusinessMetadataDefStoreV2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,14 @@
import org.apache.atlas.model.typedef.AtlasBusinessMetadataDef;
import org.apache.atlas.model.typedef.AtlasStructDef;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.type.AtlasBusinessMetadataType;
import org.apache.atlas.type.AtlasStructType;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.type.AtlasEntityType;
import org.apache.atlas.type.AtlasArrayType;
import org.apache.atlas.type.AtlasTypeRegistry;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.utils.AtlasJson;
Expand All @@ -50,6 +54,7 @@
import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.HashSet;

import static org.apache.atlas.model.typedef.AtlasBusinessMetadataDef.ATTR_OPTION_APPLICABLE_ENTITY_TYPES;

Expand All @@ -58,11 +63,14 @@ public class AtlasBusinessMetadataDefStoreV2 extends AtlasAbstractDefStoreV2<Atl

private final EntityDiscoveryService entityDiscoveryService;

private final AtlasGraph graph;

@Inject
public AtlasBusinessMetadataDefStoreV2(AtlasTypeDefGraphStoreV2 typeDefStore, AtlasTypeRegistry typeRegistry, EntityDiscoveryService entityDiscoveryService) {
public AtlasBusinessMetadataDefStoreV2(AtlasTypeDefGraphStoreV2 typeDefStore, AtlasTypeRegistry typeRegistry, EntityDiscoveryService entityDiscoveryService, AtlasGraph graph) {
super(typeDefStore, typeRegistry);

this.entityDiscoveryService = entityDiscoveryService;
this.graph = graph;
}

@Override
Expand Down Expand Up @@ -363,44 +371,111 @@ private AtlasBusinessMetadataDef toBusinessMetadataDef(AtlasVertex vertex) throw
private void checkBusinessMetadataRef(String typeName) throws AtlasBaseException {
AtlasBusinessMetadataDef businessMetadataDef = typeRegistry.getBusinessMetadataDefByName(typeName);

if (businessMetadataDef != null) {
List<AtlasStructDef.AtlasAttributeDef> attributeDefs = businessMetadataDef.getAttributeDefs();
if (businessMetadataDef == null || CollectionUtils.isEmpty(businessMetadataDef.getAttributeDefs())) {
return;
}

for (AtlasStructDef.AtlasAttributeDef attributeDef : attributeDefs) {
String qualifiedName = AtlasStructType.AtlasAttribute.getQualifiedAttributeName(businessMetadataDef, attributeDef.getName());
String vertexPropertyName = AtlasStructType.AtlasAttribute.generateVertexPropertyName(businessMetadataDef, attributeDef, qualifiedName);
Set<String> applicableTypes = AtlasJson.fromJson(attributeDef.getOption(AtlasBusinessMetadataDef.ATTR_OPTION_APPLICABLE_ENTITY_TYPES), Set.class);
for (AtlasStructDef.AtlasAttributeDef attributeDef : businessMetadataDef.getAttributeDefs()) {
Copy link
Contributor

@sheetalshah1007 sheetalshah1007 Dec 29, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Extract the attribute iteration logic in checkBusinessMetadataRef to a private method (e.g., validateAttributeReferences) for better readability.

Copy link
Author

@UmeshPatil-1 UmeshPatil-1 Dec 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done

validateAttributeReferences(businessMetadataDef, attributeDef);
}
}

if (CollectionUtils.isNotEmpty(applicableTypes) && isBusinessAttributePresent(vertexPropertyName, applicableTypes)) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_REFERENCES, typeName);
}
}
private void validateAttributeReferences(AtlasBusinessMetadataDef bmDef, AtlasStructDef.AtlasAttributeDef attributeDef) throws AtlasBaseException {
String applicableTypesStr = attributeDef.getOption(ATTR_OPTION_APPLICABLE_ENTITY_TYPES);
Set<String> applicableTypes = StringUtils.isBlank(applicableTypesStr) ? null : AtlasJson.fromJson(applicableTypesStr, Set.class);

if (CollectionUtils.isEmpty(applicableTypes)) {
return;
}

Set<String> allApplicableTypes = getApplicableTypesWithSubTypes(applicableTypes);
String qualifiedName = AtlasStructType.AtlasAttribute.getQualifiedAttributeName(bmDef, attributeDef.getName());
String vertexPropertyName = AtlasStructType.AtlasAttribute.generateVertexPropertyName(bmDef, attributeDef, qualifiedName);

boolean isPresent;
long startTime = System.currentTimeMillis();

if (isArrayAttribute(attributeDef)) {
isPresent = isBusinessAttributePresentInGraph(vertexPropertyName, allApplicableTypes);
} else {
isPresent = isBusinessAttributePresent(qualifiedName, allApplicableTypes);
}

if (LOG.isDebugEnabled()) {
LOG.info("Reference check for attribute {} took {} ms. Found: {}",
attributeDef.getName(), (System.currentTimeMillis() - startTime), isPresent);
}

if (isPresent) {
throw new AtlasBaseException(AtlasErrorCode.TYPE_HAS_REFERENCES, bmDef.getName());
}
}

private boolean isBusinessAttributePresent(String attrName, Set<String> applicableTypes) throws AtlasBaseException {
SearchParameters.FilterCriteria criteria = new SearchParameters.FilterCriteria();

criteria.setAttributeName(attrName);
criteria.setOperator(SearchParameters.Operator.NOT_EMPTY);

SearchParameters.FilterCriteria entityFilters = new SearchParameters.FilterCriteria();

entityFilters.setCondition(SearchParameters.FilterCriteria.Condition.OR);
entityFilters.setCriterion(Collections.singletonList(criteria));

SearchParameters searchParameters = new SearchParameters();

searchParameters.setTypeName(String.join(SearchContext.TYPENAME_DELIMITER, applicableTypes));
searchParameters.setExcludeDeletedEntities(true);
searchParameters.setExcludeDeletedEntities(false);
searchParameters.setIncludeSubClassifications(false);
searchParameters.setEntityFilters(entityFilters);
searchParameters.setAttributes(Collections.singleton(attrName));
searchParameters.setOffset(0);
searchParameters.setLimit(1);

AtlasSearchResult atlasSearchResult = entityDiscoveryService.searchWithParameters(searchParameters);
return atlasSearchResult != null && CollectionUtils.isNotEmpty(atlasSearchResult.getEntities());
}

private boolean isBusinessAttributePresentInGraph(String vertexPropertyName, Set<String> allApplicableTypes) {
if (graph == null || CollectionUtils.isEmpty(allApplicableTypes)) {
return false;
}

try {
Iterable<AtlasVertex> vertices = graph.query()
.has(vertexPropertyName, AtlasGraphQuery.ComparisionOperator.NOT_EQUAL, (Object) null)
.vertices();

return CollectionUtils.isNotEmpty(atlasSearchResult.getEntities());
if (vertices != null) {
Iterator<AtlasVertex> iterator = vertices.iterator();
if (iterator.hasNext()) {
if (LOG.isDebugEnabled()) {
LOG.info("Found ARRAY BM reference for property {}",vertexPropertyName);
}
return true;
}
}
} catch (Exception e) {
LOG.error("Error occurred while querying graph for references of property: {}", vertexPropertyName, e);
return true;
}
return false;
}

private boolean isArrayAttribute(AtlasStructDef.AtlasAttributeDef attrDef) throws AtlasBaseException {
AtlasType type = typeRegistry.getType(attrDef.getTypeName());
return type instanceof AtlasArrayType;
}

private Set<String> getApplicableTypesWithSubTypes(Set<String> applicableTypes) throws AtlasBaseException {
Set<String> allTypes = new HashSet<>();

for (String typeName : applicableTypes) {
AtlasType type = typeRegistry.getType(typeName);

if (type instanceof AtlasEntityType) {
AtlasEntityType entityType = (AtlasEntityType) type;
allTypes.add(entityType.getTypeName());
allTypes.addAll(entityType.getAllSubTypes());
} else {
allTypes.add(typeName);
}
}
return allTypes;
}
}
}
2 changes: 1 addition & 1 deletion ...ry/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasTypeDefGraphStoreV2.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ protected AtlasDefStore<AtlasRelationshipDef> getRelationshipDefStore(AtlasTypeR

@Override
protected AtlasDefStore<AtlasBusinessMetadataDef> getBusinessMetadataDefStore(AtlasTypeRegistry typeRegistry) {
return new AtlasBusinessMetadataDefStoreV2(this, typeRegistry, this.entityDiscoveryService);
return new AtlasBusinessMetadataDefStoreV2(this, typeRegistry, this.entityDiscoveryService, this.atlasGraph);
}

@Override
Expand Down