fix(bulk-ops): exclude soft-deleted entities from column grid (#28653) [1.13]

openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/ColumnBulkUpdateIT.java

@@ -14,6 +14,7 @@
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import org.awaitility.Awaitility;
 import org.junit.jupiter.api.Test;
@@ -29,6 +30,7 @@
 import org.openmetadata.it.util.TestNamespaceExtension;
 import org.openmetadata.schema.api.classification.CreateClassification;
 import org.openmetadata.schema.api.classification.CreateTag;
+import org.openmetadata.schema.api.data.ColumnGridResponse;
 import org.openmetadata.schema.api.data.CreateDashboardDataModel;
 import org.openmetadata.schema.api.data.CreateGlossary;
 import org.openmetadata.schema.api.data.CreateGlossaryTerm;
@@ -47,6 +49,7 @@
 import org.openmetadata.schema.type.DataModelType;
 import org.openmetadata.schema.type.TagLabel;
 import org.openmetadata.sdk.client.OpenMetadataClient;
+import org.openmetadata.sdk.network.HttpMethod;
 
 /**
  * Integration tests for Column bulk update API.
@@ -1924,6 +1927,61 @@ private Table getTableWithColumns(String tableId) throws Exception {
     return OBJECT_MAPPER.readValue(response.body(), Table.class);
   }
 
+  @Test
+  void test_columnGrid_excludesSoftDeletedTables(TestNamespace ns) throws Exception {
+    OpenMetadataClient client = SdkClients.adminClient();
+    DatabaseService service = DatabaseServiceTestFactory.createPostgres(ns);
+    DatabaseSchema schema = DatabaseSchemaTestFactory.createSimple(ns, service);
+
+    String colName = ns.prefix("soft_del_col");
+    Column col =
+        new Column().withName(colName).withDataType(ColumnDataType.VARCHAR).withDataLength(255);
+    CreateTable createTable =
+        new CreateTable()
+            .withName(ns.prefix("soft_del_table"))
+            .withDatabaseSchema(schema.getFullyQualifiedName())
+            .withColumns(List.of(col));
+    Table table = client.tables().create(createTable);
+
+    Awaitility.await("Column must appear in grid before soft-delete")
+        .atMost(30, TimeUnit.SECONDS)
+        .pollInterval(2, TimeUnit.SECONDS)
+        .untilAsserted(
+            () -> {
+              ColumnGridResponse before = getColumnGrid(client, service.getName());
+              assertNotNull(before.getColumns());
+              assertTrue(
+                  before.getColumns().stream().anyMatch(c -> c.getColumnName().equals(colName)),
+                  "Column must be visible in grid before soft-delete");
+            });
+
+    client.tables().delete(table.getId().toString(), Map.of());
+
+    Awaitility.await("Soft-deleted table columns must not appear in grid")
+        .atMost(30, TimeUnit.SECONDS)
+        .pollInterval(2, TimeUnit.SECONDS)
+        .untilAsserted(
+            () -> {
+              ColumnGridResponse after = getColumnGrid(client, service.getName());
+              assertNotNull(after.getColumns());
+              assertFalse(
+                  after.getColumns().stream().anyMatch(c -> c.getColumnName().equals(colName)),
+                  "Column from soft-deleted table must not appear in column grid");
+            });
+  }
+
+  private ColumnGridResponse getColumnGrid(OpenMetadataClient client, String serviceName)
+      throws Exception {
+    String response =
+        client
+            .getHttpClient()
+            .executeForString(
+                HttpMethod.GET,
+                "/v1/columns/grid?entityTypes=table&serviceName=" + serviceName,
+                null);
+    return OBJECT_MAPPER.readValue(response, ColumnGridResponse.class);
+  }
+
   private DashboardDataModel getDashboardDataModelWithColumns(String dataModelId) throws Exception {
     HttpRequest request =
         HttpRequest.newBuilder()

openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchColumnAggregator.java

@@ -448,6 +448,7 @@ private Query buildTagFilterQuery(ColumnAggregationRequest request, String colum
 
     String columnFieldPath = columnNameKeyword.replace(".name.keyword", "");
     boolBuilder.filter(Query.of(q -> q.exists(e -> e.field(columnFieldPath))));
+    boolBuilder.filter(Query.of(q -> q.term(t -> t.field("deleted").value(false))));
 
     addEntityTypeFilter(boolBuilder, request);
     addServiceFilter(boolBuilder, request);
@@ -530,6 +531,7 @@ private Query buildFilters(
 
     String columnFieldPath = columnNameKeyword.replace(".name.keyword", "");
     boolBuilder.filter(Query.of(q -> q.exists(e -> e.field(columnFieldPath))));
+    boolBuilder.filter(Query.of(q -> q.term(t -> t.field("deleted").value(false))));
 
     addEntityTypeFilter(boolBuilder, request);
     addServiceFilter(boolBuilder, request);

openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchColumnAggregator.java

@@ -367,6 +367,7 @@ private Query buildTagFilterQuery(ColumnAggregationRequest request) {
     BoolQuery.Builder boolBuilder = new BoolQuery.Builder();
 
     boolBuilder.filter(Query.of(q -> q.exists(e -> e.field("columns"))));
+    boolBuilder.filter(Query.of(q -> q.term(t -> t.field("deleted").value(FieldValue.of(false)))));
 
     addEntityTypeFilter(boolBuilder, request);
     addServiceFilter(boolBuilder, request);
@@ -416,6 +417,7 @@ private Query buildFilters(
     BoolQuery.Builder boolBuilder = new BoolQuery.Builder();
 
     boolBuilder.filter(Query.of(q -> q.exists(e -> e.field("columns"))));
+    boolBuilder.filter(Query.of(q -> q.term(t -> t.field("deleted").value(FieldValue.of(false)))));
 
     addEntityTypeFilter(boolBuilder, request);
     addServiceFilter(boolBuilder, request);