feat(ingestion): add ability to exclude archived mode reports.

.github/workflows/metadata-io.yml

@@ -34,7 +34,7 @@ env:
 
 jobs:
   setup:
-    runs-on: ${{ vars.DEPOT_PROJECT_ID != '' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
+    runs-on: ${{ vars.DEPOT_PROJECT_ID != '' && 'depot-ubuntu-latest-2' || 'ubuntu-latest' }}
     outputs:
       frontend_change: ${{ steps.ci-optimize.outputs.frontend-change == 'true' || github.event_name == 'release' }}
       ingestion_change: ${{ steps.ci-optimize.outputs.ingestion-change == 'true' || github.event_name == 'release' }}

.github/workflows/verify-quickstart-compose.yml

 run: | 
 run: | 
 run: | 
 run: | 
@@ -6,6 +6,8 @@ on:
     branches:
       - master
   pull_request:
+    branches:
+      - master
 jobs:
   verify-quickstart-compose-updated:
     name: Verify quickstart compose file is up-to-date

CLAUDE.md

@@ -67,6 +67,15 @@ Each Python module has a gradle setup similar to `metadata-ingestion/` (document
 - **MCE/MCL**: Metadata Change Events/Logs for updates
 - **Entity Registry**: YAML config defining entity-aspect relationships (`metadata-models/src/main/resources/entity-registry.yml`)
 
+### Validation Architecture
+
+**IMPORTANT**: Validation must work across all APIs (GraphQL, OpenAPI, RestLI).
+
+- **Never add validation in API-specific layers** (GraphQL resolvers, REST controllers) - this only protects one API
+- **Always implement AspectPayloadValidators** in `metadata-io/src/main/java/com/linkedin/metadata/aspect/validation/`
+- **Register as Spring beans** in `SpringStandardPluginConfiguration.java`
+- **Follow existing patterns**: See `SystemPolicyValidator.java` and `PolicyFieldTypeValidator.java` as examples
+
 ## Development Flow
 
 1. **Schema changes** in `metadata-models/` trigger code generation across all languages

build.gradle

@@ -216,7 +216,7 @@ project.ext.externalDependency = [
     'log4jApi': "org.apache.logging.log4j:log4j-api:$log4jVersion",
     'log4j12Api': "org.slf4j:log4j-over-slf4j:$slf4jVersion",
     'log4j2Api': "org.apache.logging.log4j:log4j-to-slf4j:$log4jVersion",
-    'lombok': 'org.projectlombok:lombok:1.18.30',
+    'lombok': 'org.projectlombok:lombok:1.18.42',
     'mariadbConnector': 'org.mariadb.jdbc:mariadb-java-client:2.6.0',
     'mavenArtifact': "org.apache.maven:maven-artifact:$mavenVersion",
     'mixpanel': 'com.mixpanel:mixpanel-java:1.4.4',
@@ -257,6 +257,7 @@ project.ext.externalDependency = [
     'postgresql': 'org.postgresql:postgresql:42.7.8',
     'protobuf': 'com.google.protobuf:protobuf-java:4.32.0',
     'grpcProtobuf': 'io.grpc:grpc-protobuf:1.75.0',
+    'grpcNettyShaded': 'io.grpc:grpc-netty-shaded:1.75.0',
     'rangerCommons': 'org.apache.ranger:ranger-plugins-common:2.3.0',
     'reflections': 'org.reflections:reflections:0.9.12',
     'resilience4j': 'io.github.resilience4j:resilience4j-retry:1.7.1',

datahub-frontend/app/auth/sso/oidc/OidcCallbackLogic.java

@@ -3,7 +3,6 @@
 import static auth.AuthUtils.*;
 import static com.linkedin.metadata.Constants.CORP_USER_ENTITY_NAME;
 import static com.linkedin.metadata.Constants.GROUP_MEMBERSHIP_ASPECT_NAME;
-import static org.pac4j.play.store.PlayCookieSessionStore.*;
 import static play.mvc.Results.internalServerError;
 import static utils.FrontendConstants.SSO_LOGIN;
 
@@ -50,7 +49,6 @@
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Arrays;
-import java.util.Base64;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
@@ -71,6 +69,7 @@
 import org.pac4j.core.context.Cookie;
 import org.pac4j.core.context.FrameworkParameters;
 import org.pac4j.core.context.WebContext;
+import org.pac4j.core.context.session.SessionStore;
 import org.pac4j.core.credentials.Credentials;
 import org.pac4j.core.engine.DefaultCallbackLogic;
 import org.pac4j.core.exception.http.HttpAction;
@@ -80,10 +79,10 @@
 import org.pac4j.core.profile.UserProfile;
 import org.pac4j.core.util.CommonHelper;
 import org.pac4j.core.util.Pac4jConstants;
-import org.pac4j.play.store.PlayCookieSessionStore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import play.mvc.Result;
+import utils.SerializationUtils;
 
 /**
  * This class contains the logic that is executed when an OpenID Connect Identity Provider redirects
@@ -207,21 +206,15 @@ private Pair<CallContext, Object> superPerform(
 
   private void setContextRedirectUrl(CallContext ctx) {
     WebContext context = ctx.webContext();
-    PlayCookieSessionStore sessionStore = (PlayCookieSessionStore) ctx.sessionStore();
-
-    Optional<Cookie> redirectUrl =
-        context.getRequestCookies().stream()
-            .filter(cookie -> REDIRECT_URL_COOKIE_NAME.equals(cookie.getName()))
-            .findFirst();
-    redirectUrl.ifPresent(
-        cookie ->
-            sessionStore.set(
-                context,
-                Pac4jConstants.REQUESTED_URL,
-                sessionStore
-                    .getSerializer()
-                    .deserializeFromBytes(
-                        uncompressBytes(Base64.getDecoder().decode(cookie.getValue())))));
+    SessionStore sessionStore = ctx.sessionStore();
+
+    context.getRequestCookies().stream()
+        .filter(cookie -> REDIRECT_URL_COOKIE_NAME.equals(cookie.getName()))
+        .map(Cookie::getValue)
+        .map(SerializationUtils::deserializeFoundAction)
+        .findFirst()
+        .ifPresent(
+            foundAction -> sessionStore.set(context, Pac4jConstants.REQUESTED_URL, foundAction));
   }
 
   private Result handleOidcCallback(

datahub-frontend/app/controllers/AuthenticationController.java

@@ -2,7 +2,6 @@
 
 import static auth.AuthUtils.*;
 import static org.pac4j.core.client.IndirectClient.ATTEMPTED_AUTHENTICATION_SUFFIX;
-import static org.pac4j.play.store.PlayCookieSessionStore.*;
 import static utils.FrontendConstants.FALLBACK_LOGIN;
 import static utils.FrontendConstants.GUEST_LOGIN;
 import static utils.FrontendConstants.PASSWORD_LOGIN;
@@ -28,7 +27,6 @@
 import java.net.URLEncoder;
 import java.nio.charset.StandardCharsets;
 import java.time.Duration;
-import java.util.Base64;
 import java.util.Optional;
 import javax.annotation.Nonnull;
 import javax.inject.Inject;
@@ -37,11 +35,11 @@
 import org.pac4j.core.client.Client;
 import org.pac4j.core.context.CallContext;
 import org.pac4j.core.context.WebContext;
+import org.pac4j.core.context.session.SessionStore;
 import org.pac4j.core.exception.http.FoundAction;
 import org.pac4j.core.exception.http.RedirectionAction;
 import org.pac4j.play.PlayWebContext;
 import org.pac4j.play.http.PlayHttpActionAdapter;
-import org.pac4j.play.store.PlayCookieSessionStore;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import play.data.validation.Constraints;
@@ -51,6 +49,7 @@
 import play.mvc.Result;
 import play.mvc.Results;
 import security.AuthenticationManager;
+import utils.SerializationUtils;
 
 public class AuthenticationController extends Controller {
   public static final String AUTH_VERBOSE_LOGGING = "auth.verbose.logging";
@@ -75,7 +74,7 @@ public class AuthenticationController extends Controller {
 
   @Inject private org.pac4j.core.config.Config ssoConfig;
 
-  @VisibleForTesting @Inject protected PlayCookieSessionStore playCookieSessionStore;
+  @VisibleForTesting @Inject protected SessionStore sessionStore;
 
   @VisibleForTesting @Inject protected SsoManager ssoManager;
 
@@ -371,11 +370,9 @@ protected Result addRedirectCookie(Result result, CallContext ctx, String redire
     // to reduce size of the session cookie
     FoundAction foundAction =
         new FoundAction(BasePathUtils.addBasePath(redirectPath, this.basePath));
-    byte[] javaSerBytes =
-        ((PlayCookieSessionStore) ctx.sessionStore()).getSerializer().serializeToBytes(foundAction);
-    String serialized = Base64.getEncoder().encodeToString(compressBytes(javaSerBytes));
     Http.CookieBuilder redirectCookieBuilder =
-        Http.Cookie.builder(REDIRECT_URL_COOKIE_NAME, serialized);
+        Http.Cookie.builder(
+            REDIRECT_URL_COOKIE_NAME, SerializationUtils.serializeFoundAction(foundAction));
     redirectCookieBuilder.withPath(BasePathUtils.addBasePath("/", this.basePath));
     redirectCookieBuilder.withSecure(true);
     redirectCookieBuilder.withHttpOnly(true);
@@ -422,7 +419,7 @@ private CallContext buildCallContext(Http.RequestHeader request) {
     PlayWebContext webContext = new PlayWebContext(request);
 
     // Then create CallContext using the web context and session store
-    return new CallContext(webContext, playCookieSessionStore);
+    return new CallContext(webContext, sessionStore);
   }
 
   private void configurePac4jSessionStore(CallContext ctx, Client client) {
@@ -431,11 +428,11 @@ private void configurePac4jSessionStore(CallContext ctx, Client client) {
     // This is to prevent previous login attempts from being cached.
     // We replicate the logic here, which is buried in the Pac4j client.
     Optional<Object> attempt =
-        playCookieSessionStore.get(context, client.getName() + ATTEMPTED_AUTHENTICATION_SUFFIX);
+        ctx.sessionStore().get(context, client.getName() + ATTEMPTED_AUTHENTICATION_SUFFIX);
     if (attempt.isPresent() && !"".equals(attempt.get())) {
       logger.debug(
           "Found previous login attempt. Removing it manually to prevent unexpected errors.");
-      playCookieSessionStore.set(context, client.getName() + ATTEMPTED_AUTHENTICATION_SUFFIX, "");
+      ctx.sessionStore().set(context, client.getName() + ATTEMPTED_AUTHENTICATION_SUFFIX, "");
     }
   }
 

datahub-frontend/app/utils/SerializationUtils.java

@@ -0,0 +1,27 @@
+package utils;
+
+import static org.pac4j.play.store.PlayCookieSessionStore.compressBytes;
+import static org.pac4j.play.store.PlayCookieSessionStore.uncompressBytes;
+
+import java.util.Base64;
+import javax.annotation.Nonnull;
+import org.pac4j.core.exception.http.FoundAction;
+import org.pac4j.core.util.serializer.JavaSerializer;
+
+public class SerializationUtils {
+
+  private static final JavaSerializer JAVA_SERIALIZER = new JavaSerializer();
+
+  private SerializationUtils() {}
+
+  public static String serializeFoundAction(@Nonnull final FoundAction foundAction) {
+    byte[] javaSerBytes = JAVA_SERIALIZER.serializeToBytes(foundAction);
+    return Base64.getEncoder().encodeToString(compressBytes(javaSerBytes));
+  }
+
+  public static FoundAction deserializeFoundAction(@Nonnull final String serialized) {
+    return (FoundAction)
+        JAVA_SERIALIZER.deserializeFromBytes(
+            uncompressBytes(Base64.getDecoder().decode(serialized)));
+  }
+}

datahub-frontend/test/controllers/AuthenticationControllerTest.java

@@ -100,7 +100,7 @@ public void setUp() {
 
     // Create the controller
     controller = new AuthenticationController(mockConfig);
-    controller.playCookieSessionStore = playCookieSessionStore;
+    controller.sessionStore = playCookieSessionStore;
     controller.ssoManager = ssoManager;
     controller.authClient = authClient;
   }
@@ -353,7 +353,7 @@ public void testAuthenticateWithBasePathRedirect() {
     AuthenticationController testController = new AuthenticationController(config);
     testController.ssoManager = ssoManager;
     testController.authClient = authClient;
-    testController.playCookieSessionStore = playCookieSessionStore;
+    testController.sessionStore = playCookieSessionStore;
 
     // Configure SSO to be enabled
     when(ssoManager.isSsoEnabled()).thenReturn(true);
@@ -408,7 +408,7 @@ public void testAuthenticateWithLogOutRedirect() {
     AuthenticationController testController = new AuthenticationController(config);
     testController.ssoManager = ssoManager;
     testController.authClient = authClient;
-    testController.playCookieSessionStore = playCookieSessionStore;
+    testController.sessionStore = playCookieSessionStore;
 
     // Configure SSO to be enabled
     when(ssoManager.isSsoEnabled()).thenReturn(true);
@@ -465,7 +465,7 @@ public void testSsoWithBasePath() {
     Config config = ConfigFactory.parseMap(configMap);
     AuthenticationController testController = new AuthenticationController(config);
     testController.ssoManager = ssoManager;
-    testController.playCookieSessionStore = playCookieSessionStore;
+    testController.sessionStore = playCookieSessionStore;
 
     // Configure SSO to be enabled
     when(ssoManager.isSsoEnabled()).thenReturn(true);
@@ -619,7 +619,7 @@ public void testRedirectToIdentityProviderWithBasePath() {
     Config config = ConfigFactory.parseMap(configMap);
     AuthenticationController testController = new AuthenticationController(config);
     testController.ssoManager = ssoManager;
-    testController.playCookieSessionStore = playCookieSessionStore;
+    testController.sessionStore = playCookieSessionStore;
 
     // Configure SSO to be enabled
     when(ssoManager.isSsoEnabled()).thenReturn(true);

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpsertLinkResolver.java

@@ -59,7 +59,6 @@ public CompletableFuture<Boolean> get(DataFetchingEnvironment environment) throw
                 actor,
                 settingsInput,
                 _entityService);
-            log.warn(">>> NO ERROR RAISED");
             return true;
           } catch (Exception e) {
             log.error(

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/role/CreateInviteTokenResolver.java

@@ -25,7 +25,7 @@ public class CreateInviteTokenResolver implements DataFetcher<CompletableFuture<
   public CompletableFuture<InviteToken> get(final DataFetchingEnvironment environment)
       throws Exception {
     final QueryContext context = environment.getContext();
-    if (!canManagePolicies(context)) {
+    if (!canManageUserCredentials(context)) {
       throw new AuthorizationException(
           "Unauthorized to create invite tokens. Please contact your DataHub administrator if this needs corrective action.");
     }

datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/role/GetInviteTokenResolver.java

@@ -25,7 +25,7 @@ public class GetInviteTokenResolver implements DataFetcher<CompletableFuture<Inv
   public CompletableFuture<InviteToken> get(final DataFetchingEnvironment environment)
       throws Exception {
     final QueryContext context = environment.getContext();
-    if (!canManagePolicies(context)) {
+    if (!canManageUserCredentials(context)) {
       throw new AuthorizationException(
           "Unauthorized to get invite tokens. Please contact your DataHub administrator if this needs corrective action.");
     }

datahub-graphql-core/src/main/resources/entity.graphql

@@ -9278,14 +9278,14 @@ input UpsertLinkInput {
   label: String!
 
   """
-  Optional settings input for this link
+  The urn of the resource or entity to attach the link to, for example a dataset urn
   """
-  settings: LinkSettingsInput
+  resourceUrn: String!
 
   """
-  The urn of the resource or entity to attach the link to, for example a dataset urn
+  Optional settings input for this link
   """
-  resourceUrn: String!
+  settings: LinkSettingsInput
 }
 
 """

datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/link/UpsertLinkResolverTest.java

@@ -49,8 +49,8 @@ public void testShouldCreateLinkWhenEntityHasNoLinks() throws Exception {
             new UpsertLinkInput(
                 "https://original-url.com",
                 "Original label",
-                new LinkSettingsInput(false),
-                ASSET_URN));
+                ASSET_URN,
+                new LinkSettingsInput(false)));
     UpsertLinkResolver resolver = new UpsertLinkResolver(mockService, mockClient);
     resolver.get(mockEnv).get();
 
@@ -77,7 +77,7 @@ public void testShouldCreateLinkWhenUpsertLinkWithTheSameUrlAndDifferentLabel()
     DataFetchingEnvironment mockEnv =
         initMockEnv(
             new UpsertLinkInput(
-                "https://original-url.com", "New label", new LinkSettingsInput(false), ASSET_URN));
+                "https://original-url.com", "New label", ASSET_URN, new LinkSettingsInput(false)));
     UpsertLinkResolver resolver = new UpsertLinkResolver(mockService, mockClient);
     resolver.get(mockEnv).get();
 
@@ -104,7 +104,7 @@ public void testShouldCreateLinkWhenUpsertLinkWithDifferentUrlAndTheSameLabel()
     DataFetchingEnvironment mockEnv =
         initMockEnv(
             new UpsertLinkInput(
-                "https://new-url.com", "Original label", new LinkSettingsInput(false), ASSET_URN));
+                "https://new-url.com", "Original label", ASSET_URN, new LinkSettingsInput(false)));
     UpsertLinkResolver resolver = new UpsertLinkResolver(mockService, mockClient);
     resolver.get(mockEnv).get();
 
@@ -133,8 +133,8 @@ public void testShouldUpdateLinkWhenUpsertLinkWithTheSameUrlAndLabel() throws Ex
             new UpsertLinkInput(
                 "https://original-url.com",
                 "Original label",
-                new LinkSettingsInput(false),
-                ASSET_URN));
+                ASSET_URN,
+                new LinkSettingsInput(false)));
     UpsertLinkResolver resolver = new UpsertLinkResolver(mockService, mockClient);
     resolver.get(mockEnv).get();
 
@@ -151,8 +151,8 @@ public void testGetFailureNoEntity() throws Exception {
             new UpsertLinkInput(
                 "https://original-url.com",
                 "Original label",
-                new LinkSettingsInput(false),
-                ASSET_URN));
+                ASSET_URN,
+                new LinkSettingsInput(false)));
     UpsertLinkResolver resolver = new UpsertLinkResolver(mockService, mockClient);
     assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join());
 
@@ -172,8 +172,8 @@ public void testGetFailureNoPermission() throws Exception {
             new UpsertLinkInput(
                 "https://original-url.com",
                 "Original label",
-                new LinkSettingsInput(false),
-                ASSET_URN));
+                ASSET_URN,
+                new LinkSettingsInput(false)));
     Mockito.when(mockEnv.getContext()).thenReturn(mockContext);
 
     UpsertLinkResolver resolver = new UpsertLinkResolver(mockService, mockClient);
@@ -208,8 +208,8 @@ public void testUpsertLinkFailure() throws Exception {
             new UpsertLinkInput(
                 "https://original-url.com",
                 "Original label",
-                new LinkSettingsInput(false),
-                ASSET_URN));
+                ASSET_URN,
+                new LinkSettingsInput(false)));
     UpsertLinkResolver resolver = new UpsertLinkResolver(mockService, mockClient);
 
     assertThrows(