Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.testng.annotations.Test;

import java.io.IOException;
import java.net.ServerSocket;
import java.io.File;
import java.nio.file.Files;
import java.util.List;
import java.util.UUID;

import static org.testng.AssertJUnit.assertFalse;

Expand All @@ -34,12 +35,22 @@ public class TestHBaseTestingUtilSpinup {
private final HBaseTestingUtility hBaseTestingUtility = new HBaseTestingUtility();

public TestHBaseTestingUtilSpinup() throws Exception {
hBaseTestingUtility.getConfiguration().set("test.hbase.zookeeper.property.clientPort", String.valueOf(getFreePort()));
hBaseTestingUtility.getConfiguration().set("hbase.master.port", String.valueOf(getFreePort()));
hBaseTestingUtility.getConfiguration().set("hbase.master.info.port", String.valueOf(getFreePort()));
hBaseTestingUtility.getConfiguration().set("hbase.regionserver.port", String.valueOf(getFreePort()));
hBaseTestingUtility.getConfiguration().set("hbase.regionserver.info.port", String.valueOf(getFreePort()));
hBaseTestingUtility.getConfiguration().set("zookeeper.znode.parent", "/hbase-unsecure");
String runId = UUID.randomUUID().toString();
File baseDir = Files.createTempDirectory("atlas-hbase-test-" + runId).toFile();

// Keep each test run isolated from stale local state and avoid fixed-port races.
hBaseTestingUtility.getConfiguration().set("hadoop.tmp.dir", new File(baseDir, "hadoop-tmp").getAbsolutePath());
hBaseTestingUtility.getConfiguration().set("hbase.rootdir", new File(baseDir, "hbase-root").toURI().toString());
hBaseTestingUtility.getConfiguration().set("hbase.zookeeper.property.dataDir", new File(baseDir, "zk-data").getAbsolutePath());
hBaseTestingUtility.getConfiguration().set("zookeeper.znode.parent", "/hbase-unsecure-" + runId);
hBaseTestingUtility.getConfiguration().set("test.hbase.zookeeper.property.clientPort", "0");
hBaseTestingUtility.getConfiguration().set("hbase.master.port", "0");
hBaseTestingUtility.getConfiguration().set("hbase.master.info.port", "0");
hBaseTestingUtility.getConfiguration().set("hbase.regionserver.port", "0");
hBaseTestingUtility.getConfiguration().set("hbase.regionserver.info.port", "0");
hBaseTestingUtility.getConfiguration().set("hbase.master.hostname", "localhost");
hBaseTestingUtility.getConfiguration().set("hbase.regionserver.hostname", "localhost");
hBaseTestingUtility.getConfiguration().set("hbase.regionserver.hostname.seen.by.master", "localhost");
hBaseTestingUtility.getConfiguration().set("hbase.table.sanity.checks", "false");
}

Expand All @@ -57,13 +68,4 @@ public void testGetMetaTableRows() throws Exception {
hBaseTestingUtility.shutdownMiniCluster();
}
}

private static int getFreePort() throws IOException {
ServerSocket serverSocket = new ServerSocket(0);
int port = serverSocket.getLocalPort();

serverSocket.close();

return port;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -233,10 +233,10 @@ public void shouldSelectActiveAmongMultipleServersIfHAIsEnabled() {

when(firstResponse.getStatus()).thenReturn(Response.Status.OK.getStatusCode());

String passiveStatus = "{\"Status\":\"PASSIVE\"}";
String becomingActiveStatus = "{\"Status\":\"BECOMING_ACTIVE\"}";

when(firstResponse.getEntity(String.class)).thenReturn(passiveStatus);
when(firstResponse.getLength()).thenReturn(passiveStatus.length());
when(firstResponse.getEntity(String.class)).thenReturn(becomingActiveStatus);
when(firstResponse.getLength()).thenReturn(becomingActiveStatus.length());

ClientResponse secondResponse = mock(ClientResponse.class);

Expand Down
11 changes: 7 additions & 4 deletions common/src/main/java/org/apache/atlas/ha/HAConfiguration.java
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@
import org.apache.atlas.security.SecurityProperties;
import org.apache.commons.configuration2.Configuration;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.List;
Expand All @@ -30,6 +32,8 @@
* A wrapper for getting configuration entries related to HighAvailability.
*/
public final class HAConfiguration {
private static final Logger LOG = LoggerFactory.getLogger(HAConfiguration.class);

public static final String ATLAS_SERVER_ZK_ROOT_DEFAULT = "/apache_atlas";
public static final String ATLAS_SERVER_HA_PREFIX = "atlas.server.ha.";
public static final String ZOOKEEPER_PREFIX = "zookeeper.";
Expand Down Expand Up @@ -57,14 +61,13 @@ private HAConfiguration() {
* @return
*/
public static boolean isHAEnabled(Configuration configuration) {
boolean ret;
boolean ret = false;

if (configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)) {
ret = configuration.getBoolean(ATLAS_SERVER_HA_ENABLED_KEY);
LOG.info("isHAEnabled: key '{}' found in config, value={}", ATLAS_SERVER_HA_ENABLED_KEY, ret);
} else {
String[] ids = configuration.getStringArray(HAConfiguration.ATLAS_SERVER_IDS);

ret = ids != null && ids.length > 1;
LOG.info("isHAEnabled: key '{}' NOT found in config, defaulting to false", ATLAS_SERVER_HA_ENABLED_KEY);
}

return ret;
Expand Down
13 changes: 13 additions & 0 deletions common/src/main/java/org/apache/atlas/repository/Constants.java
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,19 @@ public final class Constants {
public static final String PATCH_TYPE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.type");
public static final String PATCH_ACTION_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.action");
public static final String PATCH_STATE_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.state");
public static final String PATCH_APPLIED_BY_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.appliedBy");
public static final String PATCH_APPLIED_AT_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.appliedAt");
public static final String PATCH_CLAIMED_BY_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.claimedBy");
public static final String PATCH_CLAIM_STARTED_AT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "patch.claimStartedAt");
/**
* TypeDef bootstrap claim keys.
*/
public static final String TYPEDEF_BOOTSTRAP_FILE_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typedef.bootstrap.file");
public static final String TYPEDEF_BOOTSTRAP_STATE_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typedef.bootstrap.state");
public static final String TYPEDEF_BOOTSTRAP_CLAIMED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typedef.bootstrap.claimedBy");
public static final String TYPEDEF_BOOTSTRAP_CLAIM_STARTED_AT = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typedef.bootstrap.claimStartedAt");
public static final String TYPEDEF_BOOTSTRAP_APPLIED_BY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typedef.bootstrap.appliedBy");
public static final String TYPEDEF_BOOTSTRAP_APPLIED_AT_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "typedef.bootstrap.appliedAt");
/**
* The homeId field is used when saving into Atlas a copy of an object that is being imported from another
* repository. The homeId will be set to a String that identifies the other repository. The specific format
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,13 +63,13 @@ public void testIsHAEnabledByLegacyConfiguration() {
}

@Test
public void testIsHAEnabledByIds() {
public void testIsHAEnabledByIds_doesNotInferWhenFlagMissing() {
when(configuration.containsKey(HAConfiguration.ATLAS_SERVER_HA_ENABLED_KEY)).thenReturn(false);
when(configuration.getStringArray(HAConfiguration.ATLAS_SERVER_IDS)).thenReturn(TEST_ATLAS_SERVER_IDS_HA);
boolean isHAEnabled = HAConfiguration.isHAEnabled(configuration);
assertTrue(isHAEnabled);
assertFalse(isHAEnabled);

// restore
// single-id remains disabled as well when explicit flag is absent
when(configuration.getStringArray(HAConfiguration.ATLAS_SERVER_IDS)).thenReturn(new String[] {"id1"});
isHAEnabled = HAConfiguration.isHAEnabled(configuration);
assertFalse(isHAEnabled);
Expand Down
64 changes: 64 additions & 0 deletions dev-support/atlas-docker/.env.active-active
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
# =============================================================================
# Atlas Active-Active ADDITIONAL settings
#
# All infrastructure versions (HADOOP_VERSION, HBASE_VERSION, KAFKA_VERSION,
# ATLAS_VERSION, etc.) are already defined in the existing .env file — do NOT
# duplicate them here.
#
# Usage (two options):
#
# Option A — append to the shared .env (simplest):
# cat .env.active-active >> .env
# docker compose -f docker-compose.atlas-active-active.yml up -d
#
# Option B — pass both files explicitly:
# docker compose \
# --env-file .env \
# --env-file .env.active-active \
# -f docker-compose.atlas-active-active.yml up -d
#
# Scale replicas without editing this file:
# METADATA_SERVER_REPLICAS=5 docker compose \
# -f docker-compose.atlas-active-active.yml up -d
# or:
# docker compose -f docker-compose.atlas-active-active.yml \
# up -d --scale atlas-metadata-server=5
# =============================================================================

# ---------------------------------------------------------------------------
# Replica counts — only active-active specific settings below
# ---------------------------------------------------------------------------

# Atlas graph backend selector for active-active:
# hbase -> HBase graph + HBase audit
# postgres -> RDBMS graph + Postgres audit
ATLAS_BACKEND=postgres

# Number of METADATA_SERVER replicas (REST + search + entity CRUD)
METADATA_SERVER_REPLICAS=2

# Number of NOTIFICATION_PROCESSOR replicas (hook Kafka consumer)
NOTIFICATION_PROC_REPLICAS=2

# Host port for the Nginx load balancer (proxies to all metadata-server replicas)
ATLAS_LB_PORT=21000

# ---------------------------------------------------------------------------
# Patch toggles for initializer run
# ---------------------------------------------------------------------------
# Rebuild mixed indexes (JAVA_PATCH_0000_006 / ReIndexPatch)
ATLAS_REBUILD_INDEX=true
# Enable composite index status promotion (JAVA_PATCH_0000_010)
ATLAS_UPDATE_COMPOSITE_INDEX_STATUS=true

# ---------------------------------------------------------------------------
# Index recovery service toggles
# ---------------------------------------------------------------------------
# Enable background index recovery monitor on metadata nodes
ATLAS_INDEX_RECOVERY_ENABLE=true

# ---------------------------------------------------------------------------
# Postgres backend pool tuning (used only when ATLAS_BACKEND=postgres)
# ---------------------------------------------------------------------------
ATLAS_RDBMS_MAX_POOL_SIZE=15
ATLAS_RDBMS_MIN_IDLE=2
28 changes: 28 additions & 0 deletions dev-support/atlas-docker/.env.bkp
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
BUILD_HOST_SRC=true
SKIPTESTS=true
GIT_URL=https://github.com/apache/atlas.git
BRANCH=master
PROFILE=dist,external-hbase-solr

# Java version for AtlasBase image.
# This image gets used as base docker image for all images.
# Valid values: 8, 11, 17
ATLAS_BASE_JAVA_VERSION=8

# Java version to use to build Apache Atlas
# Valid values: 8, 11, 17
ATLAS_BUILD_JAVA_VERSION=8

# Java version to use to run Atlas server
# Valid values: 8, 11, 17
ATLAS_SERVER_JAVA_VERSION=8

ATLAS_VERSION=3.0.0-SNAPSHOT
UBUNTU_VERSION=20.04
HADOOP_VERSION=3.4.2
HBASE_VERSION=2.6.4
KAFKA_VERSION=2.8.2
HIVE_VERSION=3.1.3
HIVE_HADOOP_VERSION=3.1.1

ATLAS_BACKEND=hbase
3 changes: 3 additions & 0 deletions dev-support/atlas-docker/Dockerfile.atlas
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,11 @@ ENV JAVA_HOME=/usr/lib/jvm/java-${ATLAS_SERVER_JAVA_VERSION}-openjdk-${TARGETARC
RUN update-java-alternatives --set /usr/lib/jvm/java-1.${ATLAS_SERVER_JAVA_VERSION}.0-openjdk-${TARGETARCH}

COPY ./scripts/atlas.sh ${ATLAS_SCRIPTS}/
COPY ./scripts/atlas-active-active.sh ${ATLAS_SCRIPTS}/
COPY ./dist/apache-atlas-${ATLAS_VERSION}-server.tar.gz /home/atlas/dist/

RUN chmod +x ${ATLAS_SCRIPTS}/atlas.sh ${ATLAS_SCRIPTS}/atlas-active-active.sh

RUN tar xfz /home/atlas/dist/apache-atlas-${ATLAS_VERSION}-server.tar.gz --directory=/opt/ && \
ln -s /opt/apache-atlas-${ATLAS_VERSION} ${ATLAS_HOME} && \
rm -f /home/atlas/dist/apache-atlas-${ATLAS_VERSION}-server.tar.gz && \
Expand Down
Loading
Loading