clickhouse-config.cluster-3node.example.xml

<?xml version="1.0"?>
<!--
  ClickHouse Cluster Configuration Example: Single Shard, 3 Replicas

  This example shows the recommended production setup:
  - 1 shard (not sharded for simplicity)
  - 3 replicas (full redundancy)
  - All nodes have 100% of the data
  - If any node fails, queries automatically failover to healthy replicas

  Architecture:
  Node 1: Shard 1, Replica 1 of all data
  Node 2: Shard 1, Replica 2 of all data
  Node 3: Shard 1, Replica 3 of all data

  Key Differences from Single-Node:
  1. remote_servers has 3 replicas in single shard (not 3 separate shards)
  2. Each node has different {replica} macro but same {shard} macro
  3. Keeper runs on all 3 nodes for coordination
  4. internal_replication=true enables smart replication
-->
<clickhouse>
    <!-- Network Ports -->
    <http_port>8123</http_port>
    <tcp_port>9000</tcp_port>
    <interserver_http_port>9009</interserver_http_port>
    <listen_host>0.0.0.0</listen_host>
    <layer>1</layer>

    <!-- Data Directories -->
    <path>/var/lib/clickhouse/</path>
    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
    <format_schema_path>/var/lib/clickhouse/format_schemas/</format_schema_path>

    <!-- Logging -->
    <logger>
        <log>/var/log/clickhouse-server/clickhouse-server.log</log>
        <errorlog>/var/log/clickhouse-server/clickhouse-server.err.log</errorlog>
        <size>1000M</size>
        <count>10</count>
    </logger>

    <users_config>/etc/clickhouse-server/users.xml</users_config>

    <!-- ==========================================
         Embedded ClickHouse Keeper (ZooKeeper)
         Runs on all 3 nodes for coordination
         =========================================== -->
    <keeper_server>
        <tcp_port>2181</tcp_port>         <!-- ZooKeeper-compatible port -->
        <server_id>REPLACE_SERVER_ID</server_id>  <!-- 1, 2, or 3 (set per node) -->

        <log_storage_path>/var/lib/clickhouse/coordination/log</log_storage_path>
        <snapshot_storage_path>/var/lib/clickhouse/coordination/snapshots</snapshot_storage_path>

        <coordination_settings>
            <operation_timeout_ms>10000</operation_timeout_ms>
            <session_timeout_ms>30000</session_timeout_ms>
            <raft_logs_level>information</raft_logs_level>
        </coordination_settings>

        <!-- Raft cluster configuration (same on all 3 nodes) -->
        <raft_configuration>
            <server>
                <id>1</id>
                <hostname>clickhouse-1</hostname>  <!-- Node 1 hostname/IP -->
                <port>9444</port>                  <!-- Internal Raft port -->
            </server>
            <server>
                <id>2</id>
                <hostname>clickhouse-2</hostname>  <!-- Node 2 hostname/IP -->
                <port>9444</port>
            </server>
            <server>
                <id>3</id>
                <hostname>clickhouse-3</hostname>  <!-- Node 3 hostname/IP -->
                <port>9444</port>
            </server>
        </raft_configuration>
    </keeper_server>

    <!-- ==========================================
         CRITICAL: Cluster Topology Configuration
         Single Shard with 3 Replicas (all nodes)
         =========================================== -->
    <remote_servers>
        <!-- Cluster name must match ON CLUSTER directives in DDL -->
        <tracker_cluster>
            <!-- Single shard containing 3 replicas -->
            <shard>
                <!-- Flag: Enable automatic replication between replicas -->
                <internal_replication>true</internal_replication>

                <!-- Replica 1 (typically on Node 1) -->
                <replica>
                    <host>clickhouse-1</host>    <!-- Node 1 hostname/IP -->
                    <port>9000</port>             <!-- ClickHouse native protocol port -->
                </replica>

                <!-- Replica 2 (typically on Node 2) -->
                <replica>
                    <host>clickhouse-2</host>    <!-- Node 2 hostname/IP -->
                    <port>9000</port>
                </replica>

                <!-- Replica 3 (typically on Node 3) -->
                <replica>
                    <host>clickhouse-3</host>    <!-- Node 3 hostname/IP -->
                    <port>9000</port>
                </replica>
            </shard>
        </tracker_cluster>
    </remote_servers>

    <!-- ==========================================
         Macros (substitutions in CREATE TABLE)
         IMPORTANT: Each node has DIFFERENT {replica}
                    but SAME {shard}
         =========================================== -->
    <!-- Node 1 macros: -->
    <macros>
        <shard>1</shard>        <!-- Same on all nodes! -->
        <replica>1</replica>    <!-- DIFFERENT per node (1, 2, or 3) -->
        <layer>1</layer>
    </macros>

    <!-- ==========================================
         Default Replica Paths (for ReplicatedMergeTree)
         These paths are used by all replicated tables
         =========================================== -->
    <default_replica_path>/clickhouse/tables/{layer}-{shard}/{database}/{table}</default_replica_path>
    <default_replica_name>{replica}</default_replica_name>

    <!-- ==========================================
         Distributed DDL (ON CLUSTER support)
         =========================================== -->
    <distributed_ddl>
        <path>/clickhouse/task_queue/ddl</path>
        <task_max_lifetime>300</task_max_lifetime>  <!-- 5 minute timeout for DDL tasks -->
    </distributed_ddl>

    <!-- ==========================================
         User Profiles and Quotas
         =========================================== -->
    <profiles>
        <default>
            <!-- DDL task timeout -->
            <distributed_ddl_task_timeout>300</distributed_ddl_task_timeout>

            <!-- Async Insert Settings for High Performance -->
            <async_insert>1</async_insert>
            <async_insert_threads>16</async_insert_threads>
            <async_insert_max_data_size>10485760</async_insert_max_data_size>  <!-- 10MB -->
            <async_insert_busy_timeout_ms>200</async_insert_busy_timeout_ms>
            <async_insert_stale_timeout_ms>1000</async_insert_stale_timeout_ms>
            <wait_for_async_insert>0</wait_for_async_insert>
            <async_insert_deduplicate>0</async_insert_deduplicate>

            <!-- Memory settings for high throughput -->
            <max_insert_block_size>1048576</max_insert_block_size>     <!-- 1M rows -->
            <min_insert_block_size_rows>1048576</min_insert_block_size_rows>
            <min_insert_block_size_bytes>268435456</min_insert_block_size_bytes>  <!-- 256MB -->
        </default>
    </profiles>

    <!-- ==========================================
         Replication Settings
         =========================================== -->
    <replication>
        <!-- Allow replicated tables to become leader -->
        <replicated_can_become_leader>1</replicated_can_become_leader>
        <!-- Keeper log retention -->
        <min_replicated_logs_to_keep>10</min_replicated_logs_to_keep>
        <max_replicated_logs_to_keep>1000</max_replicated_logs_to_keep>
    </replication>

    <!-- ==========================================
         MergeTree Engine Settings
         =========================================== -->
    <merge_tree>
        <allow_experimental_reverse_key>1</allow_experimental_reverse_key>
    </merge_tree>
</clickhouse>

<!--
================================================================================
SETUP INSTRUCTIONS FOR 3-NODE CLUSTER
================================================================================

1. PREREQUISITES
   - 3 machines (Node 1, 2, 3) with ClickHouse 23.4+ installed
   - Network connectivity between all nodes on ports 9000, 2181, 9444
   - Hostnames or IPs accessible as: clickhouse-1, clickhouse-2, clickhouse-3

2. ON EACH NODE
   a) Edit config file:
      cp clickhouse-config.cluster-3node.example.xml /etc/clickhouse-server/config.d/cluster.xml

   b) Update macros for each node:
      Node 1: <replica>1</replica>
      Node 2: <replica>2</replica>
      Node 3: <replica>3</replica>

   c) Verify keeper configuration has correct SERVER_ID:
      Node 1: <server_id>1</server_id>
      Node 2: <server_id>2</server_id>
      Node 3: <server_id>3</server_id>

   d) Start ClickHouse:
      systemctl restart clickhouse-server

3. VERIFY CLUSTER SETUP
   # From any node, check cluster status:
   clickhouse-client --query "SELECT * FROM system.clusters WHERE cluster='tracker_cluster'"

   # Should show 3 nodes (replicas):
   # tracker_cluster  1  1  1  clickhouse-1  9000  1  0
   # tracker_cluster  1  2  1  clickhouse-2  9000  1  0
   # tracker_cluster  1  3  1  clickhouse-3  9000  1  0

4. CREATE TABLES ON CLUSTER
   # Use ON CLUSTER directive (automatically creates on all replicas):
   clickhouse-client --multiquery < schema/core.1.sql

   # Tables will be created on all 3 nodes simultaneously

5. TEST FAULT TOLERANCE
   a) Write data:
      INSERT INTO tracker_cluster.events VALUES (...)

   b) Stop Node 1:
      systemctl stop clickhouse-server

   c) Query from Node 2 - should still work:
      clickhouse-client -h clickhouse-2 --query "SELECT count() FROM events FINAL"

   d) Restart Node 1:
      systemctl start clickhouse-server
      # Node 1 automatically syncs with Node 2 & 3

6. DISTRIBUTED TABLE PATTERN
   # Create replicated table on all nodes:
   CREATE TABLE events_local ON CLUSTER tracker_cluster (...)
   ENGINE = ReplicatedMergeTree(...)

   # Create distributed view (queries any healthy replica):
   CREATE TABLE events ON CLUSTER tracker_cluster AS events_local
   ENGINE = Distributed(tracker_cluster, default, events_local, rand())

================================================================================
KEY POINTS
================================================================================

✅ ADVANTAGES:
   - Full redundancy: any node can fail safely
   - Automatic failover for queries
   - No data loss with proper replication
   - Simple topology (single shard)

❌ LIMITATIONS:
   - No horizontal scaling (each node stores 100% of data)
   - Replication overhead: writes slower than single-node
   - Each write is replicated to all 3 nodes
   - Max throughput is limited by replication lag

⚠️  REPLICATION COST:
   - Single-node: 1M events/sec write throughput
   - 3-node replica: ~300K-400K events/sec (100x slower)
   - Solution: Use batching + async inserts to minimize impact

🚀 PRODUCTION OPTIMIZATION:
   If you need both fault tolerance AND high throughput:
   - Use Sharding + Replication (2 shards, 2 replicas each)
   - Or accept replication overhead and optimize batching
   - Or increase hardware capacity on each node

================================================================================
-->