feat: advanced rolling ops using etcd

.github/workflows/test-package.yaml

@@ -184,7 +184,7 @@ jobs:
   integration:
     needs: init
     if: contains(fromJson(needs.init.outputs.tests), 'integration') && !inputs.skip-juju
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     strategy:
       fail-fast: false
       matrix:

CODEOWNERS

@@ -27,6 +27,7 @@
 /nginx_k8s/ @canonical/tracing-and-profiling
 /passwd/ @canonical/charmlibs-maintainers
 /pathops/ @canonical/charmlibs-maintainers
+/rollingops/ @canonical/data
 /snap/ @canonical/charmlibs-maintainers
 /sysctl/ @canonical/charmlibs-maintainers
 /systemd/ @canonical/charmlibs-maintainers

rollingops/README.md

@@ -0,0 +1,29 @@
+# charmlibs.rollingops
+
+The `rollingops` library.
+
+`rollingops` provides a rolling-operations manager for Juju charms backed by etcd.
+
+It coordinates operations across units by using etcd as a shared lock and queue backend,
+and uses TLS client credentials to authenticate requests to the etcd cluster.
+
+To install, add `charmlibs-rollingops` to your Python dependencies. Then in your Python code, import as:
+
+```py
+from charmlibs import rollingops
+```
+
+See the [reference documentation](https://documentation.ubuntu.com/charmlibs/reference/charmlibs/rollingops) for more.
+
+## Unit tests
+```py
+just python=3.12 unit rollingops
+```
+## Pack
+```py
+just python=3.12  pack-machine rollingops
+```
+## Integration tests
+```py
+just python=3.12  integration-machine rollingops
+```

rollingops/pyproject.toml

@@ -0,0 +1,78 @@
+[project]
+name = "charmlibs-rollingops"
+description = "The charmlibs.rollingops package."
+readme = "README.md"
+requires-python = ">=3.12"
+authors = [
+    {name="Data Platform"},
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: Apache Software License",
+    "Intended Audience :: Developers",
+    "Operating System :: POSIX :: Linux",
+    "Development Status :: 5 - Production/Stable",
+]
+dynamic = ["version"]
+dependencies = [
+    "ops",
+    "charmlibs-interfaces-tls-certificates>=1.8.1",
+    "charmlibs-pathops>=1.2.1",
+    "dpcharmlibs-interfaces==1.0.0",
+    "tenacity"
+]
+
+[dependency-groups]
+lint = [  # installed for `just lint rollingops` (unit, functional, and integration are also installed)
+    # "typing_extensions",
+]
+unit = [  # installed for `just unit rollingops`
+    "ops[testing]",
+]
+functional = [  # installed for `just functional rollingops`
+]
+integration = [  # installed for `just integration rollingops`
+    "jubilant",
+    "tenacity",
+    "charmlibs-apt",
+]
+
+[project.urls]
+"Repository" = "https://github.com/canonical/charmlibs"
+"Issues" = "https://github.com/canonical/charmlibs/issues"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/charmlibs"]
+
+[tool.hatch.version]
+path = "src/charmlibs/rollingops/_version.py"
+
+[tool.ruff]
+extend = "../pyproject.toml"
+src = ["src", "tests/unit", "tests/functional", "tests/integration"]  # correctly sort local imports in tests
+
+[tool.ruff.lint.extend-per-file-ignores]
+# add additional per-file-ignores here to avoid overriding repo-level config
+"tests/**/*" = [
+    # "E501",  # line too long
+]
+
+[tool.pyright]
+extends = "../pyproject.toml"
+include = ["src", "tests"]
+exclude = ["tests/integration/.tmp/**"]
+pythonVersion = "3.12"  # check no python > 3.12 features are used
+
+[tool.charmlibs.functional]
+ubuntu = []  # ubuntu versions to run functional tests with, e.g. "24.04" (defaults to just "latest")
+pebble = []  # pebble versions to run functional tests with, e.g. "v1.0.0", "master" (defaults to no pebble versions)
+sudo = false  # whether to run functional tests with sudo (defaults to false)
+
+[tool.charmlibs.integration]
+# tags to run integration tests with (defaults to running once with no tag, i.e. tags = [''])
+# Available in CI in tests/integration/pack.sh and integration tests as CHARMLIBS_TAG
+tags = []  # Not used by the pack.sh and integration tests generated by the template

rollingops/src/charmlibs/rollingops/__init__.py

@@ -0,0 +1,30 @@
+# Copyright 2026 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""The charmlibs.rollingops package."""
+
+from ._manager import EtcdRollingOpsManager
+from ._models import (
+    OperationResult,
+    RollingOpsEtcdNotConfiguredError,
+    RollingOpsInvalidLockRequestError,
+)
+from ._version import __version__ as __version__
+
+__all__ = (
+    'EtcdRollingOpsManager',
+    'OperationResult',
+    'RollingOpsEtcdNotConfiguredError',
+    'RollingOpsInvalidLockRequestError',
+)

rollingops/src/charmlibs/rollingops/_certificates.py

@@ -0,0 +1,170 @@
+# Copyright 2026 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Manage generation and persistence of TLS certificates for etcd client access.
+
+This file contains functions responsible for creating and storing a client Certificate
+Authority (CA) and a client certificate/key pair used to authenticate
+with etcd via TLS. Certificates are generated only once and persisted
+under a local directory so they can be reused across charm executions.
+
+Certificates are valid for 20 years. They are not renewed or rotated.
+"""
+
+from datetime import timedelta
+
+from charmlibs import pathops
+from charmlibs.interfaces.tls_certificates import (
+    Certificate,
+    CertificateRequestAttributes,
+    CertificateSigningRequest,
+    PrivateKey,
+    TLSCertificatesError,
+)
+from charmlibs.rollingops._models import (
+    RollingOpsFileSystemError,
+    SharedCertificate,
+    with_pebble_retry,
+)
+
+BASE_DIR = pathops.LocalPath('/var/lib/rollingops/tls')
+CA_CERT_PATH = BASE_DIR / 'client-ca.pem'
+CLIENT_KEY_PATH = BASE_DIR / 'client.key'
+CLIENT_CERT_PATH = BASE_DIR / 'client.pem'
+VALIDITY_DAYS = 365 * 50
+KEY_SIZE = 4096
+
+
+def persist_client_cert_key_and_ca(shared: SharedCertificate) -> None:
+    """Persist the provided client certificate, key, and CA to disk.
+
+    Raises:
+        PebbleConnectionError: if the remote container cannot be reached
+        RollingOpsFileSystemError: if there is a problem when writing the certificates
+    """
+    if _has_client_cert_key_and_ca(shared):
+        return
+    try:
+        with_pebble_retry(lambda: BASE_DIR.mkdir(parents=True, exist_ok=True))
+        shared.write_to_paths(CLIENT_CERT_PATH, CLIENT_KEY_PATH, CA_CERT_PATH)
+
+    except (FileNotFoundError, LookupError, NotADirectoryError, PermissionError) as e:
+        raise RollingOpsFileSystemError('Failed to persist client certificates and key.') from e
+
+
+def _has_client_cert_key_and_ca(shared: SharedCertificate) -> bool:
+    """Return whether the provided certificate material matches local files.
+
+    Raises:
+        PebbleConnectionError: if the remote container cannot be reached
+        RollingOpsFileSystemError: if there is a problem when writing the certificates
+    """
+    if not _exists():
+        return False
+    try:
+        stored = SharedCertificate.from_paths(
+            CLIENT_CERT_PATH,
+            CLIENT_KEY_PATH,
+            CA_CERT_PATH,
+        )
+        return stored == shared
+
+    except (
+        FileNotFoundError,
+        IsADirectoryError,
+        PermissionError,
+        TLSCertificatesError,
+        ValueError,
+    ) as e:
+        raise RollingOpsFileSystemError('Failed to read certificates and key.') from e
+
+
+def generate(common_name: str) -> SharedCertificate:
+    """Generate a client CA and client certificate if they do not exist.
+
+    This method creates:
+    1. A CA private key and self-signed CA certificate.
+    2. A client private key.
+    3. A certificate signing request (CSR) using the provided common name.
+    4. A client certificate signed by the generated CA.
+
+    The generated files are written to disk and reused in future runs.
+    If the certificates already exist, this method does nothing.
+
+    Args:
+        common_name: Common Name (CN) used in the client certificate
+            subject. This value should not contain slashes.
+
+    Raises:
+        PebbleConnectionError: if the remote container cannot be reached
+        RollingOpsFileSystemError: if there is a problem when writing the certificates
+    """
+    if _exists():
+        return SharedCertificate.from_paths(
+            CLIENT_CERT_PATH,
+            CLIENT_KEY_PATH,
+            CA_CERT_PATH,
+        )
+
+    ca_key = PrivateKey.generate(key_size=KEY_SIZE)
+    ca_attributes = CertificateRequestAttributes(
+        common_name=common_name,
+        is_ca=True,
+        add_unique_id_to_subject_name=False,
+    )
+    ca_crt = Certificate.generate_self_signed_ca(
+        attributes=ca_attributes,
+        private_key=ca_key,
+        validity=timedelta(days=VALIDITY_DAYS),
+    )
+
+    client_key = PrivateKey.generate(key_size=KEY_SIZE)
+
+    csr_attributes = CertificateRequestAttributes(
+        common_name=common_name, add_unique_id_to_subject_name=False
+    )
+    csr = CertificateSigningRequest.generate(
+        attributes=csr_attributes,
+        private_key=client_key,
+    )
+
+    client_crt = Certificate.generate(
+        csr=csr,
+        ca=ca_crt,
+        ca_private_key=ca_key,
+        validity=timedelta(days=VALIDITY_DAYS),
+        is_ca=False,
+    )
+
+    shared = SharedCertificate(
+        certificate=client_crt,
+        key=client_key,
+        ca=ca_crt,
+    )
+
+    persist_client_cert_key_and_ca(shared)
+    return shared
+
+
+def _exists() -> bool:
+    """Check whether the client certificates and CA certificate already exist.
+
+    Raises:
+        PebbleConnectionError: if the remote container cannot be reached
+    """
+    return (
+        with_pebble_retry(lambda: CA_CERT_PATH.exists())
+        and with_pebble_retry(lambda: CLIENT_KEY_PATH.exists())
+        and with_pebble_retry(lambda: CLIENT_CERT_PATH.exists())
+    )

rollingops/src/charmlibs/rollingops/_etcd_rollingops.py

@@ -0,0 +1,40 @@
+# Copyright 2026 Canonical Ltd.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+import subprocess
+import time
+
+
+def main():
+    """Juju hook event dispatcher."""
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--run-cmd', required=True)
+    parser.add_argument('--unit-name', required=True)
+    parser.add_argument('--charm-dir', required=True)
+    parser.add_argument('--owner', required=True)
+    args = parser.parse_args()
+
+    time.sleep(10)
+
+    dispatch_sub_cmd = (
+        f'JUJU_DISPATCH_PATH=hooks/rollingops_lock_granted {args.charm_dir}/dispatch'
+    )
+    res = subprocess.run([args.run_cmd, '-u', args.unit_name, dispatch_sub_cmd])
+    res.check_returncode()
+
+
+if __name__ == '__main__':
+    main()