diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml deleted file mode 100644 index 7635880..0000000 --- a/.github/workflows/pre-release.yml +++ /dev/null @@ -1,177 +0,0 @@ -name: Pre-Release - -on: - push: - branches: - - main - -env: - CARGO_TERM_COLOR: always - YQ_VERSION: 4.50.1 - -jobs: - build: - name: build - ${{ matrix.arch }} - strategy: - fail-fast: true - matrix: - include: - - target: x86_64-unknown-linux-gnu - arch: x86_64 - runner: ubuntu-latest - os: ubuntu - - target: aarch64-unknown-linux-gnu - arch: arm64 - runner: ubuntu-24.04-arm - os: ubuntu - runs-on: ${{ matrix.runner }} - - steps: - - uses: actions/checkout@v4 - - - name: update apt cache - run: sudo apt-get update - - - name: install protoc - run: sudo apt-get install -y protobuf-compiler - - - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - target: ${{ matrix.target }} - rustflags: "" - - - name: install cargo lambda - run: | - pip3 install cargo-lambda - - - name: build - uses: actions-rs/cargo@v1 - with: - command: lambda - args: build --release -o zip --target ${{ matrix.target }} - - - name: upload artifact - uses: actions/upload-artifact@v4 - with: - name: rotel-lambda-forwarder-${{ matrix.arch }} - path: target/lambda/rotel-lambda-forwarder/bootstrap.zip - if-no-files-found: error - - upload: - name: upload - ${{ matrix.arch }} - needs: build - strategy: - fail-fast: true - matrix: - include: - - target: x86_64-unknown-linux-gnu - arch: x86_64 - - target: aarch64-unknown-linux-gnu - arch: arm64 - runs-on: ubuntu-latest - - env: - S3_BUCKET: rotel-lambda-forwarder-dev - - permissions: - id-token: write - contents: read - - steps: - - name: download artifact - uses: actions/download-artifact@v4 - with: - name: rotel-lambda-forwarder-${{ matrix.arch }} - path: ./artifacts - - - name: configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.S3_FORWARDER_DEV_ARN }} - aws-region: us-east-1 - - - name: upload to S3 by version - run: | - S3_PATH="s3://${{ env.S3_BUCKET }}/rotel-lambda-forwarder/${{ github.ref_name }}/${{ matrix.arch }}/rotel-lambda-forwarder.zip" - aws s3 cp ./artifacts/bootstrap.zip "$S3_PATH" --acl public-read - echo "Uploaded to $S3_PATH" - - - name: upload to S3 latest - if: github.ref_name == 'main' - run: | - S3_PATH="s3://${{ env.S3_BUCKET }}/rotel-lambda-forwarder/latest/${{ matrix.arch }}/rotel-lambda-forwarder.zip" - aws s3 cp ./artifacts/bootstrap.zip "$S3_PATH" --acl public-read - echo "Uploaded to $S3_PATH" - - upload-cloudformation: - name: upload cloudformation - ${{ matrix.arch }} - needs: build - strategy: - fail-fast: true - matrix: - include: - - target: x86_64-unknown-linux-gnu - arch: x86_64 - - target: aarch64-unknown-linux-gnu - arch: arm64 - - runs-on: ubuntu-latest - - env: - ARTIFACT_S3_BUCKET: rotel-lambda-forwarder-dev - S3_BUCKET: rotel-cloudformation-dev - - permissions: - id-token: write - contents: read - - steps: - - uses: actions/checkout@v4 - - run: wget https://github.com/mikefarah/yq/releases/download/v$YQ_VERSION/yq_linux_amd64.tar.gz -O - | tar xz && mv yq_linux_amd64 /usr/local/bin/yq - - - run: | - mkdir -p dist/latest - mkdir -p dist/versioned - - - run: | - yq '.Parameters.S3Bucket.Default = "${{ env.ARTIFACT_S3_BUCKET }}" | - .Resources.ForwarderLambda.Properties.Architectures = ["${{ matrix.arch }}"] | - .Parameters.S3Key.Default = "rotel-lambda-forwarder/latest/${{ matrix.arch }}/rotel-lambda-forwarder.zip"' \ - ./cloudformation-stacks/forwarder-otlp.template.yaml > dist/latest/forwarder-otlp.yaml - - yq '.Parameters.S3Bucket.Default = "${{ env.ARTIFACT_S3_BUCKET }}" | - .Resources.ForwarderLambda.Properties.Architectures = ["${{ matrix.arch }}"] | - .Parameters.S3Key.Default = "rotel-lambda-forwarder/latest/${{ matrix.arch }}/rotel-lambda-forwarder.zip"' \ - ./cloudformation-stacks/forwarder-clickhouse.template.yaml > dist/latest/forwarder-clickhouse.yaml - - - run: | - yq '.Parameters.S3Bucket.Default = "${{ env.ARTIFACT_S3_BUCKET }}" | - .Resources.ForwarderLambda.Properties.Architectures = ["${{ matrix.arch }}"] | - .Parameters.S3Key.Default = "rotel-lambda-forwarder/${{ github.ref_name }}/${{ matrix.arch }}/rotel-lambda-forwarder.zip"' \ - ./cloudformation-stacks/forwarder-otlp.template.yaml > dist/versioned/forwarder-otlp.yaml - - yq '.Parameters.S3Bucket.Default = "${{ env.ARTIFACT_S3_BUCKET }}" | - .Resources.ForwarderLambda.Properties.Architectures = ["${{ matrix.arch }}"] | - .Parameters.S3Key.Default = "rotel-lambda-forwarder/${{ github.ref_name }}/${{ matrix.arch }}/rotel-lambda-forwarder.zip"' \ - ./cloudformation-stacks/forwarder-clickhouse.template.yaml > dist/versioned/forwarder-clickhouse.yaml - - - name: configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ secrets.S3_CLOUDFORMATION_DEV_ARN }} - aws-region: us-east-1 - - - name: upload to S3 by version - run: | - S3_PATH_BASE="s3://${{ env.S3_BUCKET }}/stacks/${{ github.ref_name }}/${{ matrix.arch }}" - - aws s3 cp ./dist/versioned/forwarder-otlp.yaml "${S3_PATH_BASE}/rotel-lambda-forwarder-otlp.yaml" --acl public-read - aws s3 cp ./dist/versioned/forwarder-clickhouse.yaml "${S3_PATH_BASE}/rotel-lambda-forwarder-clickhouse.yaml" --acl public-read - - - name: upload to S3 latest - if: github.ref_name == 'main' - run: | - S3_PATH_BASE="s3://${{ env.S3_BUCKET }}/stacks/latest/${{ matrix.arch }}" - - aws s3 cp ./dist/latest/forwarder-otlp.yaml "${S3_PATH_BASE}/rotel-lambda-forwarder-otlp.yaml" --acl public-read - aws s3 cp ./dist/latest/forwarder-clickhouse.yaml "${S3_PATH_BASE}/rotel-lambda-forwarder-clickhouse.yaml" --acl public-read diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 5c90d3e..f733433 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,16 +1,28 @@ +# Build and release the rotel-lambda-forwarder. +# +# This runs on two different events: +# 1. Push to main branch (or others as configured): the artifacts and CF files are uploaded to dev S3 buckets, image tagged as main +# 2. Release: artifacts and CF files are uploaded to main S3 buckets, images tagged with versions and latest +# +# name: Release on: + push: + branches: + - main release: types: [created] env: CARGO_TERM_COLOR: always YQ_VERSION: 4.50.1 + PUBLIC_ECR_IMAGE: "public.ecr.aws/streamfold/rotel-lambda-forwarder" jobs: - build: - name: build - ${{ matrix.arch }} + # Build 1: Direct cargo lambda build (produces bootstrap.zip) + build-zip: + name: build-zip - ${{ matrix.arch }} strategy: fail-fast: true matrix: @@ -18,11 +30,9 @@ jobs: - target: x86_64-unknown-linux-gnu arch: x86_64 runner: ubuntu-latest - os: ubuntu - target: aarch64-unknown-linux-gnu arch: arm64 runner: ubuntu-24.04-arm - os: ubuntu runs-on: ${{ matrix.runner }} steps: @@ -52,59 +62,150 @@ jobs: - name: upload artifact uses: actions/upload-artifact@v4 with: - name: rotel-lambda-forwarder-${{ matrix.arch }} + name: rotel-lambda-forwarder-zip-${{ matrix.arch }} path: target/lambda/rotel-lambda-forwarder/bootstrap.zip if-no-files-found: error - upload: - name: upload - ${{ matrix.arch }} - needs: build + # Build 2: Container build with pyo3 feature + build-container: + name: build-container - ${{ matrix.arch }} strategy: fail-fast: true matrix: include: - target: x86_64-unknown-linux-gnu arch: x86_64 + platform: linux/amd64 + runner: ubuntu-latest - target: aarch64-unknown-linux-gnu arch: arm64 - runs-on: ubuntu-latest + platform: linux/arm64 + runner: ubuntu-24.04-arm + runs-on: ${{ matrix.runner }} - env: - S3_BUCKET: rotel-lambda-forwarder + permissions: + id-token: write + contents: read + + steps: + - uses: actions/checkout@v4 + + - name: Parse Rust version from rust-toolchain.toml + id: rust-version + run: | + RUST_VERSION=$(grep '^channel = ' rust-toolchain.toml | sed 's/channel = "\(.*\)"/\1/') + echo "version=$RUST_VERSION" >> $GITHUB_OUTPUT + echo "Rust version: $RUST_VERSION" + + - name: configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.PUBLIC_ECR_ROLE_ARN }} + aws-region: us-east-1 + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + + - name: Docker metadata + id: meta + uses: docker/metadata-action@v5 + with: + images: | + ${{ env.PUBLIC_ECR_IMAGE }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build container + id: build + uses: docker/build-push-action@v6 + with: + cache-from: type=gha + cache-to: type=gha,mode=max + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + tags: "${{ env.PUBLIC_ECR_IMAGE }}" + outputs: type=image,push-by-digest=true,name-canonical=true,push=true + build-args: | + TARGET_PLATFORM=${{ matrix.target }} + RUST_VERSION=${{ steps.rust-version.outputs.version }} + + - name: Export digest + run: | + mkdir -p ${{ runner.temp }}/digests + digest="${{ steps.build.outputs.digest }}" + touch "${{ runner.temp }}/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ matrix.arch }} + path: ${{ runner.temp }}/digests/* + if-no-files-found: error + retention-days: 1 + + merge-containers: + runs-on: ubuntu-latest permissions: id-token: write contents: read + needs: + - build-container + steps: - - name: download artifact + - name: Download digests uses: actions/download-artifact@v4 with: - name: rotel-lambda-forwarder-${{ matrix.arch }} - path: ./artifacts + path: ${{ runner.temp }}/digests + pattern: digests-* + merge-multiple: true - name: configure AWS credentials uses: aws-actions/configure-aws-credentials@v4 with: - role-to-assume: ${{ secrets.S3_FORWARDER_ARN }} + role-to-assume: ${{ secrets.PUBLIC_ECR_ROLE_ARN }} aws-region: us-east-1 - - name: upload to S3 by version + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + with: + registry-type: public + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.PUBLIC_ECR_IMAGE }} + tags: | + type=ref,event=branch + type=ref,event=tag + type=sha + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + + - name: Create manifest list and push + working-directory: ${{ runner.temp }}/digests run: | - TAG_NAME="${{ github.event.release.tag_name }}" - S3_PATH="s3://${{ env.S3_BUCKET }}/rotel-lambda-forwarder/${TAG_NAME}/${{ matrix.arch }}/rotel-lambda-forwarder.zip" - aws s3 cp ./artifacts/bootstrap.zip "$S3_PATH" --acl public-read - echo "Uploaded to $S3_PATH" + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.PUBLIC_ECR_IMAGE }}@sha256:%s ' *) - - name: upload to S3 latest + - name: Inspect image run: | - S3_PATH="s3://${{ env.S3_BUCKET }}/rotel-lambda-forwarder/latest/${{ matrix.arch }}/rotel-lambda-forwarder.zip" - aws s3 cp ./artifacts/bootstrap.zip "$S3_PATH" --acl public-read - echo "Uploaded to $S3_PATH" + docker buildx imagetools inspect ${{ env.PUBLIC_ECR_IMAGE }}:${{ steps.meta.outputs.version }} - upload-gh-release: - name: upload GH release - ${{ matrix.arch }} - needs: build + # Upload zip builds to S3 + upload-zip: + name: upload-zip - ${{ matrix.arch }} + needs: build-zip strategy: fail-fast: true matrix: @@ -115,23 +216,63 @@ jobs: arch: arm64 runs-on: ubuntu-latest + env: + ARTIFACT_S3_BUCKET_DEV: rotel-lambda-forwarder-dev + ARTIFACT_S3_BUCKET: rotel-lambda-forwarder + + permissions: + id-token: write + contents: read + steps: - name: download artifact uses: actions/download-artifact@v4 with: - name: rotel-lambda-forwarder-${{ matrix.arch }} + name: rotel-lambda-forwarder-zip-${{ matrix.arch }} path: ./artifacts + + - name: set env for push + if: github.event_name == 'push' + run: | + echo "ref_name=${{ github.ref_name }}" >> $GITHUB_ENV + echo "artifact_bucket_name=${{ env.ARTIFACT_S3_BUCKET_DEV }}" >> $GITHUB_ENV + + - name: set env for release + if: github.event_name == 'release' + run: | + echo "ref_name=${{ github.event.release.tag_name }}" >> $GITHUB_ENV + echo "artifact_bucket_name=${{ env.ARTIFACT_S3_BUCKET }}" >> $GITHUB_ENV + + - name: configure AWS credentials (push) + if: github.event_name == 'push' + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.S3_FORWARDER_DEV_ARN }} + aws-region: us-east-1 + + - name: configure AWS credentials (release) + if: github.event_name == 'release' + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.S3_FORWARDER_ARN }} + aws-region: us-east-1 + + - name: upload to S3 by version + run: | + S3_PATH="s3://${{ env.artifact_bucket_name }}/rotel-lambda-forwarder/${{ env.ref_name }}/${{ matrix.arch }}/rotel-lambda-forwarder.zip" + aws s3 cp ./artifacts/bootstrap.zip "$S3_PATH" --acl public-read + echo "Uploaded to $S3_PATH" - - name: upload to GH release - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: upload to S3 latest + if: github.ref_name == 'main' || github.event_name == 'release' run: | - id=$(gh api -H "Accept: application/vnd.github+json" /repos/${{ github.repository }}/releases/tags/${{ github.event.release.tag_name }} --jq .id) - curl --fail-with-body -sS -X POST --data-binary @"./artifacts/bootstrap.zip" -H 'Content-Type: application/octet-stream' -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" "https://uploads.github.com/repos/${{ github.repository }}/releases/$id/assets?name=rotel-lambda-forwarder-${{ github.event.release.tag_name }}_${{ matrix.arch }}.zip" + S3_PATH="s3://${{ env.artifact_bucket_name }}/rotel-lambda-forwarder/latest/${{ matrix.arch }}/rotel-lambda-forwarder.zip" + aws s3 cp ./artifacts/bootstrap.zip "$S3_PATH" --acl public-read + echo "Uploaded to $S3_PATH" upload-cloudformation: name: upload cloudformation - ${{ matrix.arch }} - needs: build + needs: build-zip strategy: fail-fast: true matrix: @@ -144,8 +285,10 @@ jobs: runs-on: ubuntu-latest env: + ARTIFACT_S3_BUCKET_DEV: rotel-lambda-forwarder-dev ARTIFACT_S3_BUCKET: rotel-lambda-forwarder - S3_BUCKET: rotel-cloudformation + CF_S3_BUCKET_DEV: rotel-cloudformation-dev + CF_S3_BUCKET: rotel-cloudformation permissions: id-token: write @@ -158,30 +301,48 @@ jobs: - run: | mkdir -p dist/latest mkdir -p dist/versioned + + - name: set env for push + if: github.event_name == 'push' + run: | + echo "ref_name=${{ github.ref_name }}" >> $GITHUB_ENV + echo "cf_bucket_name=${{ env.CF_S3_BUCKET_DEV }}" >> $GITHUB_ENV + echo "artifact_bucket_name=${{ env.ARTIFACT_S3_BUCKET_DEV }}" >> $GITHUB_ENV + + - name: set env for release + if: github.event_name == 'release' + run: | + echo "ref_name=${{ github.event.release.tag_name }}" >> $GITHUB_ENV + echo "cf_bucket_name=${{ env.CF_S3_BUCKET }}" >> $GITHUB_ENV + echo "artifact_bucket_name=${{ env.ARTIFACT_S3_BUCKET }}" >> $GITHUB_ENV - run: | - yq '.Parameters.S3Bucket.Default = "${{ env.ARTIFACT_S3_BUCKET }}" | - .Resources.ForwarderLambda.Properties.Architectures = ["${{ matrix.arch }}"] | - .Parameters.S3Key.Default = "rotel-lambda-forwarder/latest/${{ matrix.arch }}/rotel-lambda-forwarder.zip"' \ + yq '.Parameters.ForwarderImageTag.Default = "latest" | + .Resources.ForwarderLambda.Properties.Architectures = ["${{ matrix.arch }}"]' \ ./cloudformation-stacks/forwarder-otlp.template.yaml > dist/latest/forwarder-otlp.yaml - yq '.Parameters.S3Bucket.Default = "${{ env.ARTIFACT_S3_BUCKET }}" | - .Resources.ForwarderLambda.Properties.Architectures = ["${{ matrix.arch }}"] | - .Parameters.S3Key.Default = "rotel-lambda-forwarder/latest/${{ matrix.arch }}/rotel-lambda-forwarder.zip"' \ + yq '.Parameters.ForwarderImageTag.Default = "latest" | + .Resources.ForwarderLambda.Properties.Architectures = ["${{ matrix.arch }}"]' \ ./cloudformation-stacks/forwarder-clickhouse.template.yaml > dist/latest/forwarder-clickhouse.yaml - run: | - yq '.Parameters.S3Bucket.Default = "${{ env.ARTIFACT_S3_BUCKET }}" | - .Resources.ForwarderLambda.Properties.Architectures = ["${{ matrix.arch }}"] | - .Parameters.S3Key.Default = "rotel-lambda-forwarder/${{ github.event.release.tag_name }}/${{ matrix.arch }}/rotel-lambda-forwarder.zip"' \ + yq '.Parameters.ForwarderImageTag.Default = "${{ env.ref_name }}" | + .Resources.ForwarderLambda.Properties.Architectures = ["${{ matrix.arch }}"]' \ ./cloudformation-stacks/forwarder-otlp.template.yaml > dist/versioned/forwarder-otlp.yaml - yq '.Parameters.S3Bucket.Default = "${{ env.ARTIFACT_S3_BUCKET }}" | - .Resources.ForwarderLambda.Properties.Architectures = ["${{ matrix.arch }}"] | - .Parameters.S3Key.Default = "rotel-lambda-forwarder/${{ github.event.release.tag_name }}/${{ matrix.arch }}/rotel-lambda-forwarder.zip"' \ + yq '.Parameters.ForwarderImageTag.Default = "${{ env.ref_name }}" | + .Resources.ForwarderLambda.Properties.Architectures = ["${{ matrix.arch }}"]' \ ./cloudformation-stacks/forwarder-clickhouse.template.yaml > dist/versioned/forwarder-clickhouse.yaml - - name: configure AWS credentials + - name: configure AWS credentials (push) + if: github.event_name == 'push' + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.S3_CLOUDFORMATION_DEV_ARN }} + aws-region: us-east-1 + + - name: configure AWS credentials (release) + if: github.event_name == 'release' uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ secrets.S3_CLOUDFORMATION_ARN }} @@ -189,15 +350,15 @@ jobs: - name: upload to S3 by version run: | - TAG_NAME="${{ github.event.release.tag_name }}" - S3_PATH_BASE="s3://${{ env.S3_BUCKET }}/stacks/${TAG_NAME}/${{ matrix.arch }}" + S3_PATH_BASE="s3://${{ env.cf_bucket_name }}/stacks/${{ env.ref_name }}/${{ matrix.arch }}" aws s3 cp ./dist/versioned/forwarder-otlp.yaml "${S3_PATH_BASE}/rotel-lambda-forwarder-otlp.yaml" --acl public-read aws s3 cp ./dist/versioned/forwarder-clickhouse.yaml "${S3_PATH_BASE}/rotel-lambda-forwarder-clickhouse.yaml" --acl public-read - name: upload to S3 latest + if: github.ref_name == 'main' || github.event_name == 'release' run: | - S3_PATH_BASE="s3://${{ env.S3_BUCKET }}/stacks/latest/${{ matrix.arch }}" + S3_PATH_BASE="s3://${{ env.cf_bucket_name }}/stacks/latest/${{ matrix.arch }}" aws s3 cp ./dist/latest/forwarder-otlp.yaml "${S3_PATH_BASE}/rotel-lambda-forwarder-otlp.yaml" --acl public-read aws s3 cp ./dist/latest/forwarder-clickhouse.yaml "${S3_PATH_BASE}/rotel-lambda-forwarder-clickhouse.yaml" --acl public-read diff --git a/Cargo.lock b/Cargo.lock index c956b07..dc2c46f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1806,12 +1806,30 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indoc" +version = "2.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + [[package]] name = "inlinable_string" version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8fae54786f62fb2918dcfae3d568594e50eb9b5c25bf04371af6fe7516452fb" +[[package]] +name = "inventory" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc61209c082fbeb19919bee74b176221b27223e27b65d781eb91af24eb1fb46e" +dependencies = [ + "rustversion", +] + [[package]] name = "ipnet" version = "2.11.0" @@ -1824,6 +1842,15 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.14.0" @@ -1995,6 +2022,12 @@ version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08ab2867e3eeeca90e844d1940eab391c9dc5228783db2ed999acbc0a9ed375a" +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matchers" version = "0.2.0" @@ -2010,6 +2043,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + [[package]] name = "md-5" version = "0.10.6" @@ -2026,6 +2069,15 @@ version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" +[[package]] +name = "memoffset" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" +dependencies = [ + "autocfg", +] + [[package]] name = "mime" version = "0.3.17" @@ -2068,6 +2120,21 @@ dependencies = [ "getrandom 0.2.16", ] +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -2077,6 +2144,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" @@ -2111,6 +2187,22 @@ dependencies = [ "libc", ] +[[package]] +name = "numpy" +version = "0.27.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7aac2e6a6e4468ffa092ad43c39b81c79196c2bb773b8db4085f695efe3bba17" +dependencies = [ + "libc", + "ndarray", + "num-complex", + "num-integer", + "num-traits", + "pyo3", + "pyo3-build-config", + "rustc-hash", +] + [[package]] name = "once_cell" version = "1.21.3" @@ -2301,6 +2393,21 @@ dependencies = [ "spki", ] +[[package]] +name = "portable-atomic" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f89776e4d69bb58bc6993e99ffa1d11f228b839984854c7daeb5d37f87cbe950" + +[[package]] +name = "portable-atomic-util" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8a2f0d8d040d7848a709caf78912debcc3f33ee4b3cac47d73d1e1069e83507" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.4" @@ -2374,7 +2481,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf" dependencies = [ "heck", - "itertools", + "itertools 0.14.0", "log", "multimap", "once_cell", @@ -2394,7 +2501,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools", + "itertools 0.14.0", "proc-macro2", "quote", "syn", @@ -2409,6 +2516,98 @@ dependencies = [ "prost", ] +[[package]] +name = "pyo3" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab53c047fcd1a1d2a8820fe84f05d6be69e9526be40cb03b73f86b6b03e6d87d" +dependencies = [ + "indoc", + "libc", + "memoffset", + "once_cell", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b455933107de8642b4487ed26d912c2d899dec6114884214a0b3bb3be9261ea6" +dependencies = [ + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c85c9cbfaddf651b1221594209aed57e9e5cff63c4d11d1feead529b872a089" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a5b10c9bf9888125d917fb4d2ca2d25c8df94c7ab5a52e13313a07e050a3b02" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03b51720d314836e53327f5871d4c0cfb4fb37cc2c4a11cc71907a86342c40f9" +dependencies = [ + "heck", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn", +] + +[[package]] +name = "pyo3-stub-gen" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca7c2d6e22cba51cc9766b6dee4087218cc445fdf99db62fa4f269e074351b46" +dependencies = [ + "anyhow", + "chrono", + "inventory", + "itertools 0.13.0", + "log", + "maplit", + "num-complex", + "numpy", + "pyo3", + "pyo3-stub-gen-derive", + "serde", + "toml", +] + +[[package]] +name = "pyo3-stub-gen-derive" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ee49d727163163a0c6fc3fee4636c8b5c82e1bb868e85cf411be7ae9e4e5b40" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "quote" version = "1.0.42" @@ -2483,6 +2682,12 @@ dependencies = [ "getrandom 0.3.4", ] +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + [[package]] name = "read-restrict" version = "0.3.0" @@ -2560,8 +2765,8 @@ dependencies = [ [[package]] name = "rotel" -version = "0.1.3" -source = "git+https://github.com/streamfold/rotel?rev=v0.1.3#395d691ffc64eccff4dc1bc2897093a3340d61ed" +version = "0.1.7" +source = "git+https://github.com/streamfold/rotel?rev=v0.1.7#94e95eae6150176a1e89b38a6bc43d778b898739" dependencies = [ "bstr", "bytes", @@ -2595,9 +2800,11 @@ dependencies = [ "pin-project", "prost", "prost-build", + "pyo3", "rand 0.8.5", "read-restrict", "regex", + "rotel_python_processor_sdk", "rustls 0.23.35", "rustls-pki-types", "serde", @@ -2639,12 +2846,15 @@ dependencies = [ "http 1.4.0", "http-body-util", "hyper 1.8.1", + "hyper-rustls 0.27.7", + "hyper-util", "lambda-extension", "lambda_runtime", "lambda_runtime_api_client", "opentelemetry-proto", "regex", "rotel", + "rustls 0.23.35", "serde", "serde_json", "thiserror", @@ -2657,6 +2867,27 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "rotel_python_processor_sdk" +version = "0.1.0" +source = "git+https://github.com/streamfold/rotel?rev=v0.1.7#94e95eae6150176a1e89b38a6bc43d778b898739" +dependencies = [ + "opentelemetry-proto", + "pyo3", + "pyo3-stub-gen", + "pyo3-stub-gen-derive", + "tower", + "tracing", + "tracing-log 0.2.0", + "utilities", +] + +[[package]] +name = "rustc-hash" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" + [[package]] name = "rustc_version" version = "0.4.1" @@ -2713,6 +2944,7 @@ dependencies = [ "aws-lc-rs", "log", "once_cell", + "ring", "rustls-pki-types", "rustls-webpki 0.103.8", "subtle", @@ -2896,6 +3128,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "sha1" version = "0.10.6" @@ -3082,6 +3323,12 @@ dependencies = [ "libc", ] +[[package]] +name = "target-lexicon" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba" + [[package]] name = "tempfile" version = "3.24.0" @@ -3237,6 +3484,47 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "toml_write", + "winnow", +] + +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "tonic" version = "0.13.1" @@ -3463,6 +3751,12 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" +[[package]] +name = "unindent" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" + [[package]] name = "untrusted" version = "0.9.0" @@ -3502,7 +3796,7 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "utilities" version = "0.0.1" -source = "git+https://github.com/streamfold/rotel?rev=v0.1.3#395d691ffc64eccff4dc1bc2897093a3340d61ed" +source = "git+https://github.com/streamfold/rotel?rev=v0.1.7#94e95eae6150176a1e89b38a6bc43d778b898739" dependencies = [ "chrono", "opentelemetry-proto", @@ -3871,6 +4165,15 @@ version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" +[[package]] +name = "winnow" +version = "0.7.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" +dependencies = [ + "memchr", +] + [[package]] name = "wit-bindgen" version = "0.46.0" diff --git a/Cargo.toml b/Cargo.toml index 2a03668..8e80e64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -15,6 +15,8 @@ hex = "0.4" chrono = "0.4" clap = { version = "4.5.53", features = ["derive", "env"] } hyper = { version = "1", features = ["full"] } +hyper-rustls = { version = "0.27", default-features = false, features = ["http1", "http2", "native-tokio", "ring"] } +hyper-util = { version = "0.1", features = ["client", "client-legacy", "http1", "http2", "tokio"] } lambda_runtime = { version = "1.0.1", features = ["graceful-shutdown"] } lambda_runtime_api_client = "1.0.1" async-stream = "0.3" @@ -22,6 +24,7 @@ http-body-util = "0.1" http = "1.0" tokio-stream = "0.1" regex = "1.10" +rustls = "0.23.20" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0.145" tokio = { version = "1", features = ["macros"] } @@ -32,11 +35,14 @@ tracing-appender = "0.2.3" tracing-subscriber = { version = "0.3.20", features = ["env-filter"] } tower = { version = "0.5.2", features = ["retry", "timeout"] } dotenvy = { git = "https://github.com/streamfold/dotenvy", branch = "custom-substitution" } -rotel = { git = "https://github.com/streamfold/rotel", rev = "v0.1.3", default-features = false} -#rotel = { path = "../rotel", default-features = false} +rotel = { git = "https://github.com/streamfold/rotel", rev = "v0.1.7", default-features = false, optional = true } lambda-extension = "1.0.1" opentelemetry-proto = "0.30.0" flate2 = "1.1" [dev-dependencies] futures-util = "0.3" + +[features] +default = ["rotel"] +pyo3 = ["rotel/pyo3"] diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..fe6b958 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,60 @@ +FROM public.ecr.aws/lambda/python:3.14 AS builder + +ARG TARGET_PLATFORM +ARG RUST_VERSION + +ENV DEBIAN_FRONTEND=noninteractive + +# Install build dependencies +RUN dnf install -y \ + clang \ + make \ + cmake \ + openssl-devel \ + protobuf-compiler \ + libzstd-devel \ + git \ + tar \ + gzip \ + perl \ + ca-certificates \ + file \ + && dnf clean all + +# Install Rust +RUN curl -sSf https://sh.rustup.rs | sh -s -- -y --default-toolchain ${RUST_VERSION} +ENV PATH="/root/.cargo/bin:${PATH}" + +RUN rustup target add ${TARGET_PLATFORM} + +RUN pip3 install cargo-lambda + +WORKDIR /build + +# Copy manifests first for better layer caching +COPY Cargo.toml Cargo.lock ./ +COPY rust-toolchain.toml ./ + +# Create a dummy main.rs to build dependencies +RUN mkdir -p src && \ + echo "fn main() {}" > src/main.rs && \ + cargo build --release --features pyo3; \ + rm -rf src + +# Copy actual source code +COPY src ./src + +# Build with pyo3 feature enabled, targeting >=2.34 glibc required for linking libpython +RUN touch src/main.rs && \ + cargo lambda build --release --features pyo3 --target ${TARGET_PLATFORM}.2.34 + +FROM public.ecr.aws/lambda/python:3.14 + +# Copy the bootstrap binary to the Lambda expected location +COPY --from=builder /build/target/lambda/rotel-lambda-forwarder/bootstrap ${LAMBDA_TASK_ROOT}/bootstrap + +# Ensure the binary is executable +RUN chmod +x ${LAMBDA_TASK_ROOT}/bootstrap + +# Set the ENTRYPOINT to run the bootstrap binary as a custom runtime +ENTRYPOINT [ "/var/task/bootstrap" ] diff --git a/Makefile b/Makefile index c1deadd..241e69d 100644 --- a/Makefile +++ b/Makefile @@ -1,14 +1,70 @@ -.PHONY: build clippy deploy +.PHONY: clippy build-zip build-container deploy-zip deploy-container clean +# Required for deployment IAM_ROLE ?= "arn:aws:iam::123456789012:role/lambda-execution-role" +# Build configuration FUNC_NAME ?= "rotel-lambda-forwarder" +ARCH ?= x86_64 +RUST_VERSION := $(shell grep '^channel = ' rust-toolchain.toml | sed 's/channel = "\(.*\)"/\1/') +BOOTSTRAP_ZIP := bootstrap.zip -build: - cargo lambda build --release +# Docker platform mapping +ifeq ($(ARCH),x86_64) + TARGET_PLATFORM := x86_64-unknown-linux-gnu + DOCKER_PLATFORM := linux/amd64 +else ifeq ($(ARCH),arm64) + TARGET_PLATFORM := aarch64-unknown-linux-gnu + DOCKER_PLATFORM := linux/arm64 +else + $(error Unsupported architecture: $(ARCH). Use x86_64 or arm64) +endif clippy: cargo clippy -deploy: build - cargo lambda deploy --role ${IAM_ROLE} --binary-name "rotel-lambda-forwarder" ${FUNC_NAME} +# Build 1: Direct cargo lambda build (produces bootstrap.zip) +build-zip: + @echo "Building $(FUNC_NAME) for $(ARCH) using cargo lambda..." + @echo "Target: $(TARGET_PLATFORM)" + + cargo lambda build --release -o zip --target $(TARGET_PLATFORM) + + @mkdir -p tmp/ + @cp target/lambda/rotel-lambda-forwarder/bootstrap.zip ./tmp/$(BOOTSTRAP_ZIP) + @echo "Build complete: ./tmp/$(BOOTSTRAP_ZIP)" + +# Build 2: Docker container build with pyo3 feature (produces Docker image) +build-container: + @echo "Building $(FUNC_NAME) container for $(ARCH) with pyo3 feature..." + @echo "Rust version: $(RUST_VERSION)" + @echo "Docker platform: $(DOCKER_PLATFORM)" + + # Build Docker image with pyo3 feature + docker buildx build \ + --platform $(DOCKER_PLATFORM) \ + --load \ + -t rotel-lambda-forwarder:$(ARCH) \ + --build-arg TARGET_PLATFORM=$(TARGET_PLATFORM) \ + --build-arg RUST_VERSION=$(RUST_VERSION) \ + --progress plain \ + . + + @echo "Container build complete: rotel-lambda-forwarder:$(ARCH)" + +# Deploy using zip artifact +deploy-zip: build-zip + @echo "Deploying $(FUNC_NAME) from zip..." + IAM_ROLE=$(IAM_ROLE) ./scripts/deploy-lambda.sh ./tmp/$(BOOTSTRAP_ZIP) $(FUNC_NAME) + +# Deploy using container image +deploy-container: build-container + @echo "Container deployment requires pushing to ECR and updating Lambda function" + @echo "Image: rotel-lambda-forwarder:$(ARCH)" + @echo "Please use AWS CLI or console to deploy the container image" + +# Clean build artifacts +clean: + rm -rf ./tmp/ + rm -rf target/ + docker rmi rotel-lambda-forwarder:$(ARCH) 2>/dev/null || true diff --git a/README.md b/README.md index 9216e84..dd630ac 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ _Performance of 12 hours of VPC flow log forwarding to ClickHouse Cloud. Average - **OpenTelemetry Native**: Transforms all logs to OpenTelemetry format - **Multiple Export Targets**: Supports OTLP HTTP/gRPC and other exporters via Rotel +- **Python Log Processors**: Filter, transform, and enrich logs with Python before exporting - **Automatic parsing**: Support for JSON and key=value parsing, with automatic detection - **Log stream parser mapping**: Pre-built parser rules for known AWS CW log groups/streams - **AWS Resource Attributes**: Automatically enriches logs with AWS Lambda and CloudWatch log group tags @@ -34,32 +35,262 @@ expand as we verify support for additional services. ## Deploying -### Deploy with Cloudformation +There are two deployment methods available: -At the moment these only support deploying in the `us-east-1` AWS region. See the section "Manual Deployment" for multi-region -instructions. +1. **Docker Container (Recommended)** - Deploy using container images from Amazon ECR Public + - Supports all features including Python log processors + - Automatic image management via CloudFormation + - Available for both x86_64 and arm64 architectures + +2. **ZIP File** - Deploy using pre-built Lambda ZIP packages + - Simpler deployment for basic use cases + - Does not support Python log processors + - Available in release downloads + +### Deploy with CloudFormation (Docker Container - Recommended) + +The CloudFormation templates automatically pull container images from Amazon ECR Public and copy them to your private ECR repository. + +**Note:** Python log processors are only supported when using the Docker container deployment method. #### Export to OTLP endpoint Launch this stack to export CloudWatch logs to any OTLP compatible endpoint. -| **Region** | **x86_64** | **arm64** | -| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `us-east-1` | [![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=rotel-lambda-forwarder-otlp&templateURL=https://rotel-cloudformation.s3.us-east-1.amazonaws.com/stacks/latest/x86_64/rotel-lambda-forwarder-otlp.yaml) | [![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=rotel-lambda-forwarder-otlp&templateURL=https://rotel-cloudformation.s3.us-east-1.amazonaws.com/stacks/latest/arm64/rotel-lambda-forwarder-otlp.yaml) | +| **x86_64** | **arm64**| +|------------|----------| +| [![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)][otlp-stack-x86-84] | [![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)][otlp-stack-arm64] | + +[otlp-stack-x86-84]: https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=rotel-lambda-forwarder-otlp&templateURL=https://rotel-cloudformation.s3.us-east-1.amazonaws.com/stacks/latest/x86_64/rotel-lambda-forwarder-otlp.yaml +[otlp-stack-arm64]: https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=rotel-lambda-forwarder-otlp&templateURL=https://rotel-cloudformation.s3.us-east-1.amazonaws.com/stacks/latest/arm64/rotel-lambda-forwarder-otlp.yaml #### Export to ClickHouse Launch this stack to export CloudWatch logs to ClickHouse. -| **Region** | **x86_64** | **arm64** | -| ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `us-east-1` | [![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=rotel-lambda-forwarder-clickhouse&templateURL=https://rotel-cloudformation.s3.us-east-1.amazonaws.com/stacks/latest/x86_64/rotel-lambda-forwarder-clickhouse.yaml) | [![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=rotel-lambda-forwarder-clickhouse&templateURL=https://rotel-cloudformation.s3.us-east-1.amazonaws.com/stacks/latest/arm64/rotel-lambda-forwarder-clickhouse.yaml) | +| **x86_64** | **arm64**| +|------------|----------| +| [![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)][ch-stack-x86-84] | [![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)][ch-stack-arm64] | + +[ch-stack-x86-84]: https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=rotel-lambda-forwarder-clickhouse&templateURL=https://rotel-cloudformation.s3.us-east-1.amazonaws.com/stacks/latest/x86_64/rotel-lambda-forwarder-clickhouse.yaml +[ch-stack-arm64]: https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=rotel-lambda-forwarder-clickhouse&templateURL=https://rotel-cloudformation.s3.us-east-1.amazonaws.com/stacks/latest/arm64/rotel-lambda-forwarder-clickhouse.yaml + +### Upgrading with CloudFormation + +To upgrade the Lambda function to use the latest upstream image, you can update the CloudFormation stack using the **ForceRedeploy** parameter: + +#### Method 1: Force Redeploy with Same Tag + +If you're using a tag like `latest` and want to pull the newest version: + +1. Navigate to your CloudFormation stack in the AWS Console +2. Click **Update** on the stack +3. Select **Use current template** +4. Find the **ForceRedeploy** parameter +5. Increment the value (e.g., change from `1` to `2`, or use a timestamp like `2024-01-15`) +6. Complete the stack update + +The CloudFormation stack will automatically: +- Pull the latest image from Amazon ECR Public (`public.ecr.aws/streamfold/rotel-lambda-forwarder`) +- Copy it to your private ECR repository +- Redeploy the Lambda function with the updated image + +#### Method 2: Upgrade to Specific Version + +To upgrade to a specific version tag (e.g., `v1.2.3`): + +1. Navigate to your CloudFormation stack in the AWS Console +2. Click **Update** on the stack +3. Select **Use current template** +4. Find the **ForwarderImageTag** parameter +5. Change the value to the desired version (e.g., from `latest` to `v1.2.3`) +6. Complete the stack update + +**Note:** When using CloudFormation deployment, you don't need to manually pull and push images - the stack handles this automatically through a CodeBuild project. ### Manual Deployment to AWS -For production deployments, follow these steps to manually deploy the Lambda function using pre-built artifacts. +You can deploy the forwarder manually using either the Docker container (recommended) or ZIP file method. + +#### Option 1: Docker Container Deployment (Recommended) + +**Supports:** All features including Python log processors -You can download pre-built deployment Lambda .zip files for x86_64 and arm64 architectures from the [Releases](https://github.com/streamfold/rotel-lambda-forwarder/releases) page, or from the following links: +The forwarder is available as a container image in [Amazon ECR Public](https://gallery.ecr.aws/streamfold/rotel-lambda-forwarder): + +``` +public.ecr.aws/streamfold/rotel-lambda-forwarder:latest +``` + +##### Step 1: Copy Image to Your Private ECR + +First, create a private ECR repository: + +```bash +aws ecr create-repository \ + --repository-name rotel-lambda-forwarder \ + --region YOUR_REGION +``` + +Pull the image from ECR Public and push to your private repository: + +```bash +# Login to ECR Public (us-east-1 only) +aws ecr-public get-login-password --region us-east-1 | \ + docker login --username AWS --password-stdin public.ecr.aws + +# Pull the image +docker pull public.ecr.aws/streamfold/rotel-lambda-forwarder:latest + +# Login to your private ECR +aws ecr get-login-password --region YOUR_REGION | \ + docker login --username AWS --password-stdin YOUR_ACCOUNT_ID.dkr.ecr.YOUR_REGION.amazonaws.com + +# Tag the image for your private registry +docker tag public.ecr.aws/streamfold/rotel-lambda-forwarder:latest \ + YOUR_ACCOUNT_ID.dkr.ecr.YOUR_REGION.amazonaws.com/rotel-lambda-forwarder:latest + +# Push to your private ECR +docker push YOUR_ACCOUNT_ID.dkr.ecr.YOUR_REGION.amazonaws.com/rotel-lambda-forwarder:latest +``` + +Replace: +- `YOUR_REGION` with your AWS region (e.g., `us-west-2`) +- `YOUR_ACCOUNT_ID` with your AWS account ID + +##### Step 2: Create IAM Execution Role + +Create an IAM role with the necessary permissions for the Lambda function: + +```bash +aws iam create-role \ + --role-name rotel-lambda-forwarder-role \ + --assume-role-policy-document '{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": {"Service": "lambda.amazonaws.com"}, + "Action": "sts:AssumeRole" + }] + }' +``` + +Attach the basic Lambda execution policy: + +```bash +aws iam attach-role-policy \ + --role-name rotel-lambda-forwarder-role \ + --policy-arn arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole +``` + +Create and attach a custom policy with required permissions: + +```bash +aws iam create-policy \ + --policy-name rotel-lambda-forwarder-policy \ + --policy-document '{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "logs:ListTagsForResource" + ], + "Resource": "arn:aws:logs:*:*:log-group:*" + }, + { + "Effect": "Allow", + "Action": [ + "ec2:DescribeFlowLogs" + ], + "Resource": "*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:PutObject" + ], + "Resource": "arn:aws:s3:::YOUR_BUCKET_NAME/rotel-lambda-forwarder/*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": "arn:aws:s3:::YOUR_BUCKET_NAME" + }, + { + "Effect": "Allow", + "Action": [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:BatchGetImage" + ], + "Resource": "*" + } + ] + }' + +aws iam attach-role-policy \ + --role-name rotel-lambda-forwarder-role \ + --policy-arn arn:aws:iam::YOUR_ACCOUNT_ID:policy/rotel-lambda-forwarder-policy +``` + +##### Step 3: Create Lambda Function from Container Image + +```bash +aws lambda create-function \ + --function-name rotel-lambda-forwarder \ + --package-type Image \ + --code ImageUri=YOUR_ACCOUNT_ID.dkr.ecr.YOUR_REGION.amazonaws.com/rotel-lambda-forwarder:latest \ + --role arn:aws:iam::YOUR_ACCOUNT_ID:role/rotel-lambda-forwarder-role \ + --timeout 30 \ + --memory-size 256 \ + --architectures x86_64 \ + --region YOUR_REGION \ + --environment Variables="{ + ROTEL_EXPORTER=otlp, + ROTEL_OTLP_EXPORTER_ENDPOINT=https://your-otlp-endpoint.com, + FORWARDER_S3_BUCKET=your-cache-bucket-name + }" +``` + +**Important parameters:** +- `--package-type Image`: Indicates this is a container-based Lambda +- `--code ImageUri`: The full URI of your container image in ECR +- `--architectures`: Must match the image architecture (`x86_64` or `arm64`) +- `--timeout`: Adjust based on your log volume (recommended: 30 seconds) +- `--memory-size`: Adjust based on log volume (recommended: 256-512 MB) + +##### Step 4: Update Function with New Image (for updates) + +To update the function with a new image version: + +```bash +# Pull new image from ECR Public +docker pull public.ecr.aws/streamfold/rotel-lambda-forwarder:v1.2.3 + +# Tag and push to your private ECR +docker tag public.ecr.aws/streamfold/rotel-lambda-forwarder:v1.2.3 \ + YOUR_ACCOUNT_ID.dkr.ecr.YOUR_REGION.amazonaws.com/rotel-lambda-forwarder:v1.2.3 + +docker push YOUR_ACCOUNT_ID.dkr.ecr.YOUR_REGION.amazonaws.com/rotel-lambda-forwarder:v1.2.3 + +# Update Lambda function +aws lambda update-function-code \ + --function-name rotel-lambda-forwarder \ + --image-uri YOUR_ACCOUNT_ID.dkr.ecr.YOUR_REGION.amazonaws.com/rotel-lambda-forwarder:v1.2.3 +``` + +--- + +#### Option 2: ZIP File Deployment + +**Note:** Python log processors are **not supported** with ZIP file deployment. Use Docker container deployment if you need Python processor support. + +You can download pre-built Lambda ZIP files for x86_64 and arm64 architectures from the [Releases](https://github.com/streamfold/rotel-lambda-forwarder/releases) page, or from the following links: | **Region** | **x86_64** | **arm64** | | ---------- | ------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | @@ -83,7 +314,7 @@ _NOTE_: These are located in the AWS us-east-1 region, so you can only create La you need to create the function in a different region, you'll need to copy the `rotel-lambda-forwarder.zip` to a different bucket in the same region as the function. -#### 1. Create IAM Execution Role +##### Step 1: Create IAM Execution Role (ZIP Deployment) Create an IAM role with the necessary permissions for the Lambda function: @@ -164,7 +395,7 @@ Replace `YOUR_BUCKET_NAME` with your S3 bucket name and `YOUR_ACCOUNT_ID` with y Note the role ARN from the output for the next step. -#### 2. Create the Lambda Function +##### Step 2: Create the Lambda Function (ZIP Deployment) Create the Lambda function using the AWS CLI: @@ -186,13 +417,12 @@ aws lambda create-function \ ``` **Important parameters:** - - `--runtime`: Use `provided.al2023` for Amazon Linux 2023 custom runtime -- `--architectures`: Must match your build target (`x86_64`\_ +- `--architectures`: Must match your build target (`x86_64` or `arm64`) - `--timeout`: Adjust based on your log volume (recommended: 30 seconds) - `--memory-size`: Adjust based on log volume (recommended: 256-512 MB) -#### 3. Update Function Code (for updates) +##### Step 3: Update Function Code (for updates) To update an existing function with the latest version: @@ -204,7 +434,7 @@ aws lambda update-function-code \ --s3-key rotel-lambda-forwarder/latest/x86_64/rotel-lambda-forwarder.zip ``` -#### 4. Configure Function Settings (optional) +##### Step 4: Configure Function Settings (optional) Update environment variables: @@ -348,6 +578,29 @@ be set to the function's maximum timeout. ROTEL_EXPORTER_RETRY_MAX_ELAPSED_TIME=30s ``` +### OTLP Log Processors from S3 + +You can configure OTLP log processors to transform or filter logs before they are exported. The Lambda Forwarder supports loading processor configurations from any HTTP endpoint or S3 bucket. + +See the Python [Processor SDK](https://rotel.dev/docs/category/processor-sdk) for how to construct these processors. + +Set the `FORWARDER_OTLP_LOG_PROCESSORS` environment variable of the Lambda Forwarder function, or through +the `LogProcessors` CloudFormation parameter. + +```bash +FORWARDER_OTLP_LOG_PROCESSORS="https://gist.githubusercontent.com/mheffner/4d4aaa0f3f7ffc620fb740763f4e0098/raw/parse_vpc_logs.py,s3://my-bucket-name/processors/filter-ecs-logs.py" +``` + +**Important**: If you load processors from an S3 bucket, make sure the Lambda environment has IAM permissions +to read from the bucket. + +**Features**: +- Multiple URIs can be specified as a comma-separated list (supports: http, https, or s3) +- Processors are downloaded to `/tmp/log_processors/` +- Processors are executed in the order specified in `FORWARDER_OTLP_LOG_PROCESSORS` + +**Example**: See [examples/processors/parse_vpc_logs.py](examples/processors/parse_vpc_logs.py) for an example of filtering VPC logs that indicate a REJECTED flow. + ## Setting Up CloudWatch Logs Subscription To forward logs from CloudWatch Logs to the Lambda function, create a subscription filter: diff --git a/cloudformation-stacks/forwarder-clickhouse.template.yaml b/cloudformation-stacks/forwarder-clickhouse.template.yaml index 62e850a..00224d5 100644 --- a/cloudformation-stacks/forwarder-clickhouse.template.yaml +++ b/cloudformation-stacks/forwarder-clickhouse.template.yaml @@ -16,16 +16,18 @@ Metadata: default: "Processing Configuration" Parameters: - ResourceAttributes + - LogProcessors - Label: default: "Function Configuration" Parameters: - FunctionMemorySize - FunctionTimeout - Label: - default: "Artifact Configuration (Keep defaults)" + default: "Container Image Configuration" Parameters: - - S3Bucket - - S3Key + - ForwarderImageTag + - ForceRedeploy + ParameterLabels: ClickhouseEndpoint: default: "ClickHouse Endpoint" @@ -41,14 +43,16 @@ Metadata: default: "ClickHouse Enable JSON" ResourceAttributes: default: "Resource Attributes" + LogProcessors: + default: "Log Processors" FunctionMemorySize: default: "Function Memory Size" FunctionTimeout: default: "Function Timeout" - S3Bucket: - default: "S3 Bucket" - S3Key: - default: "S3 Key" + ForwarderImageTag: + default: "Forwarder Image Tag" + ForceRedeploy: + default: "Force Image Copy Trigger" Parameters: ClickhouseEndpoint: @@ -85,7 +89,11 @@ Parameters: ResourceAttributes: Type: String Description: Resource attributes to add to all telemetry data, e.g., service.name=my-service,deployment.environment=production. Leave empty for no additional attributes. - Default: "aws.function.name=rotel-lambda-forwarder" + Default: "" + LogProcessors: + Type: String + Description: "Comma-separated list of URIs of log processors to load on startup. (Supports: http://, https://, or s3:// URIs)" + Default: "" FunctionMemorySize: Type: Number Description: Memory size (in MB) allocated to the Lambda function. @@ -98,15 +106,268 @@ Parameters: Default: 30 MinValue: 15 MaxValue: 900 - S3Bucket: + ForwarderImageTag: Type: String - Description: S3 bucket containing Rotel Lambda Forwarder. - Default: "" # updated on release - S3Key: + Description: Container image tag to deploy from public.ecr.aws/streamfold/rotel-lambda-forwarder + Default: "latest" + ForceRedeploy: Type: String - Description: S3 key for the Rotel Lambda Forwarder. - Default: "" # updated on release + Description: Increment to force an upgrade by pulling a new image and redeploying (e.g., increment a number or use a timestamp) + Default: "1" + Resources: + # Private ECR Repository for the forwarder image + ForwarderECRRepository: + Type: AWS::ECR::Repository + DeletionPolicy: Retain + Properties: + RepositoryName: !Sub "rotel-lambda-forwarder-${AWS::StackName}" + ImageScanningConfiguration: + ScanOnPush: true + Tags: + - Key: "PartOf" + Value: !Ref AWS::StackName + - Key: "Platform" + Value: "Rotel" + + # CodeBuild project to copy image from public ECR to private ECR + ImageCopyProject: + Type: AWS::CodeBuild::Project + Properties: + Name: !Sub "${AWS::StackName}-image-copy" + Description: Copies container image from public ECR to private ECR + ServiceRole: !GetAtt ImageCopyRole.Arn + Artifacts: + Type: NO_ARTIFACTS + Environment: + Type: LINUX_CONTAINER + ComputeType: BUILD_GENERAL1_SMALL + Image: aws/codebuild/standard:7.0 + PrivilegedMode: true + EnvironmentVariables: + - Name: SOURCE_IMAGE + Value: !Sub "public.ecr.aws/streamfold/rotel-lambda-forwarder:${ForwarderImageTag}" + - Name: TARGET_IMAGE + Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ForwarderECRRepository}:${ForwarderImageTag}" + - Name: AWS_DEFAULT_REGION + Value: !Ref AWS::Region + - Name: AWS_ACCOUNT_ID + Value: !Ref AWS::AccountId + Source: + Type: NO_SOURCE + BuildSpec: | + version: 0.2 + phases: + pre_build: + commands: + - echo "Logging in to Amazon ECR in region $AWS_DEFAULT_REGION..." + - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com + build: + commands: + - echo "Pulling image from public ECR ($SOURCE_IMAGE)..." + - docker pull $SOURCE_IMAGE + - echo "Tagging image for private ECR..." + - docker tag $SOURCE_IMAGE $TARGET_IMAGE + - echo "Pushing image to private ECR ($TARGET_IMAGE)..." + - docker push $TARGET_IMAGE + post_build: + commands: + - echo "Image copy completed successfully" + Tags: + - Key: "PartOf" + Value: !Ref AWS::StackName + - Key: "Platform" + Value: "Rotel" + + # IAM Role for CodeBuild + ImageCopyRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: codebuild.amazonaws.com + ManagedPolicyArns: + - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser + Policies: + - PolicyName: CodeBuildLogsPolicy + PolicyDocument: + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: + - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${AWS::StackName}-image-copy" + - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${AWS::StackName}-image-copy:*" + + Tags: + - Key: "PartOf" + Value: !Ref AWS::StackName + - Key: "Platform" + Value: "Rotel" + + # Lambda function for Custom Resource + ImageCopyCustomResourceFunction: + Type: AWS::Lambda::Function + Properties: + FunctionName: !Sub "${AWS::StackName}-image-copy-cr" + Description: !Sub "Lambda function to trigger CodeBuild to pull ECR image for stack: ${AWS::StackName}" + Runtime: python3.14 + Handler: index.handler + Role: !GetAtt ImageCopyCustomResourceRole.Arn + Timeout: 900 + Code: + ZipFile: | + import json + import boto3 + import cfnresponse + import time + + codebuild = boto3.client('codebuild') + lambda_client = boto3.client('lambda') + + def handler(event, context): + print(f"Received event: {json.dumps(event)}") + + response_data = {} + physical_resource_id = event.get('PhysicalResourceId', 'ImageCopyBuild') + + try: + request_type = event['RequestType'] + project_name = event['ResourceProperties']['ProjectName'] + lambda_function_name = event['ResourceProperties'].get('LambdaFunctionName') + image_tag = event['ResourceProperties'].get('ImageTag', 'latest') + + if request_type in ['Create', 'Update']: + # Start the CodeBuild project + print(f"Starting CodeBuild project: {project_name}") + build_response = codebuild.start_build(projectName=project_name) + build_id = build_response['build']['id'] + print(f"Build started with ID: {build_id}") + + # Wait for build to complete + print("Waiting for build to complete...") + while True: + build_info = codebuild.batch_get_builds(ids=[build_id])['builds'][0] + build_status = build_info['buildStatus'] + print(f"Build status: {build_status}") + + if build_status == 'SUCCEEDED': + print("Build succeeded!") + response_data['BuildId'] = build_id + response_data['BuildStatus'] = build_status + + # If this is an Update and Lambda function exists, force Lambda to update + if request_type == 'Update' and lambda_function_name: + try: + print(f"Forcing Lambda function update: {lambda_function_name}") + + # Get Lambda function details to determine the image URI + function_details = lambda_client.get_function(FunctionName=lambda_function_name) + current_image_uri = function_details['Code']['ImageUri'] + print(f"Current ImageUri: {current_image_uri}") + + # Construct new image URI with the same repository but potentially new tag + # The image should already be copied to ECR by CodeBuild + image_uri_base = current_image_uri.rsplit(':', 1)[0] + new_image_uri = f"{image_uri_base}:{image_tag}" + print(f"New ImageUri: {new_image_uri}") + + # Update the Lambda function to force it to pull the new image + lambda_client.update_function_code( + FunctionName=lambda_function_name, + ImageUri=new_image_uri, + Publish=True + ) + print("Lambda function update triggered successfully") + response_data['LambdaUpdated'] = 'true' + response_data['NewImageUri'] = new_image_uri + except Exception as lambda_error: + print(f"Warning: Could not update Lambda function: {str(lambda_error)}") + # Don't fail the whole operation if Lambda update fails + response_data['LambdaUpdated'] = 'false' + response_data['LambdaUpdateError'] = str(lambda_error) + + cfnresponse.send(event, context, cfnresponse.SUCCESS, response_data, physical_resource_id) + return + elif build_status in ['FAILED', 'FAULT', 'TIMED_OUT', 'STOPPED']: + print(f"Build failed with status: {build_status}") + cfnresponse.send(event, context, cfnresponse.FAILED, + {'Error': f'Build failed with status: {build_status}'}, + physical_resource_id) + return + + time.sleep(10) + + elif request_type == 'Delete': + # Nothing to do on delete + print("Delete request - no action needed") + cfnresponse.send(event, context, cfnresponse.SUCCESS, response_data, physical_resource_id) + return + + except Exception as e: + print(f"Error: {str(e)}") + cfnresponse.send(event, context, cfnresponse.FAILED, + {'Error': str(e)}, physical_resource_id) + Tags: + - Key: "PartOf" + Value: !Ref AWS::StackName + - Key: "Platform" + Value: "Rotel" + + # IAM Role for Custom Resource Lambda + ImageCopyCustomResourceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: lambda.amazonaws.com + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + Policies: + - PolicyName: CodeBuildStartPolicy + PolicyDocument: + Statement: + - Effect: Allow + Action: + - codebuild:StartBuild + - codebuild:BatchGetBuilds + Resource: !GetAtt ImageCopyProject.Arn + - PolicyName: LambdaUpdatePolicy + PolicyDocument: + Statement: + - Effect: Allow + Action: + - lambda:UpdateFunctionCode + - lambda:GetFunction + Resource: !Sub "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${AWS::StackName}" + Tags: + - Key: "PartOf" + Value: !Ref AWS::StackName + - Key: "Platform" + Value: "Rotel" + + # Custom Resource to trigger image copy + ImageCopyCustomResource: + Type: Custom::ImageCopy + DependsOn: + - ForwarderECRRepository + - ImageCopyProject + Properties: + ServiceToken: !GetAtt ImageCopyCustomResourceFunction.Arn + ProjectName: !Ref ImageCopyProject + ImageTag: !Ref ForwarderImageTag + LambdaFunctionName: !Ref AWS::StackName + # This triggers re-execution when either parameter changes + Trigger: !Sub "${ForwarderImageTag}-${ForceRedeploy}" + ForwarderBucket: Type: AWS::S3::Bucket DeletionPolicy: Retain @@ -128,6 +389,7 @@ Resources: Value: !Ref AWS::StackName - Key: "Platform" Value: "Rotel" + ForwarderLogGroup: Type: "AWS::Logs::LogGroup" DeletionPolicy: Delete @@ -139,6 +401,7 @@ Resources: Value: !Ref AWS::StackName - Key: "Platform" Value: "Rotel" + ForwarderRole: Type: AWS::IAM::Role Properties: @@ -157,6 +420,7 @@ Resources: Value: "RotelLambdaForwarder" - Key: "Platform" Value: "Rotel" + ForwarderPolicy: Type: AWS::IAM::Policy Properties: @@ -167,6 +431,12 @@ Resources: - logs:PutLogEvents Effect: Allow Resource: !GetAtt ForwarderLogGroup.Arn + - Action: + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + - ecr:BatchCheckLayerAvailability + Effect: Allow + Resource: !GetAtt ForwarderECRRepository.Arn - Action: - lambda:AddPermission - lambda:RemovePermission @@ -193,15 +463,17 @@ Resources: PolicyName: !Sub "${AWS::StackName}-forwarder-lambda-policy" Roles: - !Ref "ForwarderRole" + ForwarderLambda: Type: AWS::Lambda::Function + DependsOn: + - ImageCopyCustomResource Properties: FunctionName: !Ref AWS::StackName - Runtime: provided.al2023 - Handler: bootstrap + Description: "Rotel Lambda Forwarder - Convert and forward CloudWatch logs as OTLP" + PackageType: Image Code: - S3Bucket: !Ref S3Bucket - S3Key: !Ref S3Key + ImageUri: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ForwarderECRRepository}:${ForwarderImageTag}" Role: !GetAtt - ForwarderRole - Arn @@ -230,6 +502,8 @@ Resources: ROTEL_OTEL_RESOURCE_ATTRIBUTES: !Ref "ResourceAttributes" ROTEL_EXPORTER_RETRY_MAX_ELAPSED_TIME: !Sub "${FunctionTimeout}s" FORWARDER_S3_BUCKET: !Ref ForwarderBucket + FORWARDER_OTLP_LOG_PROCESSORS: !Ref LogProcessors + ForwarderLambdaPermission: Type: AWS::Lambda::Permission DependsOn: @@ -240,6 +514,7 @@ Resources: Principal: !Sub "logs.${AWS::Region}.amazonaws.com" SourceAccount: !Ref "AWS::AccountId" SourceArn: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:*:*" + Outputs: ForwarderLambdaARN: Description: The ARN of the created Forwarder Lambda @@ -250,6 +525,9 @@ Outputs: ForwarderBucketName: Description: The name of the S3 bucket used for tag caching Value: !Ref ForwarderBucket + ForwarderECRRepositoryUri: + Description: The URI of the private ECR repository + Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ForwarderECRRepository}" SetupInstructions: Description: Instructions for setting up CloudWatch Logs subscription filters Value: !Sub | @@ -259,3 +537,11 @@ Outputs: --filter-name forward-to-rotel \ --filter-pattern "" \ --destination-arn ${ForwarderLambda.Arn} + ImageUpdateInstructions: + Description: How to update the container image + Value: | + To pull a new version of the image from public ECR: + 1. Update the stack with a new ForwarderImageTag value (e.g., change from "latest" to "v1.2.3"), OR + 2. Increment the ForceRedeploy parameter (e.g., change from "1" to "2") to re-pull the same tag, OR + + The CodeBuild project will automatically pull the image from public ECR and push it to your private ECR. diff --git a/cloudformation-stacks/forwarder-otlp.template.yaml b/cloudformation-stacks/forwarder-otlp.template.yaml index 6a4b92e..e475279 100644 --- a/cloudformation-stacks/forwarder-otlp.template.yaml +++ b/cloudformation-stacks/forwarder-otlp.template.yaml @@ -9,17 +9,22 @@ Metadata: - OtlpEndpoint - OtlpProtocol - OtlpCustomHeaders + - Label: + default: "Processing Configuration" + Parameters: - ResourceAttributes + - LogProcessors - Label: default: "Function Configuration" Parameters: - FunctionMemorySize - FunctionTimeout - Label: - default: "Artifact Configuration (Keep defaults)" + default: "Container Image Configuration" Parameters: - - S3Bucket - - S3Key + - ForwarderImageTag + - ForceRedeploy + ParameterLabels: OtlpEndpoint: default: "OTLP Endpoint" @@ -29,14 +34,16 @@ Metadata: default: "Custom Headers" ResourceAttributes: default: "Resource Attributes" + LogProcessors: + default: "Log Processors" FunctionMemorySize: default: "Function Memory Size" FunctionTimeout: default: "Function Timeout" - S3Bucket: - default: "S3 Bucket" - S3Key: - default: "S3 Key" + ForwarderImageTag: + default: "Forwarder Image Tag" + ForceRedeploy: + default: "Force Image Copy Trigger" Parameters: OtlpEndpoint: @@ -57,7 +64,11 @@ Parameters: ResourceAttributes: Type: String Description: Resource attributes to add to all telemetry data, e.g., service.name=my-service,deployment.environment=production. Leave empty for no additional attributes. - Default: "aws.function.name=rotel-lambda-forwarder" + Default: "" + LogProcessors: + Type: String + Description: "Comma-separated list of URIs of log processors to load on startup. (Supports: http://, https://, or s3:// URIs)" + Default: "" FunctionMemorySize: Type: Number Description: Memory size (in MB) allocated to the Lambda function. @@ -70,15 +81,268 @@ Parameters: Default: 30 MinValue: 15 MaxValue: 900 - S3Bucket: + ForwarderImageTag: Type: String - Description: S3 bucket containing Rotel Lambda Forwarder. - Default: "" # updated on release - S3Key: + Description: Container image tag to deploy from public.ecr.aws/streamfold/rotel-lambda-forwarder + Default: "latest" + ForceRedeploy: Type: String - Description: S3 key for the Rotel Lambda Forwarder. - Default: "" # updated on release + Description: Increment to force an upgrade by pulling a new image and redeploying (e.g., increment a number or use a timestamp) + Default: "1" + Resources: + # Private ECR Repository for the forwarder image + ForwarderECRRepository: + Type: AWS::ECR::Repository + DeletionPolicy: Retain + Properties: + RepositoryName: !Sub "rotel-lambda-forwarder-${AWS::StackName}" + ImageScanningConfiguration: + ScanOnPush: true + Tags: + - Key: "PartOf" + Value: !Ref AWS::StackName + - Key: "Platform" + Value: "Rotel" + + # CodeBuild project to copy image from public ECR to private ECR + ImageCopyProject: + Type: AWS::CodeBuild::Project + Properties: + Name: !Sub "${AWS::StackName}-image-copy" + Description: Copies container image from public ECR to private ECR + ServiceRole: !GetAtt ImageCopyRole.Arn + Artifacts: + Type: NO_ARTIFACTS + Environment: + Type: LINUX_CONTAINER + ComputeType: BUILD_GENERAL1_SMALL + Image: aws/codebuild/standard:7.0 + PrivilegedMode: true + EnvironmentVariables: + - Name: SOURCE_IMAGE + Value: !Sub "public.ecr.aws/streamfold/rotel-lambda-forwarder:${ForwarderImageTag}" + - Name: TARGET_IMAGE + Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ForwarderECRRepository}:${ForwarderImageTag}" + - Name: AWS_DEFAULT_REGION + Value: !Ref AWS::Region + - Name: AWS_ACCOUNT_ID + Value: !Ref AWS::AccountId + Source: + Type: NO_SOURCE + BuildSpec: | + version: 0.2 + phases: + pre_build: + commands: + - echo "Logging in to Amazon ECR in region $AWS_DEFAULT_REGION..." + - aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com + build: + commands: + - echo "Pulling image from public ECR ($SOURCE_IMAGE)..." + - docker pull $SOURCE_IMAGE + - echo "Tagging image for private ECR..." + - docker tag $SOURCE_IMAGE $TARGET_IMAGE + - echo "Pushing image to private ECR ($TARGET_IMAGE)..." + - docker push $TARGET_IMAGE + post_build: + commands: + - echo "Image copy completed successfully" + Tags: + - Key: "PartOf" + Value: !Ref AWS::StackName + - Key: "Platform" + Value: "Rotel" + + # IAM Role for CodeBuild + ImageCopyRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: codebuild.amazonaws.com + ManagedPolicyArns: + - arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryPowerUser + Policies: + - PolicyName: CodeBuildLogsPolicy + PolicyDocument: + Statement: + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:CreateLogStream + - logs:PutLogEvents + Resource: + - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${AWS::StackName}-image-copy" + - !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:/aws/codebuild/${AWS::StackName}-image-copy:*" + + Tags: + - Key: "PartOf" + Value: !Ref AWS::StackName + - Key: "Platform" + Value: "Rotel" + + # Lambda function for Custom Resource + ImageCopyCustomResourceFunction: + Type: AWS::Lambda::Function + Properties: + FunctionName: !Sub "${AWS::StackName}-image-copy-cr" + Description: !Sub "Lambda function to trigger CodeBuild to pull ECR image for stack: ${AWS::StackName}" + Runtime: python3.14 + Handler: index.handler + Role: !GetAtt ImageCopyCustomResourceRole.Arn + Timeout: 900 + Code: + ZipFile: | + import json + import boto3 + import cfnresponse + import time + + codebuild = boto3.client('codebuild') + lambda_client = boto3.client('lambda') + + def handler(event, context): + print(f"Received event: {json.dumps(event)}") + + response_data = {} + physical_resource_id = event.get('PhysicalResourceId', 'ImageCopyBuild') + + try: + request_type = event['RequestType'] + project_name = event['ResourceProperties']['ProjectName'] + lambda_function_name = event['ResourceProperties'].get('LambdaFunctionName') + image_tag = event['ResourceProperties'].get('ImageTag', 'latest') + + if request_type in ['Create', 'Update']: + # Start the CodeBuild project + print(f"Starting CodeBuild project: {project_name}") + build_response = codebuild.start_build(projectName=project_name) + build_id = build_response['build']['id'] + print(f"Build started with ID: {build_id}") + + # Wait for build to complete + print("Waiting for build to complete...") + while True: + build_info = codebuild.batch_get_builds(ids=[build_id])['builds'][0] + build_status = build_info['buildStatus'] + print(f"Build status: {build_status}") + + if build_status == 'SUCCEEDED': + print("Build succeeded!") + response_data['BuildId'] = build_id + response_data['BuildStatus'] = build_status + + # If this is an Update and Lambda function exists, force Lambda to update + if request_type == 'Update' and lambda_function_name: + try: + print(f"Forcing Lambda function update: {lambda_function_name}") + + # Get Lambda function details to determine the image URI + function_details = lambda_client.get_function(FunctionName=lambda_function_name) + current_image_uri = function_details['Code']['ImageUri'] + print(f"Current ImageUri: {current_image_uri}") + + # Construct new image URI with the same repository but potentially new tag + # The image should already be copied to ECR by CodeBuild + image_uri_base = current_image_uri.rsplit(':', 1)[0] + new_image_uri = f"{image_uri_base}:{image_tag}" + print(f"New ImageUri: {new_image_uri}") + + # Update the Lambda function to force it to pull the new image + lambda_client.update_function_code( + FunctionName=lambda_function_name, + ImageUri=new_image_uri, + Publish=True + ) + print("Lambda function update triggered successfully") + response_data['LambdaUpdated'] = 'true' + response_data['NewImageUri'] = new_image_uri + except Exception as lambda_error: + print(f"Warning: Could not update Lambda function: {str(lambda_error)}") + # Don't fail the whole operation if Lambda update fails + response_data['LambdaUpdated'] = 'false' + response_data['LambdaUpdateError'] = str(lambda_error) + + cfnresponse.send(event, context, cfnresponse.SUCCESS, response_data, physical_resource_id) + return + elif build_status in ['FAILED', 'FAULT', 'TIMED_OUT', 'STOPPED']: + print(f"Build failed with status: {build_status}") + cfnresponse.send(event, context, cfnresponse.FAILED, + {'Error': f'Build failed with status: {build_status}'}, + physical_resource_id) + return + + time.sleep(10) + + elif request_type == 'Delete': + # Nothing to do on delete + print("Delete request - no action needed") + cfnresponse.send(event, context, cfnresponse.SUCCESS, response_data, physical_resource_id) + return + + except Exception as e: + print(f"Error: {str(e)}") + cfnresponse.send(event, context, cfnresponse.FAILED, + {'Error': str(e)}, physical_resource_id) + Tags: + - Key: "PartOf" + Value: !Ref AWS::StackName + - Key: "Platform" + Value: "Rotel" + + # IAM Role for Custom Resource Lambda + ImageCopyCustomResourceRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Statement: + - Action: sts:AssumeRole + Effect: Allow + Principal: + Service: lambda.amazonaws.com + ManagedPolicyArns: + - arn:aws:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole + Policies: + - PolicyName: CodeBuildStartPolicy + PolicyDocument: + Statement: + - Effect: Allow + Action: + - codebuild:StartBuild + - codebuild:BatchGetBuilds + Resource: !GetAtt ImageCopyProject.Arn + - PolicyName: LambdaUpdatePolicy + PolicyDocument: + Statement: + - Effect: Allow + Action: + - lambda:UpdateFunctionCode + - lambda:GetFunction + Resource: !Sub "arn:aws:lambda:${AWS::Region}:${AWS::AccountId}:function:${AWS::StackName}" + Tags: + - Key: "PartOf" + Value: !Ref AWS::StackName + - Key: "Platform" + Value: "Rotel" + + # Custom Resource to trigger image copy + ImageCopyCustomResource: + Type: Custom::ImageCopy + DependsOn: + - ForwarderECRRepository + - ImageCopyProject + Properties: + ServiceToken: !GetAtt ImageCopyCustomResourceFunction.Arn + ProjectName: !Ref ImageCopyProject + ImageTag: !Ref ForwarderImageTag + LambdaFunctionName: !Ref AWS::StackName + # This triggers re-execution when either parameter changes + Trigger: !Sub "${ForwarderImageTag}-${ForceRedeploy}" + ForwarderBucket: Type: AWS::S3::Bucket DeletionPolicy: Retain @@ -100,6 +364,7 @@ Resources: Value: !Ref AWS::StackName - Key: "Platform" Value: "Rotel" + ForwarderLogGroup: Type: "AWS::Logs::LogGroup" DeletionPolicy: Delete @@ -111,6 +376,7 @@ Resources: Value: !Ref AWS::StackName - Key: "Platform" Value: "Rotel" + ForwarderRole: Type: AWS::IAM::Role Properties: @@ -129,6 +395,7 @@ Resources: Value: "RotelLambdaForwarder" - Key: "Platform" Value: "Rotel" + ForwarderPolicy: Type: AWS::IAM::Policy Properties: @@ -139,6 +406,12 @@ Resources: - logs:PutLogEvents Effect: Allow Resource: !GetAtt ForwarderLogGroup.Arn + - Action: + - ecr:GetDownloadUrlForLayer + - ecr:BatchGetImage + - ecr:BatchCheckLayerAvailability + Effect: Allow + Resource: !GetAtt ForwarderECRRepository.Arn - Action: - lambda:AddPermission - lambda:RemovePermission @@ -165,15 +438,17 @@ Resources: PolicyName: !Sub "${AWS::StackName}-forwarder-lambda-policy" Roles: - !Ref "ForwarderRole" + ForwarderLambda: Type: AWS::Lambda::Function + DependsOn: + - ImageCopyCustomResource Properties: FunctionName: !Ref AWS::StackName - Runtime: provided.al2023 - Handler: bootstrap + Description: "Rotel Lambda Forwarder - Convert and forward CloudWatch logs as OTLP" + PackageType: Image Code: - S3Bucket: !Ref S3Bucket - S3Key: !Ref S3Key + ImageUri: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ForwarderECRRepository}:${ForwarderImageTag}" Role: !GetAtt - ForwarderRole - Arn @@ -199,6 +474,8 @@ Resources: ROTEL_OTEL_RESOURCE_ATTRIBUTES: !Ref "ResourceAttributes" ROTEL_EXPORTER_RETRY_MAX_ELAPSED_TIME: !Sub "${FunctionTimeout}s" FORWARDER_S3_BUCKET: !Ref ForwarderBucket + FORWARDER_OTLP_LOG_PROCESSORS: !Ref LogProcessors + ForwarderLambdaPermission: Type: AWS::Lambda::Permission DependsOn: @@ -209,6 +486,7 @@ Resources: Principal: !Sub "logs.${AWS::Region}.amazonaws.com" SourceAccount: !Ref "AWS::AccountId" SourceArn: !Sub "arn:aws:logs:${AWS::Region}:${AWS::AccountId}:log-group:*:*" + Outputs: ForwarderLambdaARN: Description: The ARN of the created Forwarder Lambda @@ -219,6 +497,9 @@ Outputs: ForwarderBucketName: Description: The name of the S3 bucket used for tag caching Value: !Ref ForwarderBucket + ForwarderECRRepositoryUri: + Description: The URI of the private ECR repository + Value: !Sub "${AWS::AccountId}.dkr.ecr.${AWS::Region}.amazonaws.com/${ForwarderECRRepository}" SetupInstructions: Description: Instructions for setting up CloudWatch Logs subscription filters Value: !Sub | @@ -228,3 +509,11 @@ Outputs: --filter-name forward-to-rotel \ --filter-pattern "" \ --destination-arn ${ForwarderLambda.Arn} + ImageUpdateInstructions: + Description: How to update the container image + Value: | + To pull a new version of the image from public ECR: + 1. Update the stack with a new ForwarderImageTag value (e.g., change from "latest" to "v1.2.3"), OR + 2. Increment the ForceRedeploy parameter (e.g., change from "1" to "2") to re-pull the same tag, OR + + The CodeBuild project will automatically pull the image from public ECR and push it to your private ECR. diff --git a/examples/processors/parse_vpc_logs.py b/examples/processors/parse_vpc_logs.py new file mode 100644 index 0000000..824f29b --- /dev/null +++ b/examples/processors/parse_vpc_logs.py @@ -0,0 +1,38 @@ +import itertools +import json + +from rotel_sdk.open_telemetry.common.v1 import AnyValue, KeyValue +from rotel_sdk.open_telemetry.logs.v1 import ResourceLogs + +def process_logs(resource_logs: ResourceLogs): + """ + Look for VPC flow logs and filter logs that have action == REJECT + """ + + if resource_logs.resource and resource_logs.resource.attributes: + # Check if this is an AWS VPC Flow Log + vpc_flow_log_found = False + for attr in resource_logs.resource.attributes: + if attr.key == "cloud.platform" and isinstance(attr.value.value, str) and attr.value.value == "aws_vpc_flow_log": + vpc_flow_log_found = True + break + + if not vpc_flow_log_found: + return + + # Filter out VPC flow logs with action == "REJECT" + for scope_log in resource_logs.scope_logs: + filtered_log_records = [] + for log_record in scope_log.log_records: + # Check if this log record has action == "REJECT" + should_keep = True + for attr in log_record.attributes: + if attr.key == "action" and isinstance(attr.value.value, str) and attr.value.value == "REJECT": + should_keep = False + break + + if should_keep: + filtered_log_records.append(log_record) + + # Replace the log_records with the filtered list + scope_log.log_records = filtered_log_records diff --git a/scripts/deploy-lambda.sh b/scripts/deploy-lambda.sh new file mode 100755 index 0000000..5843d16 --- /dev/null +++ b/scripts/deploy-lambda.sh @@ -0,0 +1,100 @@ +#!/bin/bash + +set -e + +# Check arguments +if [ $# -lt 2 ] || [ $# -gt 3 ]; then + echo "Usage: $0 [aws_region]" + echo "Example: $0 bootstrap.zip my-lambda-function us-east-1" + echo "" + echo "Environment variables:" + echo " IAM_ROLE - Required for creating new functions" + echo " AWS_DEFAULT_REGION - AWS region to use (default: us-east-1)" + exit 1 +fi + +BOOTSTRAP_ZIP="$1" +FUNCTION_NAME="$2" +AWS_REGION="${3:-${AWS_DEFAULT_REGION:-us-east-1}}" + +# Verify AWS credentials are configured +echo "Verifying AWS credentials..." +if ! aws sts get-caller-identity >/dev/null 2>&1; then + echo "Error: AWS credentials are not properly configured" + echo "Please configure your AWS credentials using one of the following methods:" + echo " - Set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment variables" + echo " - Run 'aws configure' to set up credentials" + echo " - Use an IAM role (if running on EC2/ECS/Lambda)" + exit 1 +fi + +# Verify bootstrap zip exists +if [ ! -f "$BOOTSTRAP_ZIP" ]; then + echo "Error: Bootstrap zip not found at: $BOOTSTRAP_ZIP" + exit 1 +fi + +echo "Checking if Lambda function '$FUNCTION_NAME' exists in region $AWS_REGION..." + +# Check if function exists +if aws lambda get-function --function-name "$FUNCTION_NAME" --region "$AWS_REGION" >/dev/null 2>&1; then + echo "Function exists. Updating function code..." + + aws lambda update-function-code \ + --function-name "$FUNCTION_NAME" \ + --zip-file "fileb://$BOOTSTRAP_ZIP" \ + --no-cli-pager \ + --publish \ + --region "$AWS_REGION" > /dev/null + + echo "Waiting for function to be updated..." + aws lambda wait function-updated \ + --function-name "$FUNCTION_NAME" \ + --region "$AWS_REGION" + + echo "Successfully updated Lambda function '$FUNCTION_NAME'" + + # Get the updated function info + aws lambda get-function \ + --function-name "$FUNCTION_NAME" \ + --region "$AWS_REGION" \ + --query 'Configuration.[FunctionName,Runtime,LastModified,Version,CodeSize]' \ + --output table +else + echo "Function does not exist. Creating new function..." + + # Check if IAM_ROLE is set + if [ -z "$IAM_ROLE" ]; then + echo "Error: IAM_ROLE environment variable is not set" + echo "Please set IAM_ROLE to the ARN of the Lambda execution role" + echo "Example: export IAM_ROLE=arn:aws:iam::123456789012:role/lambda-execution-role" + exit 1 + fi + + echo "Creating Lambda function '$FUNCTION_NAME' with role: $IAM_ROLE" + + aws lambda create-function \ + --function-name "$FUNCTION_NAME" \ + --runtime provided.al2023 \ + --role "$IAM_ROLE" \ + --handler bootstrap \ + --zip-file "fileb://$BOOTSTRAP_ZIP" \ + --architectures x86_64 \ + --timeout 30 \ + --memory-size 256 \ + --region "$AWS_REGION" + + echo "Waiting for function to be active..." + aws lambda wait function-active \ + --function-name "$FUNCTION_NAME" \ + --region "$AWS_REGION" + + echo "Successfully created Lambda function '$FUNCTION_NAME'" + + # Get the new function info + aws lambda get-function \ + --function-name "$FUNCTION_NAME" \ + --region "$AWS_REGION" \ + --query 'Configuration.[FunctionName,Runtime,LastModified,CodeSize]' \ + --output table +fi diff --git a/src/lib.rs b/src/lib.rs index 5f3258a..0b73cef 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,6 +3,16 @@ pub mod events; pub mod flowlogs; pub mod forward; pub mod init; +pub mod log_processors; pub mod parse; pub mod s3_cache; pub mod tags; + +static INIT_CRYPTO: std::sync::Once = std::sync::Once::new(); +pub fn init_crypto() { + INIT_CRYPTO.call_once(|| { + rustls::crypto::aws_lc_rs::default_provider() + .install_default() + .expect("unable to initialize crypto provider") + }); +} diff --git a/src/log_processors.rs b/src/log_processors.rs new file mode 100644 index 0000000..e9ffa2a --- /dev/null +++ b/src/log_processors.rs @@ -0,0 +1,194 @@ +use aws_sdk_s3::Client as S3Client; +use http_body_util::{BodyExt, Empty}; +use hyper::body::Bytes; +use hyper_util::client::legacy::Client as HyperClient; +use hyper_util::rt::TokioExecutor; +use std::path::{Path, PathBuf}; +use tokio::fs; +use tokio::io::AsyncWriteExt; +use tower::BoxError; +use tracing::{debug, info}; + +use crate::init_crypto; + +const PROCESSORS_DIR: &str = "/tmp/log_processors"; + +/// Downloads log processor files from URLs specified in the FORWARDER_OTLP_LOG_PROCESSORS +/// environment variable and returns a vector of their paths. +pub async fn setup_log_processors() -> Result>, BoxError> { + let urls_str = match std::env::var("FORWARDER_OTLP_LOG_PROCESSORS") { + Ok(val) if !val.is_empty() => val, + _ => { + debug!("FORWARDER_OTLP_LOG_PROCESSORS not set, skipping log processor setup"); + return Ok(None); + } + }; + + info!( + urls = %urls_str, + "Setting up log processors from FORWARDER_OTLP_LOG_PROCESSORS" + ); + + init_crypto(); + + // Parse URLs from comma-separated list + let urls: Vec<&str> = urls_str.split(',').map(|s| s.trim()).collect(); + if urls.is_empty() { + return Ok(None); + } + + // Setup the processors directory + setup_processors_directory().await?; + + // Download all processor files + let mut processor_paths = Vec::new(); + for (index, url) in urls.iter().enumerate() { + let processor_num = index + 1; + let filename = format!("processor_{:02}.py", processor_num); + let filepath = PathBuf::from(PROCESSORS_DIR).join(&filename); + + info!( + url = %url, + path = %filepath.display(), + "Downloading log processor" + ); + + download_processor(url, &filepath).await?; + processor_paths.push(filepath.to_string_lossy().to_string()); + } + + // Return vector of paths + info!( + paths = ?processor_paths, + "Log processors setup complete" + ); + Ok(Some(processor_paths)) +} + +/// Setup the processors directory, clearing it if it exists or creating it otherwise +async fn setup_processors_directory() -> Result<(), BoxError> { + let dir_path = Path::new(PROCESSORS_DIR); + + if dir_path.exists() { + debug!( + path = %dir_path.display(), + "Processors directory exists, clearing contents" + ); + // Remove the directory and all its contents + fs::remove_dir_all(dir_path).await?; + } + + // Create the directory + fs::create_dir_all(dir_path).await?; + debug!( + path = %dir_path.display(), + "Processors directory created" + ); + + Ok(()) +} + +/// Download a processor file from a URL (http://, https://, or s3://) +async fn download_processor(url: &str, dest_path: &PathBuf) -> Result<(), BoxError> { + if url.starts_with("s3://") { + download_from_s3(url, dest_path).await + } else if url.starts_with("http://") || url.starts_with("https://") { + download_from_http(url, dest_path).await + } else { + Err(format!("Unsupported URL scheme: {}", url).into()) + } +} + +/// Download a file from an HTTP/HTTPS URL using hyper +async fn download_from_http(url: &str, dest_path: &PathBuf) -> Result<(), BoxError> { + let https = hyper_rustls::HttpsConnectorBuilder::new() + .with_native_roots() + .expect("no native root CA certificates found") + .https_or_http() + .enable_http1() + .enable_http2() + .build(); + let client = HyperClient::builder(TokioExecutor::new()).build(https); + + let uri: hyper::Uri = url.parse()?; + let req = hyper::Request::builder() + .uri(uri) + .method("GET") + .body(Empty::::new())?; + + let response = client.request(req).await?; + + if !response.status().is_success() { + return Err(format!("HTTP request failed with status: {}", response.status()).into()); + } + + let body_bytes = response.into_body().collect().await?.to_bytes(); + + // Write to file + let mut file = fs::File::create(dest_path).await?; + file.write_all(&body_bytes).await?; + file.flush().await?; + + debug!( + url = %url, + path = %dest_path.display(), + size = body_bytes.len(), + "Downloaded file from HTTP" + ); + + Ok(()) +} + +/// Download a file from S3 +async fn download_from_s3(url: &str, dest_path: &PathBuf) -> Result<(), BoxError> { + // Parse s3://bucket/key + let s3_path = url.strip_prefix("s3://").ok_or("Invalid S3 URL")?; + let parts: Vec<&str> = s3_path.splitn(2, '/').collect(); + + if parts.len() != 2 { + return Err(format!("Invalid S3 URL format: {}", url).into()); + } + + let bucket = parts[0]; + let key = parts[1]; + + debug!( + bucket = %bucket, + key = %key, + "Downloading from S3" + ); + + // Create S3 client + let aws_config = aws_config::load_from_env().await; + let s3_client = S3Client::new(&aws_config); + + // Download the object + let response = s3_client + .get_object() + .bucket(bucket) + .key(key) + .send() + .await?; + + // Stream the body to file + let mut file = fs::File::create(dest_path).await?; + let mut byte_stream = response.body; + + let mut total_bytes = 0; + while let Some(chunk_result) = byte_stream.next().await { + let chunk = chunk_result?; + file.write_all(&chunk).await?; + total_bytes += chunk.len(); + } + + file.flush().await?; + + debug!( + url = %url, + path = %dest_path.display(), + size = total_bytes, + "Downloaded file from S3" + ); + + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 25c6bc2..8e638aa 100644 --- a/src/main.rs +++ b/src/main.rs @@ -107,6 +107,21 @@ async fn run_forwarder( env: &str, s3_bucket: Option, ) -> Result<(), BoxError> { + // Setup log processors from environment variable + match rotel_lambda_forwarder::log_processors::setup_log_processors().await { + Ok(Some(processor_paths)) => { + info!(paths = ?processor_paths, "Configuring agent with log processors"); + agent_args.otlp_with_logs_processor = processor_paths; + } + Ok(None) => { + debug!("No log processors configured"); + } + Err(e) => { + warn!(error = %e, "Failed to setup log processors, continuing without them"); + // Don't fail startup, just log the warning + } + } + let (logs_tx, logs_rx) = rotel::bounded_channel::bounded(LOGS_QUEUE_SIZE); let (flush_logs_tx, flush_logs_sub) = FlushBroadcast::new().into_parts();