From 0790b63dc39be47387338d27510268f53ffb4861 Mon Sep 17 00:00:00 2001 From: continuist Date: Fri, 19 Sep 2025 21:45:56 -0400 Subject: [PATCH] Implement suggested changes to make cargo-chef work for local crates --- .forgejo/workflows/ci.yml | 66 +++++++++++++++--------------------- backend/Dockerfile.test-rust | 32 +++++++++-------- 2 files changed, 44 insertions(+), 54 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index f517b83..92800d3 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -46,32 +46,35 @@ jobs: - name: Create network run: podman --remote network create integ-${{ env.RUN_ID }} - - name: Check if dependencies changed - id: check-deps + - name: Generate cache key from lockfile + id: cache-key run: | - # Change to backend directory and get hash of Cargo.toml and Cargo.lock cd "$GITHUB_WORKSPACE/backend" - DEPS_HASH=$(sha256sum Cargo.toml Cargo.lock | sha256sum | cut -d' ' -f1) - echo "deps_hash=$DEPS_HASH" >> $GITHUB_OUTPUT - - # Check if image exists with this hash tag - if podman --remote manifest inspect "$PREBUILT_BACKEND_TEST_IMAGE:$DEPS_HASH" >/dev/null 2>&1; then - echo "rebuild_needed=false" >> $GITHUB_OUTPUT - else - echo "rebuild_needed=true" >> $GITHUB_OUTPUT - fi + LOCK_HASH=$(sha256sum Cargo.lock | cut -d' ' -f1) + SHORT_HASH=$(echo "$LOCK_HASH" | cut -c1-12) + echo "cache_key=$SHORT_HASH" >> $GITHUB_OUTPUT + echo "Using cache key: $SHORT_HASH" - - name: Build optimized Rust test image (if needed) - if: steps.check-deps.outputs.rebuild_needed == 'true' + - name: Build dependencies cache image run: | - # Build without caching to avoid policy issues - podman --remote build --no-cache \ + # Build deps stage with layer caching + podman --remote build \ + --target deps \ -f backend/Dockerfile.test-rust \ - -t "$PREBUILT_BACKEND_TEST_IMAGE:${{ steps.check-deps.outputs.deps_hash }}" \ - -t "$PREBUILT_BACKEND_TEST_IMAGE:latest" + -t "$PREBUILT_BACKEND_TEST_IMAGE:deps-${{ steps.cache-key.outputs.cache_key }}" \ + --layers - podman --remote push "$PREBUILT_BACKEND_TEST_IMAGE:${{ steps.check-deps.outputs.deps_hash }}" - podman --remote push "$PREBUILT_BACKEND_TEST_IMAGE:latest" + podman --remote push "$PREBUILT_BACKEND_TEST_IMAGE:deps-${{ steps.cache-key.outputs.cache_key }}" + + - name: Build full test image with cached dependencies + run: | + # Build final image using cached deps + podman --remote build \ + --target runner \ + -f backend/Dockerfile.test-rust \ + -t "$PREBUILT_BACKEND_TEST_IMAGE:test-${{ github.sha }}" \ + --cache-from "$PREBUILT_BACKEND_TEST_IMAGE:deps-${{ steps.cache-key.outputs.cache_key }}" \ + --layers - name: Start PostgreSQL run: | @@ -98,28 +101,13 @@ jobs: -v /home/ci-service/.cache:/c \ alpine:3.20 sh -lc 'mkdir -p /c/cargo' - - name: Run backend tests with cached dependencies + - name: Run backend tests run: | - set -euo pipefail - set -o pipefail - tar --exclude .git --exclude target -C "$GITHUB_WORKSPACE" -cf - . | \ - podman --remote run --rm -i \ + # Run tests in the pre-built test image + podman --remote run --rm \ --network integ-${{ env.RUN_ID }} \ - -e CARGO_HOME=/cargo \ -e DATABASE_URL=postgres://postgres:password@test-postgres-${{ env.RUN_ID }}:5432/sharenet_test \ - -v /home/ci-service/.cache/cargo:/cargo \ - "$PREBUILT_BACKEND_TEST_IMAGE:${{ steps.check-deps.outputs.deps_hash }}" \ - sh -euxc ' - mkdir -p /workspace - tar -x -C /workspace - cd /workspace/backend - # Use pre-built dependencies from the image - export CARGO_TARGET_DIR=/app/target - export CARGO_REGISTRY_DIR=/usr/local/cargo/registry - # Build local crates using pre-built external dependencies - cargo build --release --locked - cargo test --lib -- --test-threads=1 - ' + "$PREBUILT_BACKEND_TEST_IMAGE:test-${{ github.sha }}" - name: Cleanup if: always() diff --git a/backend/Dockerfile.test-rust b/backend/Dockerfile.test-rust index 17fff6d..9c8dfcf 100644 --- a/backend/Dockerfile.test-rust +++ b/backend/Dockerfile.test-rust @@ -1,30 +1,32 @@ -# planner: produce recipe.json -FROM docker.io/rust:1.89-slim AS planner +# Base stage with consistent working directory +FROM docker.io/rust:1.89-slim AS base WORKDIR /app -RUN apt-get update && apt-get install -y --no-install-recommends pkg-config libssl-dev ca-certificates postgresql-client curl && rm -rf /var/lib/apt/lists/* + +# Planner: produce recipe.json +FROM base AS planner +RUN apt-get update && apt-get install -y --no-install-recommends pkg-config libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/* RUN cargo install --locked cargo-chef COPY . . RUN cargo chef prepare --recipe-path recipe.json -# deps: compile only external crates -FROM docker.io/rust:1.89-slim AS deps -WORKDIR /app -RUN apt-get update && apt-get install -y --no-install-recommends pkg-config libssl-dev ca-certificates postgresql-client curl && rm -rf /var/lib/apt/lists/* +# Deps: compile only external crates +FROM base AS deps +RUN apt-get update && apt-get install -y --no-install-recommends pkg-config libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/* RUN cargo install --locked cargo-chef ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse COPY --from=planner /app/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json -# builder: compile local workspace crates using cached deps -FROM docker.io/rust:1.89-slim AS builder -WORKDIR /app +# Test-build: compile workspace with pre-built dependencies +FROM base AS test-build COPY --from=deps /usr/local/cargo /usr/local/cargo COPY --from=deps /usr/local/rustup /usr/local/rustup COPY --from=deps /app/target /app/target COPY . . -RUN cargo build --release --locked +RUN cargo test --workspace --locked --no-run -# runtime: minimal test environment - reuse builder stage which already has dependencies -FROM docker.io/rust:1.89-slim AS runtime -WORKDIR /workspace/backend -CMD ["sleep", "infinity"] \ No newline at end of file +# Runner: minimal runtime for test execution +FROM base AS runner +COPY --from=test-build /app/target /app/target +WORKDIR /app +CMD ["cargo", "test", "--workspace", "--locked", "--", "--test-threads=1"] \ No newline at end of file