From 453d888731fd30b35ef09998b1d33950df3439a7 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Fri, 18 Jul 2025 08:01:23 +0200 Subject: [PATCH] Optimize CI performance: remove duplicate web builds, parallelize tasks, improve caching (#399) --- .github/workflows/ci.yml | 2 +- .github/workflows/ios.yml | 76 ++++++------------ .github/workflows/mac.yml | 157 ++++++++++++++++++++++--------------- .github/workflows/node.yml | 79 ++++++++----------- web/scripts/build-ci.js | 21 ++++- 5 files changed, 175 insertions(+), 160 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d0602efd..cd59000f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: mac: name: Mac CI - needs: [changes] + needs: [changes, node] if: | always() && !contains(needs.*.result, 'failure') && diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index d40faf51..bb6cee89 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -32,16 +32,7 @@ jobs: xcodebuild -version swift --version - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '24' - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - with: - version: 9 - run_install: false + # Node.js/pnpm not needed for iOS builds - name: Cache Homebrew packages uses: actions/cache@v4 @@ -70,6 +61,9 @@ jobs: - name: Install all tools shell: bash run: | + # Skip Homebrew update for faster CI + export HOMEBREW_NO_AUTO_UPDATE=1 + # Retry logic for brew commands to handle concurrent access MAX_ATTEMPTS=5 WAIT_TIME=5 @@ -85,9 +79,9 @@ jobs: continue fi - # Update Homebrew and install all tools in one command + # Install tools without updating Homebrew # brew install automatically upgrades if already installed - if brew update && brew install swiftlint swiftformat xcbeautify; then + if brew install swiftlint swiftformat xcbeautify; then echo "Successfully installed/upgraded all tools" break else @@ -107,37 +101,8 @@ jobs: echo "xcbeautify: $(xcbeautify --version || echo 'not found')" echo "PATH: $PATH" - - name: Cache pnpm store - uses: actions/cache@v4 - continue-on-error: true - with: - path: ~/.local/share/pnpm/store - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('web/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install web dependencies - run: | - cd web - # Clean any stale lock files - rm -f .pnpm-store.lock .pnpm-debug.log || true - # Set pnpm to use fewer workers to avoid crashes on self-hosted runners - export NODE_OPTIONS="--max-old-space-size=4096" - pnpm config set store-dir ~/.local/share/pnpm/store - pnpm config set package-import-method copy - pnpm config set node-linker hoisted - # Install with retries - for i in 1 2 3; do - echo "Install attempt $i" - if pnpm install --frozen-lockfile; then - echo "pnpm install succeeded" - break - else - echo "pnpm install failed, cleaning and retrying..." - rm -rf node_modules .pnpm-store.lock || true - sleep 5 - fi - done + # iOS doesn't need web dependencies - skip pnpm entirely + - name: Resolve Dependencies (once) run: | cd ios @@ -151,6 +116,7 @@ jobs: # Ensure xcbeautify is in PATH export PATH="/usr/local/bin:/opt/homebrew/bin:$PATH" + # Use Release config for faster builds set -o pipefail xcodebuild build \ -workspace ../VibeTunnel.xcworkspace \ @@ -158,14 +124,13 @@ jobs: -destination "generic/platform=iOS" \ -configuration Release \ -showBuildTimingSummary \ + -quiet \ CODE_SIGNING_ALLOWED=NO \ CODE_SIGNING_REQUIRED=NO \ ONLY_ACTIVE_ARCH=NO \ -derivedDataPath build/DerivedData \ - COMPILER_INDEX_STORE_ENABLE=NO \ - 2>&1 | tee build.log || { - echo "Build failed. Last 100 lines of output:" - tail -100 build.log + COMPILER_INDEX_STORE_ENABLE=NO || { + echo "::error::Build failed" exit 1 } @@ -311,13 +276,20 @@ jobs: exit 1 fi + # Only enable coverage on main branch + if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then + ENABLE_COVERAGE="YES" + else + ENABLE_COVERAGE="NO" + fi + set -o pipefail xcodebuild test \ -workspace ../VibeTunnel.xcworkspace \ -scheme VibeTunnel-iOS \ -destination "platform=iOS Simulator,id=$SIMULATOR_ID" \ -resultBundlePath TestResults.xcresult \ - -enableCodeCoverage YES \ + -enableCodeCoverage $ENABLE_COVERAGE \ CODE_SIGN_IDENTITY="" \ CODE_SIGNING_REQUIRED=NO \ CODE_SIGNING_ALLOWED=NO \ @@ -419,13 +391,14 @@ jobs: fi # ARTIFACT UPLOADS + # Skip build artifact upload for PR builds to save time - name: Upload build artifacts uses: actions/upload-artifact@v4 - if: success() + if: success() && github.event_name == 'push' && github.ref == 'refs/heads/main' with: name: ios-build-artifacts path: ios/build/DerivedData/Build/Products/Release-iphoneos/ - retention-days: 7 + retention-days: 3 - name: Upload coverage artifacts if: always() @@ -484,7 +457,8 @@ jobs: name: Report iOS Coverage runs-on: blacksmith-8vcpu-ubuntu-2404-arm needs: [build-lint-test] - if: always() && github.event_name == 'pull_request' + # Only run coverage reporting on main branch where we actually collect coverage + if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main' steps: - name: Clean workspace diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index ca4bc6b0..88e72057 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -32,16 +32,7 @@ jobs: xcodebuild -version swift --version - - name: Setup Node.js - uses: actions/setup-node@v4 - with: - node-version: '24' - - - name: Setup pnpm - uses: pnpm/action-setup@v4 - with: - version: 9 - dest: ~/pnpm-${{ github.run_id }} + # Node.js/pnpm not needed - web artifacts are downloaded - name: Cache Homebrew packages uses: actions/cache@v4 @@ -61,15 +52,29 @@ jobs: continue-on-error: true with: path: | - ~/Library/Developer/Xcode/DerivedData ~/.swiftpm key: ${{ runner.os }}-spm-${{ hashFiles('mac/Package.resolved') }} restore-keys: | ${{ runner.os }}-spm- + - name: Cache Xcode derived data + uses: actions/cache@v4 + continue-on-error: true + with: + path: | + ~/Library/Developer/Xcode/DerivedData/**/Build/Products + ~/Library/Developer/Xcode/DerivedData/**/Build/Intermediates.noindex + ~/Library/Developer/Xcode/DerivedData/**/SourcePackages + key: ${{ runner.os }}-xcode-build-${{ hashFiles('mac/**/*.swift', 'mac/**/*.h', 'mac/**/*.m') }} + restore-keys: | + ${{ runner.os }}-xcode-build- + - name: Install all tools shell: bash run: | + # Skip Homebrew update for faster CI + export HOMEBREW_NO_AUTO_UPDATE=1 + # Retry logic for brew commands to handle concurrent access MAX_ATTEMPTS=5 WAIT_TIME=5 @@ -85,9 +90,9 @@ jobs: continue fi - # Update Homebrew and install all tools in one command + # Install tools without updating Homebrew # brew install automatically upgrades if already installed - if brew update && brew install swiftlint swiftformat xcbeautify; then + if brew install swiftlint swiftformat xcbeautify; then echo "Successfully installed/upgraded all tools" break else @@ -107,48 +112,54 @@ jobs: echo "xcbeautify: $(xcbeautify --version || echo 'not found')" echo "jq: $(which jq || echo 'not found')" - - name: Cache pnpm store - uses: actions/cache@v4 - continue-on-error: true - with: - path: ~/.local/share/pnpm/store - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('web/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- - - - name: Install web dependencies - run: | - cd web - # Clean any stale lock files - rm -f .pnpm-store.lock .pnpm-debug.log || true - # Set pnpm to use fewer workers to avoid crashes on self-hosted runners - export NODE_OPTIONS="--max-old-space-size=4096" - pnpm config set store-dir ~/.local/share/pnpm/store - pnpm config set package-import-method hardlink - # Install with retries - for i in 1 2 3; do - echo "Install attempt $i" - if pnpm install --frozen-lockfile; then - echo "pnpm install succeeded" - # Force rebuild of native modules - echo "Rebuilding native modules..." - pnpm rebuild || true - break - else - echo "pnpm install failed, cleaning and retrying..." - rm -rf node_modules .pnpm-store.lock || true - sleep 5 - fi - done + # Skip pnpm cache - testing if fresh installs are faster + # The cache was extremely large and might be slower than fresh install - - name: Build web artifacts + - name: Download web artifacts from Node.js CI + uses: actions/download-artifact@v4 + with: + name: web-build + path: web-artifacts-temp/ + + - name: Move web artifacts to correct location run: | - echo "Building web artifacts locally..." - cd web - # Skip custom Node.js build in CI to avoid timeout - export CI=true - pnpm run build - echo "Web artifacts built successfully" + # Debug: Show what was downloaded + echo "=== Contents of web-artifacts-temp ===" + find web-artifacts-temp -type f | head -20 || echo "No files found" + + # Ensure web directory structure exists + mkdir -p web/dist web/public/bundle + + # The artifacts are uploaded without the web/ prefix + # So they're at web-artifacts-temp/dist and web-artifacts-temp/public/bundle + if [ -d "web-artifacts-temp/dist" ]; then + # Copy from the root of artifacts + cp -r web-artifacts-temp/dist/* web/dist/ 2>/dev/null || true + echo "Copied dist files" + fi + if [ -d "web-artifacts-temp/public/bundle" ]; then + cp -r web-artifacts-temp/public/bundle/* web/public/bundle/ 2>/dev/null || true + echo "Copied bundle files" + fi + + # Debug: Show what we have + echo "=== Web directory structure ===" + ls -la web/ || true + echo "=== Dist contents ===" + ls -la web/dist/ | head -10 || true + echo "=== Bundle contents ===" + ls -la web/public/bundle/ | head -10 || true + + # Clean up temp directory + rm -rf web-artifacts-temp + + # Verify we have the required files + if [ ! -f "web/dist/server/server.js" ]; then + echo "ERROR: web/dist/server/server.js not found after artifact extraction!" + exit 1 + fi + + echo "Web artifacts successfully downloaded and positioned" - name: Resolve Dependencies (once) run: | @@ -161,13 +172,17 @@ jobs: xcodebuild -list -workspace VibeTunnel.xcworkspace | grep -A 20 "Schemes:" || true # BUILD PHASE - - name: Build Debug (Native Architecture) - timeout-minutes: 15 + - name: Build Debug + timeout-minutes: 10 + id: build run: | + # Always use Debug for now to match test expectations + BUILD_CONFIG="Debug" + set -o pipefail && xcodebuild build \ -workspace VibeTunnel.xcworkspace \ -scheme VibeTunnel-Mac \ - -configuration Debug \ + -configuration $BUILD_CONFIG \ -destination "platform=macOS" \ -showBuildTimingSummary \ CODE_SIGN_IDENTITY="" \ @@ -177,9 +192,12 @@ jobs: ENABLE_HARDENED_RUNTIME=NO \ PROVISIONING_PROFILE_SPECIFIER="" \ DEVELOPMENT_TEAM="" \ - COMPILER_INDEX_STORE_ENABLE=NO + COMPILER_INDEX_STORE_ENABLE=NO || { + echo "::error::Build failed" + exit 1 + } - # LINT PHASE + # LINT PHASE (after build to avoid conflicts) - name: Run SwiftFormat (check mode) id: swiftformat continue-on-error: true @@ -210,15 +228,26 @@ jobs: echo "web/dist does not exist" fi - # Use xcodebuild test for workspace testing with coverage enabled + # Use xcodebuild test for workspace testing + # Only enable coverage on main branch + if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then + ENABLE_COVERAGE="YES" + else + ENABLE_COVERAGE="NO" + fi + + # Always use Debug for tests + TEST_CONFIG="Debug" + set -o pipefail && \ xcodebuild test \ -workspace VibeTunnel.xcworkspace \ -scheme VibeTunnel-Mac \ - -configuration Debug \ + -configuration $TEST_CONFIG \ -destination "platform=macOS" \ - -enableCodeCoverage YES \ + -enableCodeCoverage $ENABLE_COVERAGE \ -resultBundlePath TestResults.xcresult \ + -quiet \ CODE_SIGN_IDENTITY="" \ CODE_SIGNING_REQUIRED=NO \ CODE_SIGNING_ALLOWED=NO \ @@ -308,13 +337,16 @@ jobs: echo "Searching for build products..." find ~/Library/Developer/Xcode/DerivedData -name "VibeTunnel.app" -type d 2>/dev/null || echo "No build products found" + # Skip build artifact upload for PR builds to save time - name: Upload build artifacts + if: github.event_name == 'push' && github.ref == 'refs/heads/main' uses: actions/upload-artifact@v4 with: name: mac-build-artifacts path: | ~/Library/Developer/Xcode/DerivedData/*/Build/Products/Debug/VibeTunnel.app ~/Library/Developer/Xcode/DerivedData/*/Build/Products/Release/VibeTunnel.app + retention-days: 3 - name: Upload coverage artifacts if: always() @@ -372,7 +404,8 @@ jobs: name: Report Coverage Results runs-on: blacksmith-8vcpu-ubuntu-2404-arm needs: [build-lint-test] - if: always() && github.event_name == 'pull_request' + # Only run coverage reporting on main branch where we actually collect coverage + if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main' steps: - name: Clean workspace diff --git a/.github/workflows/node.yml b/.github/workflows/node.yml index 0388ed36..b7945b92 100644 --- a/.github/workflows/node.yml +++ b/.github/workflows/node.yml @@ -33,19 +33,8 @@ jobs: version: 9 run_install: false - - name: Get pnpm store directory - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache - uses: useblacksmith/cache@v5 - continue-on-error: true - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('web/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- + # Skip pnpm cache - testing if fresh installs are faster + # The cache was extremely large and might be slower than fresh install - name: Install system dependencies run: | @@ -54,7 +43,10 @@ jobs: - name: Install dependencies working-directory: web - run: pnpm install --frozen-lockfile + run: | + pnpm config set network-concurrency 4 + pnpm config set child-concurrency 2 + pnpm install --frozen-lockfile --prefer-offline - name: Check formatting with Biome id: biome-format @@ -137,19 +129,8 @@ jobs: version: 9 run_install: false - - name: Get pnpm store directory - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache - uses: useblacksmith/cache@v5 - continue-on-error: true - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('web/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- + # Skip pnpm cache - testing if fresh installs are faster + # The cache was extremely large and might be slower than fresh install - name: Install system dependencies run: | @@ -170,7 +151,10 @@ jobs: - name: Install dependencies working-directory: web - run: pnpm install --frozen-lockfile + run: | + pnpm config set network-concurrency 4 + pnpm config set child-concurrency 2 + pnpm install --frozen-lockfile --prefer-offline - name: Build node-pty working-directory: web @@ -179,7 +163,10 @@ jobs: - name: Build frontend and backend working-directory: web - run: pnpm run build:ci + run: | + # Use all available cores for esbuild + export ESBUILD_MAX_WORKERS=$(nproc) + pnpm run build:ci - name: Run client tests with coverage id: test-client-coverage @@ -279,14 +266,23 @@ jobs: web/coverage/client/lcov.info web/coverage/server/lcov.info + - name: List build artifacts before upload + working-directory: web + run: | + echo "=== Contents of dist directory ===" + find dist -type f | head -20 || echo "No files in dist" + echo "=== Contents of public/bundle directory ===" + find public/bundle -type f | head -20 || echo "No files in public/bundle" + - name: Upload build artifacts uses: actions/upload-artifact@v4 with: - name: web-build-${{ github.sha }} + name: web-build path: | web/dist/ web/public/bundle/ retention-days: 1 + if-no-files-found: error type-check: name: TypeScript Type Checking @@ -309,19 +305,8 @@ jobs: version: 9 run_install: false - - name: Get pnpm store directory - shell: bash - run: | - echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV - - - name: Setup pnpm cache - uses: useblacksmith/cache@v5 - continue-on-error: true - with: - path: ${{ env.STORE_PATH }} - key: ${{ runner.os }}-pnpm-store-${{ hashFiles('web/pnpm-lock.yaml') }} - restore-keys: | - ${{ runner.os }}-pnpm-store- + # Skip pnpm cache - testing if fresh installs are faster + # The cache was extremely large and might be slower than fresh install - name: Install system dependencies run: | @@ -330,7 +315,10 @@ jobs: - name: Install dependencies working-directory: web - run: pnpm install --frozen-lockfile + run: | + pnpm config set network-concurrency 4 + pnpm config set child-concurrency 2 + pnpm install --frozen-lockfile --prefer-offline - name: Build node-pty for TypeScript working-directory: web @@ -366,9 +354,10 @@ jobs: # || true to not fail the build on vulnerabilities, but still report them report-coverage: - name: Report Coverage Results + name: Report Coverage Results runs-on: blacksmith-8vcpu-ubuntu-2404-arm needs: [build-and-test] + # Keep Node.js coverage reporting for PRs since it's fast if: always() && github.event_name == 'pull_request' steps: diff --git a/web/scripts/build-ci.js b/web/scripts/build-ci.js index 18725362..67a9eccd 100644 --- a/web/scripts/build-ci.js +++ b/web/scripts/build-ci.js @@ -25,7 +25,26 @@ execSync('esbuild src/client/sw.ts --bundle --outfile=public/sw.js --format=iife // Build server TypeScript console.log('Building server...'); -execSync('tsc', { stdio: 'inherit' }); +// Force a clean build in CI to avoid incremental build issues +execSync('npx tsc --build --force', { stdio: 'inherit' }); + +// Verify dist directory exists +if (fs.existsSync(path.join(__dirname, '../dist'))) { + const files = fs.readdirSync(path.join(__dirname, '../dist')); + console.log(`Server build created ${files.length} files in dist/`); + console.log('Files in dist:', files.join(', ')); + + // Check for the essential server.js file + if (!fs.existsSync(path.join(__dirname, '../dist/server/server.js'))) { + console.error('ERROR: dist/server/server.js not found after tsc build!'); + console.log('Contents of dist directory:'); + execSync('find dist -type f | head -20', { stdio: 'inherit', cwd: path.join(__dirname, '..') }); + process.exit(1); + } +} else { + console.error('ERROR: dist directory does not exist after tsc build!'); + process.exit(1); +} // Skip native executable build in CI console.log('Skipping native executable build in CI environment...');