Optimize CI performance: remove duplicate web builds, parallelize tasks, improve caching (#399)

.github/workflows/ci.yml

@@ -52,7 +52,7 @@ jobs:
 
   mac:
     name: Mac CI
-    needs: [changes]
+    needs: [changes, node]
     if: |
       always() && 
       !contains(needs.*.result, 'failure') && 

.github/workflows/ios.yml

@@ -32,16 +32,7 @@ jobs:
         xcodebuild -version
         swift --version
     
-    - name: Setup Node.js
-      uses: actions/setup-node@v4
-      with:
-        node-version: '24'
-    
-    - name: Setup pnpm
-      uses: pnpm/action-setup@v4
-      with:
-        version: 9
-        run_install: false
+    # Node.js/pnpm not needed for iOS builds
     
     - name: Cache Homebrew packages
       uses: actions/cache@v4
@@ -70,6 +61,9 @@ jobs:
     - name: Install all tools
       shell: bash
       run: |
+        # Skip Homebrew update for faster CI
+        export HOMEBREW_NO_AUTO_UPDATE=1
+        
         # Retry logic for brew commands to handle concurrent access
         MAX_ATTEMPTS=5
         WAIT_TIME=5
@@ -85,9 +79,9 @@ jobs:
             continue
           fi
           
-          # Update Homebrew and install all tools in one command
+          # Install tools without updating Homebrew
           # brew install automatically upgrades if already installed
-          if brew update && brew install swiftlint swiftformat xcbeautify; then
+          if brew install swiftlint swiftformat xcbeautify; then
             echo "Successfully installed/upgraded all tools"
             break
           else
@@ -107,37 +101,8 @@ jobs:
         echo "xcbeautify: $(xcbeautify --version || echo 'not found')"
         echo "PATH: $PATH"
     
-    - name: Cache pnpm store
-      uses: actions/cache@v4
-      continue-on-error: true
-      with:
-        path: ~/.local/share/pnpm/store
-        key: ${{ runner.os }}-pnpm-store-${{ hashFiles('web/pnpm-lock.yaml') }}
-        restore-keys: |
-          ${{ runner.os }}-pnpm-store-
-
-    - name: Install web dependencies
-      run: |
-        cd web
-        # Clean any stale lock files
-        rm -f .pnpm-store.lock .pnpm-debug.log || true
-        # Set pnpm to use fewer workers to avoid crashes on self-hosted runners
-        export NODE_OPTIONS="--max-old-space-size=4096"
-        pnpm config set store-dir ~/.local/share/pnpm/store
-        pnpm config set package-import-method copy
-        pnpm config set node-linker hoisted
-        # Install with retries
-        for i in 1 2 3; do
-          echo "Install attempt $i"
-          if pnpm install --frozen-lockfile; then
-            echo "pnpm install succeeded"
-            break
-          else
-            echo "pnpm install failed, cleaning and retrying..."
-            rm -rf node_modules .pnpm-store.lock || true
-            sleep 5
-          fi
-        done
+    # iOS doesn't need web dependencies - skip pnpm entirely
+    
     - name: Resolve Dependencies (once)
       run: |
         cd ios
@@ -151,6 +116,7 @@ jobs:
         # Ensure xcbeautify is in PATH
         export PATH="/usr/local/bin:/opt/homebrew/bin:$PATH"
         
+        # Use Release config for faster builds
         set -o pipefail
         xcodebuild build \
           -workspace ../VibeTunnel.xcworkspace \
@@ -158,14 +124,13 @@ jobs:
           -destination "generic/platform=iOS" \
           -configuration Release \
           -showBuildTimingSummary \
+          -quiet \
           CODE_SIGNING_ALLOWED=NO \
           CODE_SIGNING_REQUIRED=NO \
           ONLY_ACTIVE_ARCH=NO \
           -derivedDataPath build/DerivedData \
-          COMPILER_INDEX_STORE_ENABLE=NO \
-          2>&1 | tee build.log || {
-            echo "Build failed. Last 100 lines of output:"
-            tail -100 build.log
+          COMPILER_INDEX_STORE_ENABLE=NO || {
+            echo "::error::Build failed"
             exit 1
           }
     
@@ -311,13 +276,20 @@ jobs:
           exit 1
         fi
         
+        # Only enable coverage on main branch
+        if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then
+          ENABLE_COVERAGE="YES"
+        else
+          ENABLE_COVERAGE="NO"
+        fi
+        
         set -o pipefail
         xcodebuild test \
           -workspace ../VibeTunnel.xcworkspace \
           -scheme VibeTunnel-iOS \
           -destination "platform=iOS Simulator,id=$SIMULATOR_ID" \
           -resultBundlePath TestResults.xcresult \
-          -enableCodeCoverage YES \
+          -enableCodeCoverage $ENABLE_COVERAGE \
           CODE_SIGN_IDENTITY="" \
           CODE_SIGNING_REQUIRED=NO \
           CODE_SIGNING_ALLOWED=NO \
@@ -419,13 +391,14 @@ jobs:
         fi
     
     # ARTIFACT UPLOADS
+    # Skip build artifact upload for PR builds to save time
     - name: Upload build artifacts
       uses: actions/upload-artifact@v4
-      if: success()
+      if: success() && github.event_name == 'push' && github.ref == 'refs/heads/main'
       with:
         name: ios-build-artifacts
         path: ios/build/DerivedData/Build/Products/Release-iphoneos/
-        retention-days: 7
+        retention-days: 3
     
     - name: Upload coverage artifacts
       if: always()
@@ -484,7 +457,8 @@ jobs:
     name: Report iOS Coverage
     runs-on: blacksmith-8vcpu-ubuntu-2404-arm
     needs: [build-lint-test]
-    if: always() && github.event_name == 'pull_request'
+    # Only run coverage reporting on main branch where we actually collect coverage
+    if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     steps:
     - name: Clean workspace

.github/workflows/mac.yml

@@ -32,16 +32,7 @@ jobs:
         xcodebuild -version
         swift --version
     
-    - name: Setup Node.js
-      uses: actions/setup-node@v4
-      with:
-        node-version: '24'
-    
-    - name: Setup pnpm
-      uses: pnpm/action-setup@v4
-      with:
-        version: 9
-        dest: ~/pnpm-${{ github.run_id }}
+    # Node.js/pnpm not needed - web artifacts are downloaded
     
     - name: Cache Homebrew packages
       uses: actions/cache@v4
@@ -61,15 +52,29 @@ jobs:
       continue-on-error: true
       with:
         path: |
-          ~/Library/Developer/Xcode/DerivedData
           ~/.swiftpm
         key: ${{ runner.os }}-spm-${{ hashFiles('mac/Package.resolved') }}
         restore-keys: |
           ${{ runner.os }}-spm-
     
+    - name: Cache Xcode derived data
+      uses: actions/cache@v4
+      continue-on-error: true
+      with:
+        path: |
+          ~/Library/Developer/Xcode/DerivedData/**/Build/Products
+          ~/Library/Developer/Xcode/DerivedData/**/Build/Intermediates.noindex
+          ~/Library/Developer/Xcode/DerivedData/**/SourcePackages
+        key: ${{ runner.os }}-xcode-build-${{ hashFiles('mac/**/*.swift', 'mac/**/*.h', 'mac/**/*.m') }}
+        restore-keys: |
+          ${{ runner.os }}-xcode-build-
+    
     - name: Install all tools
       shell: bash
       run: |
+        # Skip Homebrew update for faster CI
+        export HOMEBREW_NO_AUTO_UPDATE=1
+        
         # Retry logic for brew commands to handle concurrent access
         MAX_ATTEMPTS=5
         WAIT_TIME=5
@@ -85,9 +90,9 @@ jobs:
             continue
           fi
           
-          # Update Homebrew and install all tools in one command
+          # Install tools without updating Homebrew
           # brew install automatically upgrades if already installed
-          if brew update && brew install swiftlint swiftformat xcbeautify; then
+          if brew install swiftlint swiftformat xcbeautify; then
             echo "Successfully installed/upgraded all tools"
             break
           else
@@ -107,48 +112,54 @@ jobs:
         echo "xcbeautify: $(xcbeautify --version || echo 'not found')"
         echo "jq: $(which jq || echo 'not found')"
     
-    - name: Cache pnpm store
-      uses: actions/cache@v4
-      continue-on-error: true
-      with:
-        path: ~/.local/share/pnpm/store
-        key: ${{ runner.os }}-pnpm-store-${{ hashFiles('web/pnpm-lock.yaml') }}
-        restore-keys: |
-          ${{ runner.os }}-pnpm-store-
-
-    - name: Install web dependencies
-      run: |
-        cd web
-        # Clean any stale lock files
-        rm -f .pnpm-store.lock .pnpm-debug.log || true
-        # Set pnpm to use fewer workers to avoid crashes on self-hosted runners
-        export NODE_OPTIONS="--max-old-space-size=4096"
-        pnpm config set store-dir ~/.local/share/pnpm/store
-        pnpm config set package-import-method hardlink
-        # Install with retries
-        for i in 1 2 3; do
-          echo "Install attempt $i"
-          if pnpm install --frozen-lockfile; then
-            echo "pnpm install succeeded"
-            # Force rebuild of native modules
-            echo "Rebuilding native modules..."
-            pnpm rebuild || true
-            break
-          else
-            echo "pnpm install failed, cleaning and retrying..."
-            rm -rf node_modules .pnpm-store.lock || true
-            sleep 5
-          fi
-        done
+    # Skip pnpm cache - testing if fresh installs are faster
+    # The cache was extremely large and might be slower than fresh install
     
-    - name: Build web artifacts
+    - name: Download web artifacts from Node.js CI
+      uses: actions/download-artifact@v4
+      with:
+        name: web-build
+        path: web-artifacts-temp/
+    
+    - name: Move web artifacts to correct location
       run: |
-        echo "Building web artifacts locally..."
-        cd web
-        # Skip custom Node.js build in CI to avoid timeout
-        export CI=true
-        pnpm run build
-        echo "Web artifacts built successfully"
+        # Debug: Show what was downloaded
+        echo "=== Contents of web-artifacts-temp ==="
+        find web-artifacts-temp -type f | head -20 || echo "No files found"
+        
+        # Ensure web directory structure exists
+        mkdir -p web/dist web/public/bundle
+        
+        # The artifacts are uploaded without the web/ prefix
+        # So they're at web-artifacts-temp/dist and web-artifacts-temp/public/bundle
+        if [ -d "web-artifacts-temp/dist" ]; then
+          # Copy from the root of artifacts
+          cp -r web-artifacts-temp/dist/* web/dist/ 2>/dev/null || true
+          echo "Copied dist files"
+        fi
+        if [ -d "web-artifacts-temp/public/bundle" ]; then
+          cp -r web-artifacts-temp/public/bundle/* web/public/bundle/ 2>/dev/null || true
+          echo "Copied bundle files"
+        fi
+        
+        # Debug: Show what we have
+        echo "=== Web directory structure ==="
+        ls -la web/ || true
+        echo "=== Dist contents ==="
+        ls -la web/dist/ | head -10 || true
+        echo "=== Bundle contents ==="
+        ls -la web/public/bundle/ | head -10 || true
+        
+        # Clean up temp directory
+        rm -rf web-artifacts-temp
+        
+        # Verify we have the required files
+        if [ ! -f "web/dist/server/server.js" ]; then
+          echo "ERROR: web/dist/server/server.js not found after artifact extraction!"
+          exit 1
+        fi
+        
+        echo "Web artifacts successfully downloaded and positioned"
     
     - name: Resolve Dependencies (once)
       run: |
@@ -161,13 +172,17 @@ jobs:
         xcodebuild -list -workspace VibeTunnel.xcworkspace | grep -A 20 "Schemes:" || true
     
     # BUILD PHASE
-    - name: Build Debug (Native Architecture)
-      timeout-minutes: 15
+    - name: Build Debug
+      timeout-minutes: 10
+      id: build
       run: |
+        # Always use Debug for now to match test expectations
+        BUILD_CONFIG="Debug"
+        
         set -o pipefail && xcodebuild build \
           -workspace VibeTunnel.xcworkspace \
           -scheme VibeTunnel-Mac \
-          -configuration Debug \
+          -configuration $BUILD_CONFIG \
           -destination "platform=macOS" \
           -showBuildTimingSummary \
           CODE_SIGN_IDENTITY="" \
@@ -177,9 +192,12 @@ jobs:
           ENABLE_HARDENED_RUNTIME=NO \
           PROVISIONING_PROFILE_SPECIFIER="" \
           DEVELOPMENT_TEAM="" \
-          COMPILER_INDEX_STORE_ENABLE=NO
+          COMPILER_INDEX_STORE_ENABLE=NO || {
+          echo "::error::Build failed"
+          exit 1
+        }
     
-    # LINT PHASE
+    # LINT PHASE (after build to avoid conflicts)
     - name: Run SwiftFormat (check mode)
       id: swiftformat
       continue-on-error: true
@@ -210,15 +228,26 @@ jobs:
           echo "web/dist does not exist"
         fi
         
-        # Use xcodebuild test for workspace testing with coverage enabled
+        # Use xcodebuild test for workspace testing
+        # Only enable coverage on main branch
+        if [[ "${{ github.event_name }}" == "push" && "${{ github.ref }}" == "refs/heads/main" ]]; then
+          ENABLE_COVERAGE="YES"
+        else
+          ENABLE_COVERAGE="NO"
+        fi
+        
+        # Always use Debug for tests
+        TEST_CONFIG="Debug"
+        
         set -o pipefail && \
         xcodebuild test \
           -workspace VibeTunnel.xcworkspace \
           -scheme VibeTunnel-Mac \
-          -configuration Debug \
+          -configuration $TEST_CONFIG \
           -destination "platform=macOS" \
-          -enableCodeCoverage YES \
+          -enableCodeCoverage $ENABLE_COVERAGE \
           -resultBundlePath TestResults.xcresult \
+          -quiet \
           CODE_SIGN_IDENTITY="" \
           CODE_SIGNING_REQUIRED=NO \
           CODE_SIGNING_ALLOWED=NO \
@@ -308,13 +337,16 @@ jobs:
         echo "Searching for build products..."
         find ~/Library/Developer/Xcode/DerivedData -name "VibeTunnel.app" -type d 2>/dev/null || echo "No build products found"
     
+    # Skip build artifact upload for PR builds to save time
     - name: Upload build artifacts
+      if: github.event_name == 'push' && github.ref == 'refs/heads/main'
       uses: actions/upload-artifact@v4
       with:
         name: mac-build-artifacts
         path: |
           ~/Library/Developer/Xcode/DerivedData/*/Build/Products/Debug/VibeTunnel.app
           ~/Library/Developer/Xcode/DerivedData/*/Build/Products/Release/VibeTunnel.app
+        retention-days: 3
     
     - name: Upload coverage artifacts
       if: always()
@@ -372,7 +404,8 @@ jobs:
     name: Report Coverage Results
     runs-on: blacksmith-8vcpu-ubuntu-2404-arm
     needs: [build-lint-test]
-    if: always() && github.event_name == 'pull_request'
+    # Only run coverage reporting on main branch where we actually collect coverage
+    if: always() && github.event_name == 'push' && github.ref == 'refs/heads/main'
 
     steps:
     - name: Clean workspace

.github/workflows/node.yml

@@ -33,19 +33,8 @@ jobs:
         version: 9
         run_install: false
 
-    - name: Get pnpm store directory
-      shell: bash
-      run: |
-        echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
-
-    - name: Setup pnpm cache
-      uses: useblacksmith/cache@v5
-      continue-on-error: true
-      with:
-        path: ${{ env.STORE_PATH }}
-        key: ${{ runner.os }}-pnpm-store-${{ hashFiles('web/pnpm-lock.yaml') }}
-        restore-keys: |
-          ${{ runner.os }}-pnpm-store-
+    # Skip pnpm cache - testing if fresh installs are faster
+    # The cache was extremely large and might be slower than fresh install
 
     - name: Install system dependencies
       run: |
@@ -54,7 +43,10 @@ jobs:
 
     - name: Install dependencies
       working-directory: web
-      run: pnpm install --frozen-lockfile
+      run: |
+        pnpm config set network-concurrency 4
+        pnpm config set child-concurrency 2
+        pnpm install --frozen-lockfile --prefer-offline
 
     - name: Check formatting with Biome
       id: biome-format
@@ -137,19 +129,8 @@ jobs:
         version: 9
         run_install: false
 
-    - name: Get pnpm store directory
-      shell: bash
-      run: |
-        echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
-
-    - name: Setup pnpm cache
-      uses: useblacksmith/cache@v5
-      continue-on-error: true
-      with:
-        path: ${{ env.STORE_PATH }}
-        key: ${{ runner.os }}-pnpm-store-${{ hashFiles('web/pnpm-lock.yaml') }}
-        restore-keys: |
-          ${{ runner.os }}-pnpm-store-
+    # Skip pnpm cache - testing if fresh installs are faster
+    # The cache was extremely large and might be slower than fresh install
 
     - name: Install system dependencies
       run: |
@@ -170,7 +151,10 @@ jobs:
 
     - name: Install dependencies
       working-directory: web
-      run: pnpm install --frozen-lockfile
+      run: |
+        pnpm config set network-concurrency 4
+        pnpm config set child-concurrency 2
+        pnpm install --frozen-lockfile --prefer-offline
 
     - name: Build node-pty
       working-directory: web
@@ -179,7 +163,10 @@ jobs:
 
     - name: Build frontend and backend
       working-directory: web
-      run: pnpm run build:ci
+      run: |
+        # Use all available cores for esbuild
+        export ESBUILD_MAX_WORKERS=$(nproc)
+        pnpm run build:ci
 
     - name: Run client tests with coverage
       id: test-client-coverage
@@ -279,14 +266,23 @@ jobs:
           web/coverage/client/lcov.info
           web/coverage/server/lcov.info
 
+    - name: List build artifacts before upload
+      working-directory: web
+      run: |
+        echo "=== Contents of dist directory ==="
+        find dist -type f | head -20 || echo "No files in dist"
+        echo "=== Contents of public/bundle directory ==="
+        find public/bundle -type f | head -20 || echo "No files in public/bundle"
+        
     - name: Upload build artifacts
       uses: actions/upload-artifact@v4
       with:
-        name: web-build-${{ github.sha }}
+        name: web-build
         path: |
           web/dist/
           web/public/bundle/
         retention-days: 1
+        if-no-files-found: error
 
   type-check:
     name: TypeScript Type Checking
@@ -309,19 +305,8 @@ jobs:
         version: 9
         run_install: false
 
-    - name: Get pnpm store directory
-      shell: bash
-      run: |
-        echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
-
-    - name: Setup pnpm cache
-      uses: useblacksmith/cache@v5
-      continue-on-error: true
-      with:
-        path: ${{ env.STORE_PATH }}
-        key: ${{ runner.os }}-pnpm-store-${{ hashFiles('web/pnpm-lock.yaml') }}
-        restore-keys: |
-          ${{ runner.os }}-pnpm-store-
+    # Skip pnpm cache - testing if fresh installs are faster
+    # The cache was extremely large and might be slower than fresh install
 
     - name: Install system dependencies
       run: |
@@ -330,7 +315,10 @@ jobs:
 
     - name: Install dependencies
       working-directory: web
-      run: pnpm install --frozen-lockfile
+      run: |
+        pnpm config set network-concurrency 4
+        pnpm config set child-concurrency 2
+        pnpm install --frozen-lockfile --prefer-offline
 
     - name: Build node-pty for TypeScript
       working-directory: web
@@ -366,9 +354,10 @@ jobs:
       # || true to not fail the build on vulnerabilities, but still report them
 
   report-coverage:
-    name: Report Coverage Results
+    name: Report Coverage Results  
     runs-on: blacksmith-8vcpu-ubuntu-2404-arm
     needs: [build-and-test]
+    # Keep Node.js coverage reporting for PRs since it's fast
     if: always() && github.event_name == 'pull_request'
 
     steps:

web/scripts/build-ci.js

@@ -25,7 +25,26 @@ execSync('esbuild src/client/sw.ts --bundle --outfile=public/sw.js --format=iife
 
 // Build server TypeScript
 console.log('Building server...');
-execSync('tsc', { stdio: 'inherit' });
+// Force a clean build in CI to avoid incremental build issues
+execSync('npx tsc --build --force', { stdio: 'inherit' });
+
+// Verify dist directory exists
+if (fs.existsSync(path.join(__dirname, '../dist'))) {
+  const files = fs.readdirSync(path.join(__dirname, '../dist'));
+  console.log(`Server build created ${files.length} files in dist/`);
+  console.log('Files in dist:', files.join(', '));
+  
+  // Check for the essential server.js file
+  if (!fs.existsSync(path.join(__dirname, '../dist/server/server.js'))) {
+    console.error('ERROR: dist/server/server.js not found after tsc build!');
+    console.log('Contents of dist directory:');
+    execSync('find dist -type f | head -20', { stdio: 'inherit', cwd: path.join(__dirname, '..') });
+    process.exit(1);
+  }
+} else {
+  console.error('ERROR: dist directory does not exist after tsc build!');
+  process.exit(1);
+}
 
 // Skip native executable build in CI
 console.log('Skipping native executable build in CI environment...');