cache/.github/workflows/workflow.yml

558 lines
23 KiB
YAML

name: Tests
on:
pull_request:
branches:
- main
- releases/**
push:
branches:
- main
- releases/**
permissions:
contents: read
jobs:
# Build and unit test
build:
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macOS-latest]
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@v5
- name: Setup Node.js 24.x
uses: actions/setup-node@v4
with:
node-version: 24.x
cache: npm
- run: npm ci
- name: Prettier Format Check
run: npm run format-check
- name: ESLint Check
run: npm run lint
- name: Build & Test
run: npm run test
# End to end save and restore
test-save:
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macOS-latest]
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@v5
- name: Generate files in working directory
shell: bash
run: __tests__/create-cache-files.sh ${{ runner.os }} test-cache
- name: Generate files outside working directory
shell: bash
run: __tests__/create-cache-files.sh ${{ runner.os }} ~/test-cache
- name: Save cache
uses: ./
with:
key: test-${{ runner.os }}-${{ github.run_id }}
path: |
test-cache
~/test-cache
test-restore:
needs: test-save
strategy:
matrix:
os: [ubuntu-latest, windows-latest, macOS-latest]
fail-fast: false
runs-on: ${{ matrix.os }}
steps:
- name: Checkout
uses: actions/checkout@v5
- name: Restore cache
uses: ./
with:
key: test-${{ runner.os }}-${{ github.run_id }}
path: |
test-cache
~/test-cache
- name: Verify cache files in working directory
shell: bash
run: __tests__/verify-cache-files.sh ${{ runner.os }} test-cache
- name: Verify cache files outside working directory
shell: bash
run: __tests__/verify-cache-files.sh ${{ runner.os }} ~/test-cache
# End to end with proxy
test-proxy-save:
runs-on: ubuntu-latest
container:
image: ubuntu:latest
options: --privileged
services:
squid-proxy:
image: ubuntu/squid:latest
ports:
- 3128:3128
env:
http_proxy: http://squid-proxy:3128
https_proxy: http://squid-proxy:3128
steps:
- name: Checkout
uses: actions/checkout@v5
- name: Install dependencies
run: |
apt-get update
apt-get install -y iptables dnsutils curl jq ipset
- name: Fetch GitHub meta and configure firewall
run: |
# Fetch GitHub meta API to get all IP ranges
echo "Fetching GitHub meta API..."
curl -sS https://api.github.com/meta > /tmp/github-meta.json
# Get squid-proxy IP address
PROXY_IP=$(getent hosts squid-proxy | awk '{ print $1 }')
echo "Proxy IP: $PROXY_IP"
# Allow established connections
iptables -A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
# Allow loopback
iptables -A OUTPUT -o lo -j ACCEPT
# Allow connections to the proxy
iptables -A OUTPUT -d $PROXY_IP -p tcp --dport 3128 -j ACCEPT
# Allow DNS
iptables -A OUTPUT -p udp --dport 53 -j ACCEPT
iptables -A OUTPUT -p tcp --dport 53 -j ACCEPT
# Create ipset for GitHub IPs (more efficient than individual rules)
ipset create github-ips hash:net
# Add all GitHub IP ranges from meta API (hooks, web, api, git, actions, etc.)
# EXCLUDING blob storage which must go through proxy
for category in hooks web api git pages importer actions actions_macos codespaces copilot; do
echo "Adding IPs for category: $category"
jq -r ".${category}[]? // empty" /tmp/github-meta.json 2>/dev/null | while read cidr; do
# Skip IPv6 for now (iptables vs ip6tables) - use case for POSIX compatibility
case "$cidr" in
*:*) ;; # IPv6, skip
*) ipset add github-ips "$cidr" 2>/dev/null || true ;;
esac
done
done
# Allow all GitHub IPs
iptables -A OUTPUT -m set --match-set github-ips dst -p tcp --dport 443 -j ACCEPT
iptables -A OUTPUT -m set --match-set github-ips dst -p tcp --dport 80 -j ACCEPT
# CRITICAL: Block direct access to blob storage and results-receiver
# These MUST go through the proxy for cache operations
echo "Blocking direct access to cache-critical endpoints..."
# Block results-receiver.actions.githubusercontent.com
for ip in $(getent ahosts "results-receiver.actions.githubusercontent.com" 2>/dev/null | awk '{print $1}' | sort -u); do
echo "Blocking direct access to results-receiver: $ip"
iptables -I OUTPUT 1 -d "$ip" -p tcp --dport 443 -j REJECT
done
# Block productionresultssa*.blob.core.windows.net (cache blob storage)
# We block ALL blob.core.windows.net traffic since we can't easily enumerate all storage accounts
# The proxy will handle these requests
echo "Note: *.blob.core.windows.net traffic will be blocked and must go through proxy"
# Block all other outbound HTTP/HTTPS traffic
iptables -A OUTPUT -p tcp --dport 80 -j REJECT
iptables -A OUTPUT -p tcp --dport 443 -j REJECT
echo "iptables rules applied:"
iptables -L OUTPUT -n -v
echo ""
echo "ipset github-ips contains $(ipset list github-ips | grep -c '^[0-9]') entries"
- name: Verify proxy enforcement
run: |
echo "=== Testing proxy enforcement ==="
# Test 1: Direct connection to github.com should work (it's in allowed IPs)
echo "Test 1: Direct connection to github.com (should SUCCEED - GitHub IP allowed)"
if curl --connect-timeout 5 --max-time 10 --noproxy '*' -sS https://api.github.com/zen 2>/dev/null; then
echo "✓ Direct GitHub API access works (expected)"
else
echo "✗ Direct GitHub API access failed (unexpected but not critical)"
fi
# Test 2: Direct connection to blob storage should FAIL
echo ""
echo "Test 2: Direct connection to blob storage (should FAIL - must use proxy)"
if curl --connect-timeout 5 --max-time 10 --noproxy '*' -sS https://productionresultssa0.blob.core.windows.net 2>/dev/null; then
echo "✗ ERROR: Direct blob storage connection succeeded but should have been blocked!"
exit 1
else
echo "✓ Direct blob storage correctly blocked"
fi
# Test 3: Connection through proxy should work
echo ""
echo "Test 3: Connection through proxy to blob storage (should SUCCEED)"
if curl --connect-timeout 5 --max-time 10 -sS https://productionresultssa0.blob.core.windows.net 2>&1 | head -5; then
echo "✓ Proxy connection works (expected - even if 4xx/5xx response, connection succeeded)"
else
echo "Note: Proxy connection may have failed, but that's OK if it's not a network block"
fi
- name: Generate files
run: __tests__/create-cache-files.sh proxy test-cache
- name: Save cache
uses: ./
with:
key: test-proxy-${{ github.run_id }}
path: test-cache
- name: Verify cache traffic went through proxy
run: |
echo "=== Verifying cache traffic went through proxy ==="
# Get the squid container ID
SQUID_CONTAINER=$(docker ps --filter "ancestor=ubuntu/squid:latest" --format "{{.ID}}" | head -1)
if [ -z "$SQUID_CONTAINER" ]; then
SQUID_CONTAINER=$(docker ps --format "{{.ID}}\t{{.Image}}" | grep squid | cut -f1)
fi
# Initialize summary
echo "## 🔒 Proxy Traffic Verification - Cache Save" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
if [ -n "$SQUID_CONTAINER" ]; then
echo "Found Squid container: $SQUID_CONTAINER"
# Get the full access log
ACCESS_LOG=$(docker exec "$SQUID_CONTAINER" cat /var/log/squid/access.log 2>/dev/null || echo "")
# Extract traffic details
RESULTS_RECEIVER_LINES=$(echo "$ACCESS_LOG" | grep -i "results-receiver" || true)
BLOB_LINES=$(echo "$ACCESS_LOG" | grep -i "blob.core.windows.net" || true)
RESULTS_RECEIVER_COUNT=$(echo "$ACCESS_LOG" | grep -ci "results-receiver" || echo "0")
BLOB_COUNT=$(echo "$ACCESS_LOG" | grep -ci "blob.core.windows.net" || echo "0")
# Build summary table
echo "### 📊 Traffic Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Endpoint | Requests | Status |" >> $GITHUB_STEP_SUMMARY
echo "|----------|----------|--------|" >> $GITHUB_STEP_SUMMARY
if [ "$RESULTS_RECEIVER_COUNT" -gt 0 ]; then
echo "| results-receiver.actions.githubusercontent.com | $RESULTS_RECEIVER_COUNT | ✅ Proxied |" >> $GITHUB_STEP_SUMMARY
else
echo "| results-receiver.actions.githubusercontent.com | 0 | ⚠️ Not detected |" >> $GITHUB_STEP_SUMMARY
fi
if [ "$BLOB_COUNT" -gt 0 ]; then
echo "| *.blob.core.windows.net | $BLOB_COUNT | ✅ Proxied |" >> $GITHUB_STEP_SUMMARY
else
echo "| *.blob.core.windows.net | 0 | ⚠️ Not detected |" >> $GITHUB_STEP_SUMMARY
fi
echo "" >> $GITHUB_STEP_SUMMARY
# Verification result
echo "### 🎯 Verification Result" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
if [ "$RESULTS_RECEIVER_COUNT" -gt 0 ] && [ "$BLOB_COUNT" -gt 0 ]; then
echo "✅ **SUCCESS**: All cache save traffic verified going through proxy!" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "- ✅ CreateCacheEntry API call routed through proxy" >> $GITHUB_STEP_SUMMARY
echo "- ✅ FinalizeCacheEntryUpload API call routed through proxy" >> $GITHUB_STEP_SUMMARY
echo "- ✅ Blob storage upload routed through proxy" >> $GITHUB_STEP_SUMMARY
VERIFY_STATUS="success"
else
echo "⚠️ **WARNING**: Some expected cache traffic not found in proxy logs" >> $GITHUB_STEP_SUMMARY
VERIFY_STATUS="warning"
fi
# Detailed traffic logs
echo "" >> $GITHUB_STEP_SUMMARY
echo "<details>" >> $GITHUB_STEP_SUMMARY
echo "<summary>📋 Detailed Proxy Traffic Logs</summary>" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "#### Results Receiver Traffic (Cache API)" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
if [ -n "$RESULTS_RECEIVER_LINES" ]; then
echo "$RESULTS_RECEIVER_LINES" >> $GITHUB_STEP_SUMMARY
else
echo "(no results-receiver traffic found)" >> $GITHUB_STEP_SUMMARY
fi
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "#### Blob Storage Traffic (Cache Upload)" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
if [ -n "$BLOB_LINES" ]; then
echo "$BLOB_LINES" >> $GITHUB_STEP_SUMMARY
else
echo "(no blob storage traffic found)" >> $GITHUB_STEP_SUMMARY
fi
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "#### Full Squid Access Log" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
if [ -n "$ACCESS_LOG" ]; then
echo "$ACCESS_LOG" >> $GITHUB_STEP_SUMMARY
else
echo "(access log empty or not accessible)" >> $GITHUB_STEP_SUMMARY
fi
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
echo "</details>" >> $GITHUB_STEP_SUMMARY
# Also output to logs for debugging
echo ""
echo "=== Traffic Summary ==="
echo "Results-receiver requests: $RESULTS_RECEIVER_COUNT"
echo "Blob storage requests: $BLOB_COUNT"
echo "Verification status: $VERIFY_STATUS"
else
echo "⚠️ **WARNING**: Could not access Squid proxy container logs" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "This may occur when service containers are isolated from the job container." >> $GITHUB_STEP_SUMMARY
echo "Could not access squid container logs"
fi
test-proxy-restore:
needs: test-proxy-save
runs-on: ubuntu-latest
container:
image: ubuntu:latest
options: --privileged
services:
squid-proxy:
image: ubuntu/squid:latest
ports:
- 3128:3128
env:
http_proxy: http://squid-proxy:3128
https_proxy: http://squid-proxy:3128
steps:
- name: Checkout
uses: actions/checkout@v5
- name: Install dependencies
run: |
apt-get update
apt-get install -y iptables dnsutils curl jq ipset
- name: Fetch GitHub meta and configure firewall
run: |
# Fetch GitHub meta API to get all IP ranges
echo "Fetching GitHub meta API..."
curl -sS https://api.github.com/meta > /tmp/github-meta.json
# Get squid-proxy IP address
PROXY_IP=$(getent hosts squid-proxy | awk '{ print $1 }')
echo "Proxy IP: $PROXY_IP"
# Allow established connections
iptables -A OUTPUT -m state --state ESTABLISHED,RELATED -j ACCEPT
# Allow loopback
iptables -A OUTPUT -o lo -j ACCEPT
# Allow connections to the proxy
iptables -A OUTPUT -d $PROXY_IP -p tcp --dport 3128 -j ACCEPT
# Allow DNS
iptables -A OUTPUT -p udp --dport 53 -j ACCEPT
iptables -A OUTPUT -p tcp --dport 53 -j ACCEPT
# Create ipset for GitHub IPs (more efficient than individual rules)
ipset create github-ips hash:net
# Add all GitHub IP ranges from meta API (hooks, web, api, git, actions, etc.)
# EXCLUDING blob storage which must go through proxy
for category in hooks web api git pages importer actions actions_macos codespaces copilot; do
echo "Adding IPs for category: $category"
jq -r ".${category}[]? // empty" /tmp/github-meta.json 2>/dev/null | while read cidr; do
# Skip IPv6 for now (iptables vs ip6tables) - use case for POSIX compatibility
case "$cidr" in
*:*) ;; # IPv6, skip
*) ipset add github-ips "$cidr" 2>/dev/null || true ;;
esac
done
done
# Allow all GitHub IPs
iptables -A OUTPUT -m set --match-set github-ips dst -p tcp --dport 443 -j ACCEPT
iptables -A OUTPUT -m set --match-set github-ips dst -p tcp --dport 80 -j ACCEPT
# CRITICAL: Block direct access to blob storage and results-receiver
# These MUST go through the proxy for cache operations
echo "Blocking direct access to cache-critical endpoints..."
# Block results-receiver.actions.githubusercontent.com
for ip in $(getent ahosts "results-receiver.actions.githubusercontent.com" 2>/dev/null | awk '{print $1}' | sort -u); do
echo "Blocking direct access to results-receiver: $ip"
iptables -I OUTPUT 1 -d "$ip" -p tcp --dport 443 -j REJECT
done
# Block productionresultssa*.blob.core.windows.net (cache blob storage)
# We block ALL blob.core.windows.net traffic since we can't easily enumerate all storage accounts
# The proxy will handle these requests
echo "Note: *.blob.core.windows.net traffic will be blocked and must go through proxy"
# Block all other outbound HTTP/HTTPS traffic
iptables -A OUTPUT -p tcp --dport 80 -j REJECT
iptables -A OUTPUT -p tcp --dport 443 -j REJECT
echo "iptables rules applied:"
iptables -L OUTPUT -n -v
echo ""
echo "ipset github-ips contains $(ipset list github-ips | grep -c '^[0-9]') entries"
- name: Verify proxy enforcement
run: |
echo "=== Testing proxy enforcement ==="
# Test 1: Direct connection to github.com should work (it's in allowed IPs)
echo "Test 1: Direct connection to github.com (should SUCCEED - GitHub IP allowed)"
if curl --connect-timeout 5 --max-time 10 --noproxy '*' -sS https://api.github.com/zen 2>/dev/null; then
echo "✓ Direct GitHub API access works (expected)"
else
echo "✗ Direct GitHub API access failed (unexpected but not critical)"
fi
# Test 2: Direct connection to blob storage should FAIL
echo ""
echo "Test 2: Direct connection to blob storage (should FAIL - must use proxy)"
if curl --connect-timeout 5 --max-time 10 --noproxy '*' -sS https://productionresultssa0.blob.core.windows.net 2>/dev/null; then
echo "✗ ERROR: Direct blob storage connection succeeded but should have been blocked!"
exit 1
else
echo "✓ Direct blob storage correctly blocked"
fi
# Test 3: Connection through proxy should work
echo ""
echo "Test 3: Connection through proxy to blob storage (should SUCCEED)"
if curl --connect-timeout 5 --max-time 10 -sS https://productionresultssa0.blob.core.windows.net 2>&1 | head -5; then
echo "✓ Proxy connection works (expected - even if 4xx/5xx response, connection succeeded)"
else
echo "Note: Proxy connection may have failed, but that's OK if it's not a network block"
fi
- name: Restore cache
uses: ./
with:
key: test-proxy-${{ github.run_id }}
path: test-cache
- name: Verify cache traffic went through proxy
run: |
echo "=== Verifying cache restore traffic went through proxy ==="
# Get the squid container ID
SQUID_CONTAINER=$(docker ps --filter "ancestor=ubuntu/squid:latest" --format "{{.ID}}" | head -1)
if [ -z "$SQUID_CONTAINER" ]; then
SQUID_CONTAINER=$(docker ps --format "{{.ID}}\t{{.Image}}" | grep squid | cut -f1)
fi
# Initialize summary
echo "## 🔒 Proxy Traffic Verification - Cache Restore" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
if [ -n "$SQUID_CONTAINER" ]; then
echo "Found Squid container: $SQUID_CONTAINER"
# Get the full access log
ACCESS_LOG=$(docker exec "$SQUID_CONTAINER" cat /var/log/squid/access.log 2>/dev/null || echo "")
# Extract traffic details
RESULTS_RECEIVER_LINES=$(echo "$ACCESS_LOG" | grep -i "results-receiver" || true)
BLOB_LINES=$(echo "$ACCESS_LOG" | grep -i "blob.core.windows.net" || true)
RESULTS_RECEIVER_COUNT=$(echo "$ACCESS_LOG" | grep -ci "results-receiver" || echo "0")
BLOB_COUNT=$(echo "$ACCESS_LOG" | grep -ci "blob.core.windows.net" || echo "0")
# Build summary table
echo "### 📊 Traffic Summary" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "| Endpoint | Requests | Status |" >> $GITHUB_STEP_SUMMARY
echo "|----------|----------|--------|" >> $GITHUB_STEP_SUMMARY
if [ "$RESULTS_RECEIVER_COUNT" -gt 0 ]; then
echo "| results-receiver.actions.githubusercontent.com | $RESULTS_RECEIVER_COUNT | ✅ Proxied |" >> $GITHUB_STEP_SUMMARY
else
echo "| results-receiver.actions.githubusercontent.com | 0 | ⚠️ Not detected |" >> $GITHUB_STEP_SUMMARY
fi
if [ "$BLOB_COUNT" -gt 0 ]; then
echo "| *.blob.core.windows.net | $BLOB_COUNT | ✅ Proxied |" >> $GITHUB_STEP_SUMMARY
else
echo "| *.blob.core.windows.net | 0 | ⚠️ Not detected |" >> $GITHUB_STEP_SUMMARY
fi
echo "" >> $GITHUB_STEP_SUMMARY
# Verification result
echo "### 🎯 Verification Result" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
if [ "$RESULTS_RECEIVER_COUNT" -gt 0 ] && [ "$BLOB_COUNT" -gt 0 ]; then
echo "✅ **SUCCESS**: All cache restore traffic verified going through proxy!" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "- ✅ GetCacheEntryDownloadURL API call routed through proxy" >> $GITHUB_STEP_SUMMARY
echo "- ✅ Blob storage download routed through proxy" >> $GITHUB_STEP_SUMMARY
VERIFY_STATUS="success"
else
echo "⚠️ **WARNING**: Some expected cache traffic not found in proxy logs" >> $GITHUB_STEP_SUMMARY
VERIFY_STATUS="warning"
fi
# Detailed traffic logs
echo "" >> $GITHUB_STEP_SUMMARY
echo "<details>" >> $GITHUB_STEP_SUMMARY
echo "<summary>📋 Detailed Proxy Traffic Logs</summary>" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "#### Results Receiver Traffic (Cache API)" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
if [ -n "$RESULTS_RECEIVER_LINES" ]; then
echo "$RESULTS_RECEIVER_LINES" >> $GITHUB_STEP_SUMMARY
else
echo "(no results-receiver traffic found)" >> $GITHUB_STEP_SUMMARY
fi
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "#### Blob Storage Traffic (Cache Download)" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
if [ -n "$BLOB_LINES" ]; then
echo "$BLOB_LINES" >> $GITHUB_STEP_SUMMARY
else
echo "(no blob storage traffic found)" >> $GITHUB_STEP_SUMMARY
fi
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "#### Full Squid Access Log" >> $GITHUB_STEP_SUMMARY
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
if [ -n "$ACCESS_LOG" ]; then
echo "$ACCESS_LOG" >> $GITHUB_STEP_SUMMARY
else
echo "(access log empty or not accessible)" >> $GITHUB_STEP_SUMMARY
fi
echo "\`\`\`" >> $GITHUB_STEP_SUMMARY
echo "</details>" >> $GITHUB_STEP_SUMMARY
# Also output to logs for debugging
echo ""
echo "=== Traffic Summary ==="
echo "Results-receiver requests: $RESULTS_RECEIVER_COUNT"
echo "Blob storage requests: $BLOB_COUNT"
echo "Verification status: $VERIFY_STATUS"
else
echo "⚠️ **WARNING**: Could not access Squid proxy container logs" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo "This may occur when service containers are isolated from the job container." >> $GITHUB_STEP_SUMMARY
echo "Could not access squid container logs"
fi
- name: Verify cache
run: __tests__/verify-cache-files.sh proxy test-cache