ci: Switch to Github Actions (#3152)

* WIP: Trying to make a new branch

* Create fuzzing.yml

* Update ci.yml

* Try using reviewdog for golangci-lint

* Only run lint on ubuntu

* Whoops, wrong matrix variable

* Let's try just ubuntu for the moment

* Remove integration tests

* Let's see what the tree looks like (where's the binary)

* Let's plant a tree

* Let's look at another tree

* Burn the tree

* Let's build in the right dir

* Turn on publishing artifacts

* Add gobin to path

* Try running golangci-lint earlier

* Try running golangci-lint on its own, with checkout@v1

* Try moving golangci-lint back into ci.yml as a separate job

* Turn off azure-pipelines

* Remove the redundant name, see how it looks

* Trim down the naming some more

* Turn on windows and mac

* Try to fix windows build, cleanup

* Try to fix strange failure on windows

* Print our the coerce reason

* Apparently $? is 'True' on Windows, not 1 or 0

* Try setting CGO_ENABLED as an env in yml

* Try enabling/fixing the fuzzer

* Print out github event to check, fix step name

* Fuzzer needs the code

* Add GOBIN to PATH for fuzzer

* Comment out fork condition, left in-case we want it again

* Remove obsolete comment

* Comment out the coverage/test conversions for now

* Set continue-on-error: true for fuzzer, it runs out of mem

* Add some clarification to the retained commented sections
This commit is contained in:
Francis Lavoie 2020-03-20 10:38:44 -04:00 committed by GitHub
parent 61b427fa47
commit 76ac28a624
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 203 additions and 263 deletions

119
.github/workflows/ci.yml vendored Normal file
View file

@ -0,0 +1,119 @@
# Used as inspiration: https://github.com/mvdan/github-actions-golang
name: Cross-Platform
on:
push:
branches:
- v2
pull_request:
branches:
- v2
jobs:
test:
strategy:
# Default is true, cancels jobs for other platforms in the matrix if one fails
fail-fast: false
matrix:
os: [ ubuntu-latest, macos-latest, windows-latest ]
go-version: [ 1.14.x ]
runs-on: ${{ matrix.os }}
steps:
- name: Install Go
uses: actions/setup-go@v1
with:
go-version: ${{ matrix.go-version }}
- name: Checkout code
uses: actions/checkout@v2
# These tools would be useful if we later decide to reinvestigate
# publishing test/coverage reports to some tool for easier consumption
# - name: Install test and coverage analysis tools
# run: |
# go get github.com/axw/gocov/gocov
# go get github.com/AlekSi/gocov-xml
# go get -u github.com/jstemmer/go-junit-report
# echo "::add-path::$(go env GOPATH)/bin"
- name: Print Go version and environment
run: |
printf "Using go at: $(which go)\n"
printf "Go version: $(go version)\n"
printf "\n\nGo environment:\n\n"
go env
printf "\n\nSystem environment:\n\n"
env
- name: Get dependencies
run: |
go get -v -t -d ./...
# mkdir test-results
- name: Build Caddy
working-directory: ./cmd/caddy
env:
CGO_ENABLED: 0
run: |
go build -trimpath -a -ldflags="-w -s" -v
- name: Publish Build Artifact (Windows)
if: matrix.os == 'windows-latest'
uses: actions/upload-artifact@v1
with:
name: caddy_v2_${{ matrix.os }}
path: ./cmd/caddy/caddy.exe
- name: Publish Build Artifact (Linux/Mac)
if: matrix.os != 'windows-latest'
uses: actions/upload-artifact@v1
with:
name: caddy_v2_${{ matrix.os }}
path: ./cmd/caddy/caddy
# Commented bits below were useful to allow the job to continue
# even if the tests fail, so we can publish the report separately
# For info about set-output, see https://stackoverflow.com/questions/57850553/github-actions-check-steps-status
- name: Run tests
# id: step_test
# continue-on-error: true
run: |
# (go test -v -coverprofile=cover-profile.out -race ./... 2>&1) > test-results/test-result.out
go test -v -coverprofile="cover-profile.out" -race ./...
# echo "::set-output name=status::$?"
# Relevant step if we reinvestigate publishing test/coverage reports
# - name: Prepare coverage reports
# run: |
# mkdir coverage
# gocov convert cover-profile.out > coverage/coverage.json
# # Because Windows doesn't work with input redirection like *nix, but output redirection works.
# (cat ./coverage/coverage.json | gocov-xml) > coverage/coverage.xml
# To return the correct result even though we set 'continue-on-error: true'
# - name: Coerce correct build result (Windows)
# if: matrix.os == 'windows-latest' && steps.step_test.outputs.status != 'True'
# run: |
# echo "step_test ${{ steps.step_test.outputs.status }}\n"
# exit 1
# - name: Coerce correct build result (Linux/Mac)
# if: matrix.os != 'windows-latest' && steps.step_test.outputs.status != 0
# run: |
# echo "step_test ${{ steps.step_test.outputs.status }}\n"
# exit 1
# From https://github.com/reviewdog/action-golangci-lint
golangci-lint:
name: runner / golangci-lint
runs-on: ubuntu-latest
steps:
- name: Checkout code into the Go module directory
uses: actions/checkout@v2
- name: Run golangci-lint
uses: reviewdog/action-golangci-lint@v1
# uses: docker://reviewdog/action-golangci-lint:v1 # pre-build docker image
with:
github_token: ${{ secrets.github_token }}

84
.github/workflows/fuzzing.yml vendored Normal file
View file

@ -0,0 +1,84 @@
name: Fuzzing
on:
# Regression testing
push:
branches:
- v2
pull_request:
branches:
- v2
# Daily midnight fuzzing
schedule:
- cron: '0 0 * * *'
jobs:
fuzzing:
name: Fuzzing
strategy:
matrix:
os: [ ubuntu-latest ]
go-version: [ 1.14.x ]
runs-on: ${{ matrix.os }}
steps:
- name: Install Go
uses: actions/setup-go@v1
with:
go-version: ${{ matrix.go-version }}
- name: Checkout code
uses: actions/checkout@v2
- name: Download go-fuzz tools and the Fuzzit CLI, move Fuzzit CLI to GOBIN
# If we decide we need to prevent this from running on forks, we can use this line:
# if: github.repository == 'caddyserver/caddy'
run: |
# Install Clang-7.0 because other versions seem to be missing the file libclang_rt.fuzzer-x86_64.a
sudo add-apt-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main"
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
sudo apt update && sudo apt install -y clang-7 lldb-7 lld-7
go get -v github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
wget -q -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.77/fuzzit_Linux_x86_64
chmod a+x fuzzit
mv fuzzit $(go env GOPATH)/bin
echo "::add-path::$(go env GOPATH)/bin"
- name: Generate fuzzers & submit them to Fuzzit
continue-on-error: true
env:
FUZZIT_API_KEY: ${{ secrets.FUZZIT_API_KEY }}
run: |
declare -A fuzzers_funcs=(\
["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="FuzzParseAddress" \
["./caddyconfig/caddyfile/parse_fuzz.go"]="FuzzParseCaddyfile" \
["./listeners_fuzz.go"]="FuzzParseNetworkAddress" \
["./replacer_fuzz.go"]="FuzzReplacer" \
)
declare -A fuzzers_targets=(\
["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="parse-address" \
["./caddyconfig/caddyfile/parse_fuzz.go"]="parse-caddyfile" \
["./listeners_fuzz.go"]="parse-network-address" \
["./replacer_fuzz.go"]="replacer" \
)
fuzz_type="local-regression"
if [[ ${{ github.event_name }} == "schedule" ]]; then
fuzz_type="fuzzing"
fi
echo "Github event: ${{ github.event_name }}"
echo "Fuzzing type: $fuzz_type"
for f in $(find . -name \*_fuzz.go); do
FUZZER_DIRECTORY=$(dirname $f)
echo "go-fuzz-build func ${fuzzers_funcs[$f]} residing in $f"
go-fuzz-build -func "${fuzzers_funcs[$f]}" -libfuzzer -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" $FUZZER_DIRECTORY
echo "Generating fuzzer binary of func ${fuzzers_funcs[$f]} which resides in $f"
clang-7 -fsanitize=fuzzer "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}"
fuzzit create job caddyserver/${fuzzers_targets[$f]} $FUZZER_DIRECTORY/${fuzzers_targets[$f]} --api-key ${FUZZIT_API_KEY} --type "${fuzz_type}" --branch "${SYSTEM_PULLREQUEST_SOURCEBRANCH}" --revision "${BUILD_SOURCEVERSION}"
echo "Completed $f"
done

View file

@ -1,263 +0,0 @@
# Mutilated beyond recognition from the example at:
# https://docs.microsoft.com/azure/devops/pipelines/languages/go
trigger:
- v2
schedules:
- cron: "0 0 * * *"
displayName: Daily midnight fuzzing
branches:
include:
- v2
always: true
variables:
GOROOT: $(gorootDir)/go
GOPATH: $(system.defaultWorkingDirectory)/gopath
GOBIN: $(GOPATH)/bin
modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)'
jobs:
- job: crossPlatformTest
displayName: "Cross-Platform Tests"
strategy:
matrix:
linux:
imageName: ubuntu-16.04
gorootDir: /usr/local
mac:
imageName: macos-10.14
gorootDir: /usr/local
windows:
imageName: windows-2019
gorootDir: C:\
pool:
vmImage: $(imageName)
steps:
- bash: |
latestGo=$(curl "https://golang.org/VERSION?m=text")
echo "##vso[task.setvariable variable=LATEST_GO]$latestGo"
echo "Latest Go version: $latestGo"
displayName: "Get latest Go version"
- bash: |
sudo rm -f $(which go)
echo '##vso[task.prependpath]$(GOBIN)'
echo '##vso[task.prependpath]$(GOROOT)/bin'
mkdir -p '$(modulePath)'
shopt -s extglob
shopt -s dotglob
mv !(gopath) '$(modulePath)'
displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH
# Install Go (this varies by platform)
- bash: |
wget "https://dl.google.com/go/$(LATEST_GO).linux-amd64.tar.gz"
sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).linux-amd64.tar.gz"
condition: eq( variables['Agent.OS'], 'Linux' )
displayName: Install Go on Linux
- bash: |
wget "https://dl.google.com/go/$(LATEST_GO).darwin-amd64.tar.gz"
sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).darwin-amd64.tar.gz"
condition: eq( variables['Agent.OS'], 'Darwin' )
displayName: Install Go on macOS
# The low performance is partly due to PowerShell's attempt to update the progress bar. Disabling it speeds up the process.
# Reference: https://github.com/PowerShell/PowerShell/issues/2138
- powershell: |
$ProgressPreference = 'SilentlyContinue'
Write-Host "Downloading Go..."
(New-Object System.Net.WebClient).DownloadFile("https://dl.google.com/go/$(LATEST_GO).windows-amd64.zip", "$(LATEST_GO).windows-amd64.zip")
Write-Host "Extracting Go... (I'm slow too)"
7z x "$(LATEST_GO).windows-amd64.zip" -o"$(gorootDir)"
condition: eq( variables['Agent.OS'], 'Windows_NT' )
displayName: Install Go on Windows
- bash: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.23.6
displayName: Install golangci-lint
- script: |
go get github.com/axw/gocov/gocov
go get github.com/AlekSi/gocov-xml
go get -u github.com/jstemmer/go-junit-report
displayName: Install test and coverage analysis tools
- bash: |
printf "Using go at: $(which go)\n"
printf "Go version: $(go version)\n"
printf "\n\nGo environment:\n\n"
go env
printf "\n\nSystem environment:\n\n"
env
displayName: Print Go version and environment
- script: |
go get -v -t -d ./...
mkdir test-results
workingDirectory: '$(modulePath)'
displayName: Get dependencies
- bash: CGO_ENABLED=0 go build -trimpath -a -ldflags="-w -s" -v
workingDirectory: '$(modulePath)/cmd/caddy'
displayName: Build Caddy
- task: PublishBuildArtifacts@1
condition: eq( variables['Agent.OS'], 'Windows_NT' )
inputs:
pathtoPublish: '$(modulePath)/cmd/caddy/caddy.exe'
artifactName: caddy_v2.exe
- task: PublishBuildArtifacts@1
condition: ne( variables['Agent.OS'], 'Windows_NT' )
inputs:
pathtoPublish: '$(modulePath)/cmd/caddy/caddy'
artifactName: 'caddy_v2_$(Agent.OS)'
# its behavior is governed by .golangci.yml
- script: |
(golangci-lint run --out-format junit-xml) > test-results/lint-result.xml
exit 0
workingDirectory: '$(modulePath)'
continueOnError: true
displayName: Run lint check
- script: |
(go test -v -coverprofile=cover-profile.out -race ./... 2>&1) > test-results/test-result.out
workingDirectory: '$(modulePath)'
continueOnError: true
displayName: Run tests
- script: |
set -e
cmd/caddy/caddy start
go test -v -count=1 ./caddytest/...
cmd/caddy/caddy stop
workingDirectory: '$(modulePath)'
continueOnError: false
displayName: Run Integration tests
- script: |
mkdir coverage
gocov convert cover-profile.out > coverage/coverage.json
# Because Windows doesn't work with input redirection like *nix, but output redirection works.
(cat ./coverage/coverage.json | gocov-xml) > coverage/coverage.xml
workingDirectory: '$(modulePath)'
displayName: Prepare coverage reports
- script: |
(cat ./test-results/test-result.out | go-junit-report) > test-results/test-result.xml
workingDirectory: '$(modulePath)'
displayName: Prepare test report
- task: PublishCodeCoverageResults@1
displayName: Publish test coverage report
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: $(modulePath)/coverage/coverage.xml
- task: PublishTestResults@2
displayName: Publish unit test
inputs:
testResultsFormat: 'JUnit'
testResultsFiles: $(modulePath)/test-results/test-result.xml
testRunTitle: $(agent.OS) Unit Test
mergeTestResults: false
- task: PublishTestResults@2
displayName: Publish lint results
inputs:
testResultsFormat: 'JUnit'
testResultsFiles: $(modulePath)/test-results/lint-result.xml
testRunTitle: $(agent.OS) Lint
mergeTestResults: false
- bash: |
exit 1
condition: eq(variables['Agent.JobStatus'], 'SucceededWithIssues')
displayName: Coerce correct build result
- job: fuzzing
displayName: 'Fuzzing'
# Only run this job on schedules or PRs for non-forks.
condition: or(eq(variables['System.PullRequest.IsFork'], 'False'), eq(variables['Build.Reason'], 'Schedule') )
strategy:
matrix:
linux:
imageName: ubuntu-16.04
gorootDir: /usr/local
pool:
vmImage: $(imageName)
steps:
- bash: |
latestGo=$(curl "https://golang.org/VERSION?m=text")
echo "##vso[task.setvariable variable=LATEST_GO]$latestGo"
echo "Latest Go version: $latestGo"
displayName: "Get latest Go version"
- bash: |
sudo rm -f $(which go)
echo '##vso[task.prependpath]$(GOBIN)'
echo '##vso[task.prependpath]$(GOROOT)/bin'
mkdir -p '$(modulePath)'
shopt -s extglob
shopt -s dotglob
mv !(gopath) '$(modulePath)'
displayName: Remove old Go, set GOBIN/GOROOT, and move project into GOPATH
- bash: |
wget "https://dl.google.com/go/$(LATEST_GO).linux-amd64.tar.gz"
sudo tar -C $(gorootDir) -xzf "$(LATEST_GO).linux-amd64.tar.gz"
condition: eq( variables['Agent.OS'], 'Linux' )
displayName: Install Go on Linux
- bash: |
# Install Clang-7.0 because other versions seem to be missing the file libclang_rt.fuzzer-x86_64.a
sudo add-apt-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial-7 main"
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
sudo apt update && sudo apt install -y clang-7 lldb-7 lld-7
go get -v github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
wget -q -O fuzzit https://github.com/fuzzitdev/fuzzit/releases/download/v2.4.77/fuzzit_Linux_x86_64
chmod a+x fuzzit
mv fuzzit $(GOBIN)
displayName: Download go-fuzz tools and the Fuzzit CLI, and move Fuzzit CLI to GOBIN
condition: and(eq(variables['System.PullRequest.IsFork'], 'False') , eq( variables['Agent.OS'], 'Linux' ))
- bash: |
declare -A fuzzers_funcs=(\
["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="FuzzParseAddress" \
["./caddyconfig/caddyfile/parse_fuzz.go"]="FuzzParseCaddyfile" \
["./listeners_fuzz.go"]="FuzzParseNetworkAddress" \
["./replacer_fuzz.go"]="FuzzReplacer" \
)
declare -A fuzzers_targets=(\
["./caddyconfig/httpcaddyfile/addresses_fuzz.go"]="parse-address" \
["./caddyconfig/caddyfile/parse_fuzz.go"]="parse-caddyfile" \
["./listeners_fuzz.go"]="parse-network-address" \
["./replacer_fuzz.go"]="replacer" \
)
fuzz_type="local-regression"
if [[ $(Build.Reason) == "Schedule" ]]; then
fuzz_type="fuzzing"
fi
echo "Fuzzing type: $fuzz_type"
for f in $(find . -name \*_fuzz.go); do
FUZZER_DIRECTORY=$(dirname $f)
echo "go-fuzz-build func ${fuzzers_funcs[$f]} residing in $f"
go-fuzz-build -func "${fuzzers_funcs[$f]}" -libfuzzer -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" $FUZZER_DIRECTORY
echo "Generating fuzzer binary of func ${fuzzers_funcs[$f]} which resides in $f"
clang-7 -fsanitize=fuzzer "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}.a" -o "$FUZZER_DIRECTORY/${fuzzers_targets[$f]}"
fuzzit create job caddyserver/${fuzzers_targets[$f]} $FUZZER_DIRECTORY/${fuzzers_targets[$f]} --api-key ${FUZZIT_API_KEY} --type "${fuzz_type}" --branch "${SYSTEM_PULLREQUEST_SOURCEBRANCH}" --revision "${BUILD_SOURCEVERSION}"
echo "Completed $f"
done
env:
FUZZIT_API_KEY: $(FUZZIT_API_KEY)
workingDirectory: '$(modulePath)'
displayName: Generate fuzzers & submit them to Fuzzit