conduit/.gitlab-ci.yml

stages:
  - build
  - build docker image
  - test
  - upload artifacts

variables:
  GIT_SUBMODULE_STRATEGY: recursive
  FF_USE_FASTZIP: 1
  CACHE_COMPRESSION_LEVEL: fastest


# --------------------------------------------------------------------- #
#  Cargo: Compiling for different architectures                         #
# --------------------------------------------------------------------- #

.build-cargo-shared-settings:
  stage: "build"
  needs: []
  rules:
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
    - if: '$CI_COMMIT_TAG'
  interruptible: true
  image: "rust:latest"
  tags: ["docker"]
  cache:
    paths:
      - cargohome
      - target/
    key: "build_cache-$TARGET-release"
  variables:
    CARGO_PROFILE_RELEASE_LTO=true
    CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1
  before_script:
    - 'echo "Building for target $TARGET"'
    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
    - "cat /etc/*-release && rustc --version && cargo --version"  # Print version info for debugging
    - 'apt-get update -yqq'
    - 'echo "Installing packages: $NEEDED_PACKAGES"'
    - "apt-get install -yqq --no-install-recommends $NEEDED_PACKAGES"
    - "rustup target add $TARGET"
  script:
    - time cargo build --target $TARGET --release
    - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'


build:release:cargo:x86_64-unknown-linux-gnu:
  extends: .build-cargo-shared-settings
  variables:
    TARGET: "x86_64-unknown-linux-gnu"
  artifacts:
    name: "conduit-x86_64-unknown-linux-gnu"
    paths:
      - "conduit-x86_64-unknown-linux-gnu"
    expose_as: "Conduit for x86_64-unknown-linux-gnu"

build:release:cargo:armv7-unknown-linux-gnueabihf:
  extends: .build-cargo-shared-settings
  variables:
    TARGET: "armv7-unknown-linux-gnueabihf"
    NEEDED_PACKAGES: "build-essential gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf libc6-dev-armhf-cross"
    CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER: arm-linux-gnueabihf-gcc
    CC_armv7_unknown_linux_gnueabihf: arm-linux-gnueabihf-gcc
    CXX_armv7_unknown_linux_gnueabihf: arm-linux-gnueabihf-g++
  artifacts:
    name: "conduit-armv7-unknown-linux-gnueabihf"
    paths:
      - "conduit-armv7-unknown-linux-gnueabihf"
    expose_as: "Conduit for armv7-unknown-linux-gnueabihf"

build:release:cargo:aarch64-unknown-linux-gnu:
  extends: .build-cargo-shared-settings
  variables:
    TARGET: "aarch64-unknown-linux-gnu"
    NEEDED_PACKAGES: "build-essential gcc-8-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross"
    CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
    CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc
    CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++
    TARGET_CC: "/usr/bin/aarch64-linux-gnu-gcc-8"
    TARGET_AR: "/usr/bin/aarch64-linux-gnu-gcc-ar-8"
  artifacts:
    name: "conduit-aarch64-unknown-linux-gnu"
    paths:
      - "conduit-aarch64-unknown-linux-gnu"
    expose_as: "Conduit for aarch64-unknown-linux-gnu"

build:release:cargo:x86_64-unknown-linux-musl:
  extends: .build-cargo-shared-settings
  image: "rust:alpine"
  variables:
    TARGET: "x86_64-unknown-linux-musl"
  before_script:
    - 'echo "Building for target $TARGET"'
    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
    - "cat /etc/*-release && rustc --version && cargo --version"  # Print version info for debugging
    - "rustup target add $TARGET"
    - "apk add libc-dev"
  artifacts:
    name: "conduit-x86_64-unknown-linux-musl"
    paths:
      - "conduit-x86_64-unknown-linux-musl"
    expose_as: "Conduit for x86_64-unknown-linux-musl"



.cargo-debug-shared-settings:
  extends: ".build-cargo-shared-settings"
  rules:
    - if: '$CI_COMMIT_BRANCH'
    - if: '$CI_COMMIT_TAG'
  cache:
    key: "build_cache-$TARGET-debug"
  script:
    - "time cargo build --target $TARGET"
    - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'

build:debug:cargo:x86_64-unknown-linux-gnu:
  extends: ".cargo-debug-shared-settings"
  variables:
    TARGET: "x86_64-unknown-linux-gnu"
  artifacts:
    name: "conduit-debug-x86_64-unknown-linux-gnu"
    paths:
      - "conduit-debug-x86_64-unknown-linux-gnu"
    expose_as: "Conduit DEBUG for x86_64-unknown-linux-gnu"

build:debug:cargo:x86_64-unknown-linux-musl:
  extends: ".cargo-debug-shared-settings"
  image: "rust:alpine"
  variables:
    TARGET: "x86_64-unknown-linux-musl"
  before_script:
    - 'echo "Building for target $TARGET"'
    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
    - "cat /etc/*-release && rustc --version && cargo --version"  # Print version info for debugging
    - "rustup target add $TARGET"
    - "apk add libc-dev"
  artifacts:
    name: "conduit-debug-x86_64-unknown-linux-musl"
    paths:
      - "conduit-debug-x86_64-unknown-linux-musl"
    expose_as: "Conduit DEBUG for x86_64-unknown-linux-musl"



# --------------------------------------------------------------------- #
#  Cargo: Compiling deb packages for different architectures            #
# --------------------------------------------------------------------- #


.build-cargo-deb-shared-settings:
  stage: "build"
  needs: [ ]
  rules:
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
    - if: '$CI_COMMIT_TAG'
  interruptible: true
  image: "rust:latest"
  tags: ["docker"]
  cache:
    paths:
      - cargohome
      - target/
    key: "build_cache-deb-$TARGET"
  before_script:
    - 'echo "Building debian package for target $TARGET"'
    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
    - "cat /etc/*-release && rustc --version && cargo --version"  # Print version info for debugging
    - 'apt-get update -yqq'
    - 'echo "Installing packages: $NEEDED_PACKAGES"'
    - "apt-get install -yqq --no-install-recommends $NEEDED_PACKAGES"
    - "rustup target add $TARGET"
    - "cargo install cargo-deb"
  script:
    - time cargo deb --target $TARGET
    - 'mv target/$TARGET/debian/*.deb "conduit-$TARGET.deb"'

build:cargo-deb:x86_64-unknown-linux-gnu:
  extends: .build-cargo-deb-shared-settings
  variables:
    TARGET: "x86_64-unknown-linux-gnu"
    NEEDED_PACKAGES: ""
  artifacts:
    name: "conduit-x86_64-unknown-linux-gnu.deb"
    paths:
      - "conduit-x86_64-unknown-linux-gnu.deb"
    expose_as: "Debian Package x86_64"



# --------------------------------------------------------------------- #
#  Create and publish docker image                                      #
# --------------------------------------------------------------------- #

.docker-shared-settings:
  stage: "build docker image"
  needs: []
  interruptible: true
  image:
    name: "gcr.io/kaniko-project/executor:debug"
    entrypoint: [""]
  tags: ["docker"]
  variables:
    # Configure Kaniko Caching: https://cloud.google.com/build/docs/kaniko-cache
    KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=120h --cache-repo $CI_REGISTRY_IMAGE/kaniko-ci-cache"
  before_script:
    - "mkdir -p /kaniko/.docker"
    - 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$DOCKER_HUB\":{\"username\":\"$DOCKER_HUB_USER\",\"password\":\"$DOCKER_HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json'


# Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image
build:docker:main:
  extends: .docker-shared-settings
  needs:
    - "build:release:cargo:x86_64-unknown-linux-musl"
  script:
    - >
      /kaniko/executor
      $KANIKO_CACHE_ARGS
      --context $CI_PROJECT_DIR
      --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
      --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
      --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
      --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
      --destination "$CI_REGISTRY_IMAGE/conduit:latest"
      --destination "$CI_REGISTRY_IMAGE/conduit:alpine"
      --destination "$CI_REGISTRY_IMAGE/conduit:commit-$CI_COMMIT_SHORT_SHA"
      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:alpine"
      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:commit-$CI_COMMIT_SHORT_SHA"
  rules:
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
    - if: '$CI_COMMIT_TAG'



# --------------------------------------------------------------------- #
#  Run tests                                                            #
# --------------------------------------------------------------------- #

test:cargo:
  stage: "test"
  needs: [ ]
  image: "rust:latest"
  tags: [ "docker" ]
  variables:
    CARGO_HOME: "cargohome"
  cache:
    paths:
      - target
      - cargohome
    key: test_cache
  interruptible: true
  before_script:
    - mkdir -p $CARGO_HOME && echo "using $CARGO_HOME to cache cargo deps"
    - apt-get update -yqq
    - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config
    - rustup component add clippy rustfmt
  script:
    - rustc --version && cargo --version  # Print version info for debugging
    - cargo fmt --all -- --check
    - cargo test --workspace --verbose --locked
    - cargo clippy

test:sytest:
  stage: "test"
  allow_failure: true
  needs:
    - "build:debug:cargo:x86_64-unknown-linux-musl"
  image:
    name: "valkum/sytest-conduit:latest"
    entrypoint: [ "" ]
  tags: [ "docker" ]
  variables:
    PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz"
  before_script:
    - "mkdir -p /app"
    - "cp ./conduit-debug-x86_64-unknown-linux-musl /app/conduit"
    - "chmod +x /app/conduit"
    - "rm -rf /src && ln -s $CI_PROJECT_DIR/ /src"
    - "mkdir -p /work/server-0/database/ && mkdir -p /work/server-1/database/ && mkdir -p /work/server-2/database/"
    - "cd /"
  script:
    - "SYTEST_EXIT_CODE=0"
    - "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1"
    - "perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml \"Sytest\" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap"
    - "exit $SYTEST_EXIT_CODE"
  artifacts:
    when: always
    paths:
      - "$CI_PROJECT_DIR/sytest.xml"
      - "$CI_PROJECT_DIR/results.tap"
    reports:
      junit: "$CI_PROJECT_DIR/sytest.xml"


test:register:element-web-stable:
  stage: "test"
  needs:
    - "build:debug:cargo:x86_64-unknown-linux-gnu"
  image: "buildkite/puppeteer:latest"
  tags: [ "docker" ]
  interruptible: true
  script:
    - "CONDUIT_CONFIG=tests/test-config.toml ./conduit-debug-x86_64-unknown-linux-gnu > conduit.log &"
    - "cd tests/client-element-web/"
    - "npm install puppeteer"
    - "node test-element-web-registration.js \"https://app.element.io/\" \"http://localhost:6167\""
    - "killall --regexp \"conduit\""
    - "cd ../.."
    - "cat conduit.log"
  artifacts:
    paths:
      - "tests/client-element-web/*.png"
      - "*.log"
    expire_in: 1 week
    when: always
  retry: 1


# --------------------------------------------------------------------- #
#  Store binaries as package so they have download urls                 #
# --------------------------------------------------------------------- #

publish:package:
  stage: "upload artifacts"
  needs:
    - "build:release:cargo:x86_64-unknown-linux-gnu"
    - "build:release:cargo:armv7-unknown-linux-gnueabihf"
    - "build:release:cargo:aarch64-unknown-linux-gnu"
    - "build:release:cargo:x86_64-unknown-linux-musl"
    - "build:cargo-deb:x86_64-unknown-linux-gnu"
  rules:
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
    - if: '$CI_COMMIT_TAG'
  image: curlimages/curl:latest
  tags: ["docker"]
  variables:
    GIT_STRATEGY: "none" # Don't need a clean copy of the code, we just operate on artifacts
  script:
    - 'BASE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/conduit-${CI_COMMIT_REF_SLUG}/build-${CI_PIPELINE_ID}"'
    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-gnu "${BASE_URL}/conduit-x86_64-unknown-linux-gnu"'
    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-armv7-unknown-linux-gnueabihf "${BASE_URL}/conduit-armv7-unknown-linux-gnueabihf"'
    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-aarch64-unknown-linux-gnu "${BASE_URL}/conduit-aarch64-unknown-linux-gnu"'
    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-musl "${BASE_URL}/conduit-x86_64-unknown-linux-musl"'
    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-gnu.deb "${BASE_URL}/conduit-x86_64-unknown-linux-gnu.deb"'