TudbuT/conduit
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

CI: New Multiarch builds and Docker images + cargo clippy/test output now integrated into GitLab

Browse source
This commit is contained in:
Jonas Zohren 2021-11-21 17:34:08 +00:00 • committed by Timo Kösters
parent 24a835647c
commit 2fff720df3
9 changed files with 356 additions and 409 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.dockerignore
View file

@ -14,6 +14,8 @@ docker-compose*
# Git folder # Git folder
.git .git
.gitea .gitea
.gitlab
.github
# Dot files # Dot files
.env .env

376
.gitlab-ci.yml
View file

@ -9,7 +9,6 @@ variables:
FF_USE_FASTZIP: 1 FF_USE_FASTZIP: 1
CACHE_COMPRESSION_LEVEL: fastest CACHE_COMPRESSION_LEVEL: fastest
# --------------------------------------------------------------------- # # --------------------------------------------------------------------- #
# Cargo: Compiling for different architectures # # Cargo: Compiling for different architectures #
# --------------------------------------------------------------------- # # --------------------------------------------------------------------- #
@ -20,7 +19,7 @@ variables:
rules: rules:
- if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH == "master"'
- if: '$CI_COMMIT_BRANCH == "next"' - if: '$CI_COMMIT_BRANCH == "next"'
- if: '$CI_COMMIT_TAG' - if: "$CI_COMMIT_TAG"
interruptible: true interruptible: true
image: "rust:latest" image: "rust:latest"
tags: ["docker"] tags: ["docker"]
@ -28,258 +27,209 @@ variables:
paths: paths:
- cargohome - cargohome
- target/ - target/
key: "build_cache-$TARGET-release" key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--release"
variables: variables:
CARGO_PROFILE_RELEASE_LTO=true CARGO_PROFILE_RELEASE_LTO: "true"
CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1 CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1"
before_script: before_script:
- 'echo "Building for target $TARGET"' - 'echo "Building for target $TARGET"'
- 'mkdir -p cargohome && CARGOHOME="cargohome"' - 'mkdir -p cargohome && CARGOHOME="cargohome"'
- "cat /etc/*-release && rustc --version && cargo --version" # Print version info for debugging - "rustc --version && cargo --version && rustup show" # Print version info for debugging
- 'apt-get update -yqq'
- 'echo "Installing packages: $NEEDED_PACKAGES"'
- "apt-get install -yqq --no-install-recommends $NEEDED_PACKAGES"
- "rustup target add $TARGET" - "rustup target add $TARGET"
script: script:
- time cargo build --target $TARGET --release - time cargo build --target $TARGET --release
- 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"' - 'cp "target/$TARGET/release/conduit" "conduit-$TARGET"'
artifacts: artifacts:
expire_in: never expire_in: never
build:release:cargo:x86_64-unknown-linux-musl-with-debug:
build:release:cargo:x86_64-unknown-linux-gnu:
extends: .build-cargo-shared-settings extends: .build-cargo-shared-settings
image: messense/rust-musl-cross:x86_64-musl
variables: variables:
TARGET: "x86_64-unknown-linux-gnu" CARGO_PROFILE_RELEASE_DEBUG: 2 # Enable debug info for flamegraph profiling
TARGET: "x86_64-unknown-linux-musl"
after_script:
- "mv ./conduit-x86_64-unknown-linux-musl ./conduit-x86_64-unknown-linux-musl-with-debug"
artifacts: artifacts:
name: "conduit-x86_64-unknown-linux-gnu" name: "conduit-x86_64-unknown-linux-musl-with-debug"
paths: paths:
- "conduit-x86_64-unknown-linux-gnu" - "conduit-x86_64-unknown-linux-musl-with-debug"
expose_as: "Conduit for x86_64-unknown-linux-gnu" expose_as: "Conduit for x86_64-unknown-linux-musl-with-debug"
build:release:cargo:armv7-unknown-linux-gnueabihf:
extends: .build-cargo-shared-settings
variables:
TARGET: "armv7-unknown-linux-gnueabihf"
NEEDED_PACKAGES: "build-essential gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf libc6-dev-armhf-cross"
CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER: arm-linux-gnueabihf-gcc
CC_armv7_unknown_linux_gnueabihf: arm-linux-gnueabihf-gcc
CXX_armv7_unknown_linux_gnueabihf: arm-linux-gnueabihf-g++
artifacts:
name: "conduit-armv7-unknown-linux-gnueabihf"
paths:
- "conduit-armv7-unknown-linux-gnueabihf"
expose_as: "Conduit for armv7-unknown-linux-gnueabihf"
build:release:cargo:aarch64-unknown-linux-gnu:
extends: .build-cargo-shared-settings
variables:
TARGET: "aarch64-unknown-linux-gnu"
NEEDED_PACKAGES: "build-essential gcc-10-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross"
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc
CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++
TARGET_CC: "/usr/bin/aarch64-linux-gnu-gcc-10"
TARGET_AR: "/usr/bin/aarch64-linux-gnu-gcc-ar-10"
artifacts:
name: "conduit-aarch64-unknown-linux-gnu"
paths:
- "conduit-aarch64-unknown-linux-gnu"
expose_as: "Conduit for aarch64-unknown-linux-gnu"
build:release:cargo:x86_64-unknown-linux-musl: build:release:cargo:x86_64-unknown-linux-musl:
extends: .build-cargo-shared-settings extends: .build-cargo-shared-settings
image: "rust:alpine" image: messense/rust-musl-cross:x86_64-musl
variables: variables:
TARGET: "x86_64-unknown-linux-musl" TARGET: "x86_64-unknown-linux-musl"
before_script:
- 'echo "Building for target $TARGET"'
- 'mkdir -p cargohome && CARGOHOME="cargohome"'
- "cat /etc/*-release && rustc --version && cargo --version" # Print version info for debugging
- "rustup target add $TARGET"
- "apk add libc-dev"
artifacts: artifacts:
name: "conduit-x86_64-unknown-linux-musl" name: "conduit-x86_64-unknown-linux-musl"
paths: paths:
- "conduit-x86_64-unknown-linux-musl" - "conduit-x86_64-unknown-linux-musl"
expose_as: "Conduit for x86_64-unknown-linux-musl" expose_as: "Conduit for x86_64-unknown-linux-musl"
build:release:cargo:arm-unknown-linux-musleabihf:
extends: .build-cargo-shared-settings
image: messense/rust-musl-cross:arm-musleabihf
variables:
TARGET: "arm-unknown-linux-musleabihf"
artifacts:
name: "conduit-arm-unknown-linux-musleabihf"
paths:
- "conduit-arm-unknown-linux-musleabihf"
expose_as: "Conduit for arm-unknown-linux-musleabihf"
build:release:cargo:armv7-unknown-linux-musleabihf:
extends: .build-cargo-shared-settings
image: messense/rust-musl-cross:armv7-musleabihf
variables:
TARGET: "armv7-unknown-linux-musleabihf"
artifacts:
name: "conduit-armv7-unknown-linux-musleabihf"
paths:
- "conduit-armv7-unknown-linux-musleabihf"
expose_as: "Conduit for armv7-unknown-linux-musleabihf"
build:release:cargo:aarch64-unknown-linux-musl:
extends: .build-cargo-shared-settings
image: messense/rust-musl-cross:aarch64-musl
variables:
TARGET: "aarch64-unknown-linux-musl"
artifacts:
name: "conduit-aarch64-unknown-linux-musl"
paths:
- "conduit-aarch64-unknown-linux-musl"
expose_as: "Conduit for aarch64-unknown-linux-musl"
.cargo-debug-shared-settings: .cargo-debug-shared-settings:
extends: ".build-cargo-shared-settings" extends: ".build-cargo-shared-settings"
rules: rules:
- if: '$CI_COMMIT_BRANCH' - if: '$CI_COMMIT_BRANCH != "master"'
- if: '$CI_COMMIT_TAG'
cache: cache:
key: "build_cache-$TARGET-debug" key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug"
script: script:
- "time cargo build --target $TARGET" - "time cargo build --target $TARGET"
- 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"' - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'
artifacts: artifacts:
expire_in: 4 weeks expire_in: 4 weeks
build:debug:cargo:x86_64-unknown-linux-gnu:
extends: ".cargo-debug-shared-settings"
variables:
TARGET: "x86_64-unknown-linux-gnu"
artifacts:
name: "conduit-debug-x86_64-unknown-linux-gnu"
paths:
- "conduit-debug-x86_64-unknown-linux-gnu"
expose_as: "Conduit DEBUG for x86_64-unknown-linux-gnu"
build:debug:cargo:x86_64-unknown-linux-musl: build:debug:cargo:x86_64-unknown-linux-musl:
extends: ".cargo-debug-shared-settings" extends: ".cargo-debug-shared-settings"
image: "rust:alpine" image: messense/rust-musl-cross:x86_64-musl
variables: variables:
TARGET: "x86_64-unknown-linux-musl" TARGET: "x86_64-unknown-linux-musl"
before_script:
- 'echo "Building for target $TARGET"'
- 'mkdir -p cargohome && CARGOHOME="cargohome"'
- "cat /etc/*-release && rustc --version && cargo --version" # Print version info for debugging
- "rustup target add $TARGET"
- "apk add libc-dev"
artifacts: artifacts:
name: "conduit-debug-x86_64-unknown-linux-musl" name: "conduit-debug-x86_64-unknown-linux-musl"
paths: paths:
- "conduit-debug-x86_64-unknown-linux-musl" - "conduit-debug-x86_64-unknown-linux-musl"
expose_as: "Conduit DEBUG for x86_64-unknown-linux-musl" expose_as: "Conduit DEBUG for x86_64-unknown-linux-musl"
# --------------------------------------------------------------------- #
# Cargo: Compiling deb packages for different architectures #
# --------------------------------------------------------------------- #
.build-cargo-deb-shared-settings:
stage: "build"
needs: [ ]
rules:
- if: '$CI_COMMIT_BRANCH == "master"'
- if: '$CI_COMMIT_BRANCH == "next"'
- if: '$CI_COMMIT_TAG'
interruptible: true
image: "rust:latest"
tags: ["docker"]
cache:
paths:
- cargohome
- target/
key: "build_cache-deb-$TARGET"
before_script:
- 'echo "Building debian package for target $TARGET"'
- 'mkdir -p cargohome && CARGOHOME="cargohome"'
- "cat /etc/*-release && rustc --version && cargo --version" # Print version info for debugging
- 'apt-get update -yqq'
- 'echo "Installing packages: $NEEDED_PACKAGES"'
- "apt-get install -yqq --no-install-recommends $NEEDED_PACKAGES"
- "rustup target add $TARGET"
- "cargo install cargo-deb"
script:
- time cargo deb --target $TARGET
- 'mv target/$TARGET/debian/*.deb "conduit-$TARGET.deb"'
build:cargo-deb:x86_64-unknown-linux-gnu:
extends: .build-cargo-deb-shared-settings
variables:
TARGET: "x86_64-unknown-linux-gnu"
NEEDED_PACKAGES: ""
artifacts:
name: "conduit-x86_64-unknown-linux-gnu.deb"
paths:
- "conduit-x86_64-unknown-linux-gnu.deb"
expose_as: "Debian Package x86_64"
# --------------------------------------------------------------------- # # --------------------------------------------------------------------- #
# Create and publish docker image # # Create and publish docker image #
# --------------------------------------------------------------------- # # --------------------------------------------------------------------- #
# Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image
.docker-shared-settings: .docker-shared-settings:
stage: "build docker image" stage: "build docker image"
needs: [] image: jdrouet/docker-with-buildx:stable
interruptible: true
image:
name: "gcr.io/kaniko-project/executor:debug"
entrypoint: [""]
tags: ["docker"] tags: ["docker"]
services:
- docker:dind
needs:
- "build:release:cargo:x86_64-unknown-linux-musl"
- "build:release:cargo:arm-unknown-linux-musleabihf"
- "build:release:cargo:armv7-unknown-linux-musleabihf"
- "build:release:cargo:aarch64-unknown-linux-musl"
variables: variables:
# Configure Kaniko Caching: https://cloud.google.com/build/docs/kaniko-cache DOCKER_HOST: tcp://docker:2375/
KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=120h --cache-repo $CI_REGISTRY_IMAGE/kaniko-ci-cache" DOCKER_TLS_CERTDIR: ""
DOCKER_DRIVER: overlay2
PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/amd64"
DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile"
before_script: before_script:
- "mkdir -p /kaniko/.docker" - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
- 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$DOCKER_HUB\":{\"username\":\"$DOCKER_HUB_USER\",\"password\":\"$DOCKER_HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json' # Only log in to Dockerhub if the credentials are given:
- if [ -n "${DOCKER_HUB}" ]; then docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWORD" "$DOCKER_HUB"; fi
script:
# Prepare buildx to build multiarch stuff:
- docker context create 'ci-context'
- docker buildx create --name 'multiarch-builder' --use 'ci-context'
# Copy binaries to their docker arch path
- mkdir -p linux/ && mv ./conduit-x86_64-unknown-linux-musl linux/amd64
- mkdir -p linux/arm/ && mv ./conduit-arm-unknown-linux-musleabihf linux/arm/v6
- mkdir -p linux/arm/ && mv ./conduit-armv7-unknown-linux-musleabihf linux/arm/v7
- mkdir -p linux/arm64/ && mv ./conduit-aarch64-unknown-linux-musl linux/arm64/v8
# Actually create multiarch image:
- >
docker buildx build
--pull
--push
--build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
--build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
--build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
--platform "$PLATFORMS"
--tag "$GL_IMAGE_TAG"
--tag "$GL_IMAGE_TAG-commit-$CI_COMMIT_SHORT_SHA"
--file "$DOCKER_FILE" .
# Only try to push to docker hub, if auth data for dockerhub exists:
- if [ -n "${DOCKER_HUB}" ]; then docker push "$DH_IMAGE_TAG"; fi
- if [ -n "${DOCKER_HUB}" ]; then docker push "$DH_IMAGE_TAG-commit-$CI_COMMIT_SHORT_SHA"; fi
build:docker:next: build:docker:next:
extends: .docker-shared-settings extends: .docker-shared-settings
needs:
- "build:release:cargo:x86_64-unknown-linux-musl"
script:
- >
/kaniko/executor
$KANIKO_CACHE_ARGS
--force
--context $CI_PROJECT_DIR
--build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
--build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
--build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
--dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
--destination "$CI_REGISTRY_IMAGE/conduit:next"
--destination "$CI_REGISTRY_IMAGE/conduit:next-alpine"
--destination "$CI_REGISTRY_IMAGE/conduit:commit-$CI_COMMIT_SHORT_SHA"
--destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
--destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next-alpine"
--destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:commit-$CI_COMMIT_SHORT_SHA"
rules: rules:
- if: '$CI_COMMIT_BRANCH == "next"' - if: '$CI_COMMIT_BRANCH == "next"'
variables:
GL_IMAGE_TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:next"
DH_IMAGE_TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
build:docker:master: build:docker:master:
extends: .docker-shared-settings extends: .docker-shared-settings
needs:
- "build:release:cargo:x86_64-unknown-linux-musl"
script:
- >
/kaniko/executor
$KANIKO_CACHE_ARGS
--context $CI_PROJECT_DIR
--build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
--build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
--build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
--dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
--destination "$CI_REGISTRY_IMAGE/conduit:latest"
--destination "$CI_REGISTRY_IMAGE/conduit:latest-alpine"
--destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
--destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest-alpine"
rules: rules:
- if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH == "master"'
variables:
GL_IMAGE_TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:latest"
DH_IMAGE_TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
## Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image
build:docker:tags: #.docker-shared-settings:
extends: .docker-shared-settings # stage: "build docker image"
needs: # needs: []
- "build:release:cargo:x86_64-unknown-linux-musl" # interruptible: true
script: # image:
- > # name: "gcr.io/kaniko-project/executor:debug"
/kaniko/executor # entrypoint: [""]
$KANIKO_CACHE_ARGS # tags: ["docker"]
--context $CI_PROJECT_DIR # variables:
--build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') # # Configure Kaniko Caching: https://cloud.google.com/build/docs/kaniko-cache
--build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) # KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=120h --cache-repo $CI_REGISTRY_IMAGE/kaniko-ci-cache"
--build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA" # before_script:
--dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile" # - "mkdir -p /kaniko/.docker"
--destination "$CI_REGISTRY_IMAGE/conduit:$CI_COMMIT_TAG" # - 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$DOCKER_HUB\":{\"username\":\"$DOCKER_HUB_USER\",\"password\":\"$DOCKER_HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json'
--destination "$CI_REGISTRY_IMAGE/conduit:$CI_COMMIT_TAG-alpine" #
--destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG" #
--destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG-alpine" #build:docker:next:
rules: # extends: .docker-shared-settings
- if: '$CI_COMMIT_TAG' # needs:
# - "build:release:cargo:x86_64-unknown-linux-musl"
# script:
# - >
# /kaniko/executor
# $KANIKO_CACHE_ARGS
# --force
# --context $CI_PROJECT_DIR
# --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
# --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
# --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
# --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
# --destination "$CI_REGISTRY_IMAGE/conduit:next"
# --destination "$CI_REGISTRY_IMAGE/conduit:next-alpine"
# --destination "$CI_REGISTRY_IMAGE/conduit:commit-$CI_COMMIT_SHORT_SHA"
# --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
# --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next-alpine"
# --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:commit-$CI_COMMIT_SHORT_SHA"
# rules:
# - if: '$CI_COMMIT_BRANCH == "next"'
#
#
# --------------------------------------------------------------------- # # --------------------------------------------------------------------- #
# Run tests # # Run tests #
@ -287,9 +237,9 @@ build:docker:tags:
test:cargo: test:cargo:
stage: "test" stage: "test"
needs: [ ] needs: []
image: "rust:latest" image: "rust:latest"
tags: [ "docker" ] tags: ["docker"]
variables: variables:
CARGO_HOME: "cargohome" CARGO_HOME: "cargohome"
cache: cache:
@ -301,13 +251,20 @@ test:cargo:
before_script: before_script:
- mkdir -p $CARGO_HOME && echo "using $CARGO_HOME to cache cargo deps" - mkdir -p $CARGO_HOME && echo "using $CARGO_HOME to cache cargo deps"
- apt-get update -yqq - apt-get update -yqq
- apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config wget
- rustup component add clippy rustfmt - rustup component add clippy rustfmt
- wget "https://faulty-storage.de/gitlab-report"
- chmod +x ./gitlab-report
script: script:
- rustc --version && cargo --version # Print version info for debugging - rustc --version && cargo --version # Print version info for debugging
- cargo fmt --all -- --check - cargo fmt --all -- --check
- cargo test --workspace --verbose --locked - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | ./gitlab-report -p test > $CI_PROJECT_DIR/report.xml"
- cargo clippy - "cargo clippy --color always --verbose --message-format=json | ./gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json"
artifacts:
when: always
reports:
junit: report.xml
codequality: gl-code-quality-report.json
test:sytest: test:sytest:
stage: "test" stage: "test"
@ -316,8 +273,8 @@ test:sytest:
- "build:debug:cargo:x86_64-unknown-linux-musl" - "build:debug:cargo:x86_64-unknown-linux-musl"
image: image:
name: "valkum/sytest-conduit:latest" name: "valkum/sytest-conduit:latest"
entrypoint: [ "" ] entrypoint: [""]
tags: [ "docker" ] tags: ["docker"]
variables: variables:
PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz" PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz"
before_script: before_script:
@ -330,7 +287,7 @@ test:sytest:
script: script:
- "SYTEST_EXIT_CODE=0" - "SYTEST_EXIT_CODE=0"
- "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1" - "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1"
- "perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml \"Sytest\" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap" - 'perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml "Sytest" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap'
- "exit $SYTEST_EXIT_CODE" - "exit $SYTEST_EXIT_CODE"
artifacts: artifacts:
when: always when: always
@ -340,7 +297,6 @@ test:sytest:
reports: reports:
junit: "$CI_PROJECT_DIR/sytest.xml" junit: "$CI_PROJECT_DIR/sytest.xml"
# --------------------------------------------------------------------- # # --------------------------------------------------------------------- #
# Store binaries as package so they have download urls # # Store binaries as package so they have download urls #
# --------------------------------------------------------------------- # # --------------------------------------------------------------------- #
@ -348,25 +304,31 @@ test:sytest:
publish:package: publish:package:
stage: "upload artifacts" stage: "upload artifacts"
needs: needs:
- "build:release:cargo:x86_64-unknown-linux-gnu"
- "build:release:cargo:armv7-unknown-linux-gnueabihf"
- "build:release:cargo:aarch64-unknown-linux-gnu"
- "build:release:cargo:x86_64-unknown-linux-musl" - "build:release:cargo:x86_64-unknown-linux-musl"
- "build:cargo-deb:x86_64-unknown-linux-gnu" - "build:release:cargo:arm-unknown-linux-musleabihf"
- "build:release:cargo:armv7-unknown-linux-musleabihf"
- "build:release:cargo:aarch64-unknown-linux-musl"
# - "build:cargo-deb:x86_64-unknown-linux-gnu"
rules: rules:
- if: '$CI_COMMIT_BRANCH == "master"' - if: '$CI_COMMIT_BRANCH == "master"'
- if: '$CI_COMMIT_BRANCH == "next"' - if: '$CI_COMMIT_BRANCH == "next"'
- if: '$CI_COMMIT_TAG' - if: "$CI_COMMIT_TAG"
image: curlimages/curl:latest image: curlimages/curl:latest
tags: ["docker"] tags: ["docker"]
variables: variables:
GIT_STRATEGY: "none" # Don't need a clean copy of the code, we just operate on artifacts GIT_STRATEGY: "none" # Don't need a clean copy of the code, we just operate on artifacts
script: script:
- 'BASE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/conduit-${CI_COMMIT_REF_SLUG}/build-${CI_PIPELINE_ID}"' - 'BASE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/conduit-${CI_COMMIT_REF_SLUG}/build-${CI_PIPELINE_ID}"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-gnu "${BASE_URL}/conduit-x86_64-unknown-linux-gnu"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-armv7-unknown-linux-gnueabihf "${BASE_URL}/conduit-armv7-unknown-linux-gnueabihf"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-aarch64-unknown-linux-gnu "${BASE_URL}/conduit-aarch64-unknown-linux-gnu"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-musl "${BASE_URL}/conduit-x86_64-unknown-linux-musl"' - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-musl "${BASE_URL}/conduit-x86_64-unknown-linux-musl"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-gnu.deb "${BASE_URL}/conduit-x86_64-unknown-linux-gnu.deb"' - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-arm-unknown-linux-musleabihf "${BASE_URL}/conduit-arm-unknown-linux-musleabihf"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-armv7-unknown-linux-musleabihf "${BASE_URL}/conduit-armv7-unknown-linux-musleabihf"'
- 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-aarch64-unknown-linux-musl "${BASE_URL}/conduit-aarch64-unknown-linux-musl"'
# Avoid duplicate pipelines
# See: https://docs.gitlab.com/ee/ci/yaml/workflow.html#switch-between-branch-pipelines-and-merge-request-pipelines
workflow:
rules:
- if: '$CI_PIPELINE_SOURCE == "merge_request_event"'
- if: "$CI_COMMIT_BRANCH && $CI_OPEN_MERGE_REQUESTS"
when: never
- if: "$CI_COMMIT_BRANCH"

36
Cargo.lock generated
View file

@ -1968,7 +1968,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma" name = "ruma"
version = "0.4.0" version = "0.4.0"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"assign", "assign",
"js_int", "js_int",
@ -1989,7 +1989,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-api" name = "ruma-api"
version = "0.18.5" version = "0.18.5"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"bytes", "bytes",
"http", "http",
@ -2005,7 +2005,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-api-macros" name = "ruma-api-macros"
version = "0.18.5" version = "0.18.5"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"proc-macro-crate", "proc-macro-crate",
"proc-macro2", "proc-macro2",
@ -2016,7 +2016,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-appservice-api" name = "ruma-appservice-api"
version = "0.4.0" version = "0.4.0"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"ruma-api", "ruma-api",
"ruma-common", "ruma-common",
@ -2030,7 +2030,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-client-api" name = "ruma-client-api"
version = "0.12.3" version = "0.12.3"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"assign", "assign",
"bytes", "bytes",
@ -2050,7 +2050,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-common" name = "ruma-common"
version = "0.6.0" version = "0.6.0"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"indexmap", "indexmap",
"js_int", "js_int",
@ -2065,7 +2065,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-events" name = "ruma-events"
version = "0.24.6" version = "0.24.6"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"indoc", "indoc",
"js_int", "js_int",
@ -2081,7 +2081,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-events-macros" name = "ruma-events-macros"
version = "0.24.6" version = "0.24.6"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"proc-macro-crate", "proc-macro-crate",
"proc-macro2", "proc-macro2",
@ -2092,7 +2092,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-federation-api" name = "ruma-federation-api"
version = "0.3.1" version = "0.3.1"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"js_int", "js_int",
"ruma-api", "ruma-api",
@ -2107,7 +2107,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-identifiers" name = "ruma-identifiers"
version = "0.20.0" version = "0.20.0"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"paste", "paste",
"percent-encoding", "percent-encoding",
@ -2122,7 +2122,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-identifiers-macros" name = "ruma-identifiers-macros"
version = "0.20.0" version = "0.20.0"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"quote", "quote",
"ruma-identifiers-validation", "ruma-identifiers-validation",
@ -2132,7 +2132,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-identifiers-validation" name = "ruma-identifiers-validation"
version = "0.5.0" version = "0.5.0"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"thiserror", "thiserror",
] ]
@ -2140,7 +2140,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-identity-service-api" name = "ruma-identity-service-api"
version = "0.3.0" version = "0.3.0"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"js_int", "js_int",
"ruma-api", "ruma-api",
@ -2153,7 +2153,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-push-gateway-api" name = "ruma-push-gateway-api"
version = "0.3.0" version = "0.3.0"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"js_int", "js_int",
"ruma-api", "ruma-api",
@ -2168,7 +2168,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-serde" name = "ruma-serde"
version = "0.5.0" version = "0.5.0"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"bytes", "bytes",
"form_urlencoded", "form_urlencoded",
@ -2182,7 +2182,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-serde-macros" name = "ruma-serde-macros"
version = "0.5.0" version = "0.5.0"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"proc-macro-crate", "proc-macro-crate",
"proc-macro2", "proc-macro2",
@ -2193,7 +2193,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-signatures" name = "ruma-signatures"
version = "0.9.0" version = "0.9.0"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"base64 0.13.0", "base64 0.13.0",
"ed25519-dalek", "ed25519-dalek",
@ -2210,7 +2210,7 @@ dependencies = [
[[package]] [[package]]
name = "ruma-state-res" name = "ruma-state-res"
version = "0.4.1" version = "0.4.1"
source = "git+https://github.com/ruma/ruma?rev=44cfd0adbc83303c19aef590ad0d71647e19f197#44cfd0adbc83303c19aef590ad0d71647e19f197" source = "git+https://github.com/ruma/ruma?rev=e7f01ca55a1eff437bad754bf0554cc09f44ec2a#e7f01ca55a1eff437bad754bf0554cc09f44ec2a"
dependencies = [ dependencies = [
"itertools 0.10.1", "itertools 0.10.1",
"js_int", "js_int",

3
Cargo.toml
View file

@ -120,13 +120,12 @@ maintainer-scripts = "debian/"
systemd-units = { unit-name = "matrix-conduit" } systemd-units = { unit-name = "matrix-conduit" }
[profile.dev] [profile.dev]
lto = 'thin' lto = 'off'
incremental = true incremental = true
[profile.release] [profile.release]
lto = 'thin' lto = 'thin'
incremental = true incremental = true
codegen-units=32 codegen-units=32
# If you want to make flamegraphs, enable debug info: # If you want to make flamegraphs, enable debug info:
# debug = true # debug = true

52
DEPLOY.md
View file

@ -2,25 +2,30 @@
## Getting help ## Getting help
If you run into any problems while setting up Conduit, write an email to `timo@koesters.xyz`, ask us in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new). If you run into any problems while setting up Conduit, write an email to `timo@koesters.xyz`, ask us
in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new).
## Installing Conduit ## Installing Conduit
Although you might be able to compile Conduit for Windows, we do recommend running it on a linux server. We therefore
only offer Linux binaries.
You may simply download the binary that fits your machine. Run `uname -m` to see what you need. Now copy the right url: You may simply download the binary that fits your machine. Run `uname -m` to see what you need. Now copy the right url:
| CPU Architecture | GNU (Ubuntu, Debian, ArchLinux, ...) | MUSL (Alpine, ... ) | | CPU Architecture | Download link |
| -------------------- | ------------------------------------- | ----------------------- | | ------------------------------------------- | ----------------------- |
| x84_64 / amd64 | [Download][x84_64-gnu] | [Download][x84_64-musl] | | x84_64 / amd64 (Most servers and computers) | [Download][x84_64-musl] |
| armv7 (Raspberry Pi) | [Download][armv7-gnu] | - | | armv6 | [Download][armv6-musl] |
| armv8 / aarch64 | [Download][armv8-gnu] | - | | armv7 (e.g. Raspberry Pi by default) | [Download][armv7-musl] |
| armv8 / aarch64 | [Download][armv8-musl] |
[x84_64-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-gnu?job=build:release:cargo:x86_64-unknown-linux-gnu
[x84_64-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl [x84_64-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl
[armv7-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-gnueabihf?job=build:release:cargo:armv7-unknown-linux-gnueabihf [armv6-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-arm-unknown-linux-musleabihf?job=build:release:cargo:arm-unknown-linux-musleabihf
[armv8-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-gnu?job=build:release:cargo:aarch64-unknown-linux-gnu [armv7-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-musleabihf?job=build:release:cargo:armv7-unknown-linux-musleabihf
[armv8-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-musl?job=build:release:cargo:aarch64-unknown-linux-musl
```bash ```bash
$ sudo wget -O /usr/local/bin/matrix-conduit <url> $ sudo wget -O /usr/local/bin/matrix-conduit <url>
@ -32,15 +37,15 @@ Alternatively, you may compile the binary yourself using
```bash ```bash
$ cargo build --release $ cargo build --release
``` ```
Note that this currently requires Rust 1.50. Note that this currently requires Rust 1.50.
If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](CROSS_COMPILE.md). If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](CROSS_COMPILE.md).
## Adding a Conduit user ## Adding a Conduit user
While Conduit can run as any user it is usually better to use dedicated users for different services. While Conduit can run as any user it is usually better to use dedicated users for different services. This also allows
This also allows you to make sure that the file permissions are correctly set up. you to make sure that the file permissions are correctly set up.
In Debian you can use this command to create a Conduit user: In Debian you can use this command to create a Conduit user:
@ -50,9 +55,8 @@ sudo adduser --system conduit --no-create-home
## Setting up a systemd service ## Setting up a systemd service
Now we'll set up a systemd service for Conduit, so it's easy to start/stop Now we'll set up a systemd service for Conduit, so it's easy to start/stop Conduit and set it to autostart when your
Conduit and set it to autostart when your server reboots. Simply paste the server reboots. Simply paste the default systemd service you can find below into
default systemd service you can find below into
`/etc/systemd/system/conduit.service`. `/etc/systemd/system/conduit.service`.
```systemd ```systemd
@ -77,10 +81,10 @@ Finally, run
$ sudo systemctl daemon-reload $ sudo systemctl daemon-reload
``` ```
## Creating the Conduit configuration file ## Creating the Conduit configuration file
Now we need to create the Conduit's config file in `/etc/matrix-conduit/conduit.toml`. Paste this in **and take a moment to read it. You need to change at least the server name.** Now we need to create the Conduit's config file in `/etc/matrix-conduit/conduit.toml`. Paste this in **and take a moment
to read it. You need to change at least the server name.**
```toml ```toml
[global] [global]
@ -128,8 +132,8 @@ address = "127.0.0.1" # This makes sure Conduit can only be reached using the re
## Setting the correct file permissions ## Setting the correct file permissions
As we are using a Conduit specific user we need to allow it to read the config. As we are using a Conduit specific user we need to allow it to read the config. To do that you can run this command on
To do that you can run this command on Debian: Debian:
```bash ```bash
sudo chown -R conduit:nogroup /etc/matrix-conduit sudo chown -R conduit:nogroup /etc/matrix-conduit
@ -142,7 +146,6 @@ sudo mkdir -p /var/lib/matrix-conduit/conduit_db
sudo chown -R conduit:nogroup /var/lib/matrix-conduit/conduit_db sudo chown -R conduit:nogroup /var/lib/matrix-conduit/conduit_db
``` ```
## Setting up the Reverse Proxy ## Setting up the Reverse Proxy
This depends on whether you use Apache, Nginx or another web server. This depends on whether you use Apache, Nginx or another web server.
@ -171,11 +174,9 @@ ProxyPassReverse /_matrix/ http://127.0.0.1:6167/_matrix/
$ sudo systemctl reload apache2 $ sudo systemctl reload apache2
``` ```
### Nginx ### Nginx
If you use Nginx and not Apache, add the following server section inside the If you use Nginx and not Apache, add the following server section inside the http section of `/etc/nginx/nginx.conf`
http section of `/etc/nginx/nginx.conf`
```nginx ```nginx
server { server {
@ -198,13 +199,13 @@ server {
include /etc/letsencrypt/options-ssl-nginx.conf; include /etc/letsencrypt/options-ssl-nginx.conf;
} }
``` ```
**You need to make some edits again.** When you are done, run **You need to make some edits again.** When you are done, run
```bash ```bash
$ sudo systemctl reload nginx $ sudo systemctl reload nginx
``` ```
## SSL Certificate ## SSL Certificate
The easiest way to get an SSL certificate, if you don't have one already, is to install `certbot` and run this: The easiest way to get an SSL certificate, if you don't have one already, is to install `certbot` and run this:
@ -213,7 +214,6 @@ The easiest way to get an SSL certificate, if you don't have one already, is to
$ sudo certbot -d your.server.name $ sudo certbot -d your.server.name
``` ```
## You're done! ## You're done!
Now you can start Conduit with: Now you can start Conduit with:

137
Dockerfile
View file

@ -1,75 +1,66 @@
# Using multistage build: # syntax=docker/dockerfile:1
# https://docs.docker.com/develop/develop-images/multistage-build/ FROM docker.io/rust:1.53-alpine AS builder
# https://whitfin.io/speeding-up-rust-docker-builds/ WORKDIR /usr/src/conduit
# Install required packages to build Conduit and it's dependencies
RUN apk add musl-dev
# == Build dependencies without our own code separately for caching ==
#
# Need a fake main.rs since Cargo refuses to build anything otherwise.
#
# See https://github.com/rust-lang/cargo/issues/2644 for a Cargo feature
# request that would allow just dependencies to be compiled, presumably
# regardless of whether source files are available.
RUN mkdir src && touch src/lib.rs && echo 'fn main() {}' > src/main.rs
COPY Cargo.toml Cargo.lock ./
RUN cargo build --release && rm -r src
# Copy over actual Conduit sources
COPY src src
# main.rs and lib.rs need their timestamp updated for this to work correctly since
# otherwise the build with the fake main.rs from above is newer than the
# source files (COPY preserves timestamps).
#
# Builds conduit and places the binary at /usr/src/conduit/target/release/conduit
RUN touch src/main.rs && touch src/lib.rs && cargo build --release
########################## BUILD IMAGE ##########################
# Alpine build image to build Conduit's statically compiled binary
FROM alpine:3.14 as builder
# Install packages needed for building all crates
RUN apk add --no-cache \
cargo \
openssl-dev
# Specifies if the local project is build or if Conduit gets build # ---------------------------------------------------------------------------------------------------------------
# from the official git repository. Defaults to the git repo. # Stuff below this line actually ends up in the resulting docker image
ARG LOCAL=false # ---------------------------------------------------------------------------------------------------------------
# Specifies which revision/commit is build. Defaults to HEAD FROM docker.io/alpine:3.14 AS runner
ARG GIT_REF=origin/master
# Copy project files from current folder # Standard port on which Conduit launches.
COPY . . # You still need to map the port when using the docker command or docker-compose.
# Build it from the copied local files or from the official git repository
RUN if [[ $LOCAL == "true" ]]; then \
mv ./docker/healthcheck.sh . ; \
echo "Building from local source..." ; \
cargo install --path . ; \
else \
echo "Building revision '${GIT_REF}' from online source..." ; \
cargo install --git "https://gitlab.com/famedly/conduit.git" --rev ${GIT_REF} ; \
echo "Loadings healthcheck script from online source..." ; \
wget "https://gitlab.com/famedly/conduit/-/raw/${GIT_REF#origin/}/docker/healthcheck.sh" ; \
fi
########################## RUNTIME IMAGE ##########################
# Create new stage with a minimal image for the actual
# runtime image/container
FROM alpine:3.14
ARG CREATED
ARG VERSION
ARG GIT_REF=origin/master
ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"
# Labels according to https://github.com/opencontainers/image-spec/blob/master/annotations.md
# including a custom label specifying the build command
LABEL org.opencontainers.image.created=${CREATED} \
org.opencontainers.image.authors="Conduit Contributors" \
org.opencontainers.image.title="Conduit" \
org.opencontainers.image.version=${VERSION} \
org.opencontainers.image.vendor="Conduit Contributors" \
org.opencontainers.image.description="A Matrix homeserver written in Rust" \
org.opencontainers.image.url="https://conduit.rs/" \
org.opencontainers.image.revision=${GIT_REF} \
org.opencontainers.image.source="https://gitlab.com/famedly/conduit.git" \
org.opencontainers.image.licenses="Apache-2.0" \
org.opencontainers.image.documentation="" \
org.opencontainers.image.ref.name="" \
org.label-schema.docker.build="docker build . -t matrixconduit/matrix-conduit:latest --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)" \
maintainer="Weasy666"
# Standard port on which Conduit launches. You still need to map the port when using the docker command or docker-compose.
EXPOSE 6167 EXPOSE 6167
# Copy config files from context and the binary from # Note from @jfowl: I would like to remove this in the future and just have the Docker version be configured with envs.
# the "builder" stage to the current stage into folder ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"
# /srv/conduit and create data folder for database
RUN mkdir -p /srv/conduit/.local/share/conduit
COPY --from=builder /root/.cargo/bin/conduit /srv/conduit/
COPY --from=builder ./healthcheck.sh /srv/conduit/
# Conduit needs:
# ca-certificates: for https
# libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big.
RUN apk add --no-cache \
ca-certificates \
curl \
libgcc
# Created directory for the database and media files
RUN mkdir -p /srv/conduit/.local/share/conduit
# Test if Conduit is still alive, uses the same endpoint as Element
COPY ./docker/healthcheck.sh /srv/conduit/
HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh
# Copy over the actual Conduit binary from the builder stage
COPY --from=builder /usr/src/conduit/target/release/conduit /srv/conduit/
# Improve security: Don't run stuff as root, that does not need to run as root:
# Add www-data user and group with UID 82, as used by alpine # Add www-data user and group with UID 82, as used by alpine
# https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install # https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install
RUN set -x ; \ RUN set -x ; \
@ -79,19 +70,13 @@ RUN set -x ; \
# Change ownership of Conduit files to www-data user and group # Change ownership of Conduit files to www-data user and group
RUN chown -cR www-data:www-data /srv/conduit RUN chown -cR www-data:www-data /srv/conduit
RUN chmod +x /srv/conduit/healthcheck.sh
# Install packages needed to run Conduit # Change user to www-data
RUN apk add --no-cache \
ca-certificates \
curl \
libgcc
# Test if Conduit is still alive, uses the same endpoint as Element
HEALTHCHECK --start-period=5s --interval=60s CMD ./healthcheck.sh
# Set user to www-data
USER www-data USER www-data
# Set container home directory # Set container home directory
WORKDIR /srv/conduit WORKDIR /srv/conduit
# Run Conduit
ENTRYPOINT [ "/srv/conduit/conduit" ] # Run Conduit and print backtraces on panics
ENV RUST_BACKTRACE=1
ENTRYPOINT [ "/srv/conduit/conduit" ]

105
docker/README.md
View file

@ -2,53 +2,41 @@
> **Note:** To run and use Conduit you should probably use it with a Domain or Subdomain behind a reverse proxy (like Nginx, Traefik, Apache, ...) with a Lets Encrypt certificate. > **Note:** To run and use Conduit you should probably use it with a Domain or Subdomain behind a reverse proxy (like Nginx, Traefik, Apache, ...) with a Lets Encrypt certificate.
## Docker ## Docker
### Build & Dockerfile ### Build & Dockerfile
The Dockerfile provided by Conduit has two stages, each of which creates an image. The Dockerfile provided by Conduit has two stages, each of which creates an image.
1. **Builder:** Builds the binary from local context or by cloning a git revision from the official repository. 1. **Builder:** Builds the binary from local context or by cloning a git revision from the official repository.
2. **Runtime:** Copies the built binary from **Builder** and sets up the runtime environment, like creating a volume to persist the database and applying the correct permissions. 2. **Runner:** Copies the built binary from **Builder** and sets up the runtime environment, like creating a volume to persist the database and applying the correct permissions.
The Dockerfile includes a few build arguments that should be supplied when building it.
``` Dockerfile
ARG LOCAL=false
ARG CREATED
ARG VERSION
ARG GIT_REF=origin/master
```
- **CREATED:** Date and time as string (date-time as defined by RFC 3339). Will be used to create the Open Container Initiative compliant label `org.opencontainers.image.created`. Supply by it like this `$(date -u +'%Y-%m-%dT%H:%M:%SZ')`
- **VERSION:** The SemVer version of Conduit, which is in the image. Will be used to create the Open Container Initiative compliant label `org.opencontainers.image.version`. If you have a `Cargo.toml` in your build context, you can get it with `$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)`
- **LOCAL:** *(Optional)* A boolean value, specifies if the local build context should be used, or if the official repository will be cloned. If not supplied with the build command, it will default to `false`.
- **GIT_REF:** *(Optional)* A git ref, like `HEAD` or a commit ID. The supplied ref will be used to create the Open Container Initiative compliant label `org.opencontainers.image.revision` and will be the ref that is cloned from the repository when not building from the local context. If not supplied with the build command, it will default to `origin/master`.
To build the image you can use the following command To build the image you can use the following command
``` bash ```bash
docker build . -t matrixconduit/matrix-conduit:latest --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) docker build --tag matrixconduit/matrix-conduit:latest .
``` ```
which also will tag the resulting image as `matrixconduit/matrix-conduit:latest`. which also will tag the resulting image as `matrixconduit/matrix-conduit:latest`.
**Note:** it ommits the two optional `build-arg`s.
### Run ### Run
After building the image you can simply run it with After building the image you can simply run it with
``` bash ```bash
docker run -d -p 8448:6167 -v ~/conduit.toml:/srv/conduit/conduit.toml -v db:/srv/conduit/.local/share/conduit matrixconduit/matrix-conduit:latest docker run -d -p 8448:6167 -v ~/conduit.toml:/srv/conduit/conduit.toml -v db:/srv/conduit/.local/share/conduit matrixconduit/matrix-conduit:latest
``` ```
or you can skip the build step and pull the image from one of the following registries: or you can skip the build step and pull the image from one of the following registries:
| Registry | Image | Size | | Registry | Image | Size |
| --------------- | ------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | | --------------- | --------------------------------------------------------------- | --------------------- |
| Docker Hub | [matrixconduit/matrix-conduit:latest](https://hub.docker.com/r/matrixconduit/matrix-conduit) | ![Image Size](https://img.shields.io/docker/image-size/matrixconduit/matrix-conduit/latest) | | Docker Hub | [matrixconduit/matrix-conduit:latest][dh] | ![Image Size][shield] |
| GitLab Registry | [registry.gitlab.com/famedly/conduit/conduit:latest](https://gitlab.com/famedly/conduit/container_registry/2134341) | ![Image Size](https://img.shields.io/docker/image-size/matrixconduit/matrix-conduit/latest) | | GitLab Registry | [registry.gitlab.com/famedly/conduit/matrix-conduit:latest][gl] | ![Image Size][shield] |
[dh]: https://hub.docker.com/r/matrixconduit/matrix-conduit
[gl]: https://gitlab.com/famedly/conduit/container_registry/
[shield]: https://img.shields.io/docker/image-size/matrixconduit/matrix-conduit/latest
The `-d` flag lets the container run in detached mode. You now need to supply a `conduit.toml` config file, an example can be found [here](../conduit-example.toml). The `-d` flag lets the container run in detached mode. You now need to supply a `conduit.toml` config file, an example can be found [here](../conduit-example.toml).
You can pass in different env vars to change config values on the fly. You can even configure Conduit completely by using env vars, but for that you need You can pass in different env vars to change config values on the fly. You can even configure Conduit completely by using env vars, but for that you need
@ -56,29 +44,26 @@ to pass `-e CONDUIT_CONFIG=""` into your container. For an overview of possible
If you just want to test Conduit for a short time, you can use the `--rm` flag, which will clean up everything related to your container after you stop it. If you just want to test Conduit for a short time, you can use the `--rm` flag, which will clean up everything related to your container after you stop it.
## Docker-compose ## Docker-compose
If the docker command is not for you or your setup, you can also use one of the provided `docker-compose` files. Depending on your proxy setup, use the [`docker-compose.traefik.yml`](docker-compose.traefik.yml) and [`docker-compose.override.traefik.yml`](docker-compose.override.traefik.yml) for Traefik (don't forget to remove `.traefik` from the filenames) or the normal [`docker-compose.yml`](../docker-compose.yml) for every other reverse proxy. Additional info about deploying If the docker command is not for you or your setup, you can also use one of the provided `docker-compose` files. Depending on your proxy setup, use the [`docker-compose.traefik.yml`](docker-compose.traefik.yml) and [`docker-compose.override.traefik.yml`](docker-compose.override.traefik.yml) for Traefik (don't forget to remove `.traefik` from the filenames) or the normal [`docker-compose.yml`](../docker-compose.yml) for every other reverse proxy. Additional info about deploying
Conduit can be found [here](../DEPLOY.md). Conduit can be found [here](../DEPLOY.md).
### Build ### Build
To build the Conduit image with docker-compose, you first need to open and modify the `docker-compose.yml` file. There you need to comment the `image:` option and uncomment the `build:` option. Then call docker-compose with: To build the Conduit image with docker-compose, you first need to open and modify the `docker-compose.yml` file. There you need to comment the `image:` option and uncomment the `build:` option. Then call docker-compose with:
``` bash ```bash
CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml) docker-compose up docker-compose up
``` ```
This will also start the container right afterwards, so if want it to run in detached mode, you also should use the `-d` flag. For possible `build-args`, please take a look at the above `Build & Dockerfile` section. This will also start the container right afterwards, so if want it to run in detached mode, you also should use the `-d` flag.
### Run ### Run
If you already have built the image or want to use one from the registries, you can just start the container and everything else in the compose file in detached mode with: If you already have built the image or want to use one from the registries, you can just start the container and everything else in the compose file in detached mode with:
``` bash ```bash
docker-compose up -d docker-compose up -d
``` ```
@ -101,32 +86,36 @@ So...step by step:
3. Create the `conduit.toml` config file, an example can be found [here](../conduit-example.toml), or set `CONDUIT_CONFIG=""` and configure Conduit per env vars. 3. Create the `conduit.toml` config file, an example can be found [here](../conduit-example.toml), or set `CONDUIT_CONFIG=""` and configure Conduit per env vars.
4. Uncomment the `element-web` service if you want to host your own Element Web Client and create a `element_config.json`. 4. Uncomment the `element-web` service if you want to host your own Element Web Client and create a `element_config.json`.
5. Create the files needed by the `well-known` service. 5. Create the files needed by the `well-known` service.
- `./nginx/matrix.conf` (relative to the compose file, you can change this, but then also need to change the volume mapping)
```nginx
server {
server_name <SUBDOMAIN>.<DOMAIN>;
listen 80 default_server;
location /.well-known/matrix/ { - `./nginx/matrix.conf` (relative to the compose file, you can change this, but then also need to change the volume mapping)
root /var/www;
default_type application/json; ```nginx
add_header Access-Control-Allow-Origin *; server {
} server_name <SUBDOMAIN>.<DOMAIN>;
} listen 80 default_server;
```
- `./nginx/www/.well-known/matrix/client` (relative to the compose file, you can change this, but then also need to change the volume mapping) location /.well-known/matrix/ {
```json root /var/www;
{ default_type application/json;
"m.homeserver": { add_header Access-Control-Allow-Origin *;
"base_url": "https://<SUBDOMAIN>.<DOMAIN>" }
} }
} ```
```
- `./nginx/www/.well-known/matrix/server` (relative to the compose file, you can change this, but then also need to change the volume mapping) - `./nginx/www/.well-known/matrix/client` (relative to the compose file, you can change this, but then also need to change the volume mapping)
```json ```json
{ {
"m.server": "<SUBDOMAIN>.<DOMAIN>:443" "m.homeserver": {
} "base_url": "https://<SUBDOMAIN>.<DOMAIN>"
``` }
}
```
- `./nginx/www/.well-known/matrix/server` (relative to the compose file, you can change this, but then also need to change the volume mapping)
```json
{
"m.server": "<SUBDOMAIN>.<DOMAIN>:443"
}
```
6. Run `docker-compose up -d` 6. Run `docker-compose up -d`
7. Connect to your homeserver with your preferred client and create a user. You should do this immediatly after starting Conduit, because the first created user is the admin. 7. Connect to your homeserver with your preferred client and create a user. You should do this immediatly after starting Conduit, because the first created user is the admin.

48
docker/ci-binaries-packaging.Dockerfile
View file

@ -1,3 +1,4 @@
# syntax=docker/dockerfile:1
# --------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------
# This Dockerfile is intended to be built as part of Conduit's CI pipeline. # This Dockerfile is intended to be built as part of Conduit's CI pipeline.
# It does not build Conduit in Docker, but just copies the matching build artifact from the build job. # It does not build Conduit in Docker, but just copies the matching build artifact from the build job.
@ -7,20 +8,26 @@
# Credit's for the original Dockerfile: Weasy666. # Credit's for the original Dockerfile: Weasy666.
# --------------------------------------------------------------------------------------------------------- # ---------------------------------------------------------------------------------------------------------
FROM alpine:3.14 FROM docker.io/alpine:3.14 AS runner
# Install packages needed to run Conduit # Standard port on which Conduit launches.
# You still need to map the port when using the docker command or docker-compose.
EXPOSE 6167
# Note from @jfowl: I would like to remove this in the future and just have the Docker version be configured with envs.
ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"
# Conduit needs:
# ca-certificates: for https
# libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big.
RUN apk add --no-cache \ RUN apk add --no-cache \
ca-certificates \ ca-certificates \
curl \
libgcc libgcc
ARG CREATED ARG CREATED
ARG VERSION ARG VERSION
ARG GIT_REF ARG GIT_REF
ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"
# Labels according to https://github.com/opencontainers/image-spec/blob/master/annotations.md # Labels according to https://github.com/opencontainers/image-spec/blob/master/annotations.md
# including a custom label specifying the build command # including a custom label specifying the build command
LABEL org.opencontainers.image.created=${CREATED} \ LABEL org.opencontainers.image.created=${CREATED} \
@ -33,19 +40,24 @@ LABEL org.opencontainers.image.created=${CREATED} \
org.opencontainers.image.revision=${GIT_REF} \ org.opencontainers.image.revision=${GIT_REF} \
org.opencontainers.image.source="https://gitlab.com/famedly/conduit.git" \ org.opencontainers.image.source="https://gitlab.com/famedly/conduit.git" \
org.opencontainers.image.licenses="Apache-2.0" \ org.opencontainers.image.licenses="Apache-2.0" \
org.opencontainers.image.documentation="" \ org.opencontainers.image.documentation="https://gitlab.com/famedly/conduit" \
org.opencontainers.image.ref.name="" org.opencontainers.image.ref.name=""
# Standard port on which Conduit launches. You still need to map the port when using the docker command or docker-compose. # Created directory for the database and media files
EXPOSE 6167
# create data folder for database
RUN mkdir -p /srv/conduit/.local/share/conduit RUN mkdir -p /srv/conduit/.local/share/conduit
# Copy the Conduit binary into the image at the latest possible moment to maximise caching: # Test if Conduit is still alive, uses the same endpoint as Element
COPY ./conduit-x86_64-unknown-linux-musl /srv/conduit/conduit
COPY ./docker/healthcheck.sh /srv/conduit/ COPY ./docker/healthcheck.sh /srv/conduit/
HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh
# Depending on the target platform (e.g. "linux/arm/v7", "linux/arm64/v8", or "linux/amd64")
# copy the matching binary into this docker image
ARG TARGETPLATFORM
COPY ./$TARGETPLATFORM /srv/conduit/conduit
# Improve security: Don't run stuff as root, that does not need to run as root:
# Add www-data user and group with UID 82, as used by alpine # Add www-data user and group with UID 82, as used by alpine
# https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install # https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install
RUN set -x ; \ RUN set -x ; \
@ -57,13 +69,11 @@ RUN set -x ; \
RUN chown -cR www-data:www-data /srv/conduit RUN chown -cR www-data:www-data /srv/conduit
RUN chmod +x /srv/conduit/healthcheck.sh RUN chmod +x /srv/conduit/healthcheck.sh
# Change user to www-data
# Test if Conduit is still alive, uses the same endpoint as Element
HEALTHCHECK --start-period=5s --interval=60s CMD ./healthcheck.sh
# Set user to www-data
USER www-data USER www-data
# Set container home directory # Set container home directory
WORKDIR /srv/conduit WORKDIR /srv/conduit
# Run Conduit
# Run Conduit and print backtraces on panics
ENV RUST_BACKTRACE=1
ENTRYPOINT [ "/srv/conduit/conduit" ] ENTRYPOINT [ "/srv/conduit/conduit" ]

6
docker/healthcheck.sh
View file

@ -7,7 +7,7 @@ fi
# The actual health check. # The actual health check.
# We try to first get a response on HTTP and when that fails on HTTPS and when that fails, we exit with code 1. # We try to first get a response on HTTP and when that fails on HTTPS and when that fails, we exit with code 1.
# TODO: Change this to a single curl call. Do we have a config value that we can check for that? # TODO: Change this to a single wget call. Do we have a config value that we can check for that?
curl --fail -s "http://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \ wget --no-verbose --tries=1 --spider "http://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \
curl -k --fail -s "https://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \ wget --no-verbose --tries=1 --spider "https://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \
exit 1 exit 1