Merge remote-tracking branch 'origin/next' into command-refactor

Fixed conflict with commit 78502aa6b1

.gitlab-ci.yml

@@ -8,6 +8,10 @@ variables:
   GIT_SUBMODULE_STRATEGY: recursive
   FF_USE_FASTZIP: 1
   CACHE_COMPRESSION_LEVEL: fastest
+  # Docker in Docker
+  DOCKER_HOST: tcp://docker:2375/
+  DOCKER_TLS_CERTDIR: ""
+  DOCKER_DRIVER: overlay2
 
 # --------------------------------------------------------------------- #
 #  Cargo: Compiling for different architectures                         #
@@ -20,28 +24,45 @@ variables:
     - if: '$CI_COMMIT_BRANCH == "master"'
     - if: '$CI_COMMIT_BRANCH == "next"'
     - if: "$CI_COMMIT_TAG"
+    - if: '($CI_MERGE_REQUEST_APPROVED == "true") || $BUILD_EVERYTHING' # Once MR is approved, test all builds. Or if BUILD_EVERYTHING is set.
   interruptible: true
-  image: "rust:1.56"
+  image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest"
   tags: ["docker"]
+  services: ["docker:dind"]
   variables:
+    SHARED_PATH: $CI_PROJECT_DIR/shared
     CARGO_PROFILE_RELEASE_LTO: "true"
     CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1"
     CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
   before_script:
     - 'echo "Building for target $TARGET"'
-    - "rustc --version && cargo --version && rustup show" # Print version info for debugging
+    - "rustup show && rustc --version && cargo --version" # Print version info for debugging
-    - "rustup target add $TARGET"
+    # fix cargo and rustup mounts from this container (https://gitlab.com/gitlab-org/gitlab-foss/-/issues/41227)
-    # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
+    - "mkdir -p $SHARED_PATH/cargo"
-    - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi
+    - "cp -r $CARGO_HOME/bin $SHARED_PATH/cargo"
+    - "cp -r $RUSTUP_HOME $SHARED_PATH"
+    - "export CARGO_HOME=$SHARED_PATH/cargo RUSTUP_HOME=$SHARED_PATH/rustup"
+    # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results.
+    - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/sccache; fi
   script:
-    - time cargo build --target $TARGET --release
+    # cross-compile conduit for target
-    - 'cp "target/$TARGET/release/conduit" "conduit-$TARGET"'
+    - 'time cross build --target="$TARGET" --locked --release'
+    - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'
+    # print information about linking for debugging
+    - "file conduit-$TARGET" # print file information
+    - 'readelf --dynamic conduit-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked
+  cache:
+    # https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
+    key: "cargo-cache-$TARGET"
+    paths:
+      - $SHARED_PATH/cargo/registry/index
+      - $SHARED_PATH/cargo/registry/cache
+      - $SHARED_PATH/cargo/git/db
   artifacts:
     expire_in: never
 
 build:release:cargo:x86_64-unknown-linux-musl-with-debug:
   extends: .build-cargo-shared-settings
-  image: messense/rust-musl-cross:x86_64-musl
   variables:
     CARGO_PROFILE_RELEASE_DEBUG: 2 # Enable debug info for flamegraph profiling
     TARGET: "x86_64-unknown-linux-musl"
@@ -55,7 +76,6 @@ build:release:cargo:x86_64-unknown-linux-musl-with-debug:
 
 build:release:cargo:x86_64-unknown-linux-musl:
   extends: .build-cargo-shared-settings
-  image: messense/rust-musl-cross:x86_64-musl
   variables:
     TARGET: "x86_64-unknown-linux-musl"
   artifacts:
@@ -66,7 +86,6 @@ build:release:cargo:x86_64-unknown-linux-musl:
 
 build:release:cargo:arm-unknown-linux-musleabihf:
   extends: .build-cargo-shared-settings
-  image: messense/rust-musl-cross:arm-musleabihf
   variables:
     TARGET: "arm-unknown-linux-musleabihf"
   artifacts:
@@ -77,7 +96,6 @@ build:release:cargo:arm-unknown-linux-musleabihf:
 
 build:release:cargo:armv7-unknown-linux-musleabihf:
   extends: .build-cargo-shared-settings
-  image: messense/rust-musl-cross:armv7-musleabihf
   variables:
     TARGET: "armv7-unknown-linux-musleabihf"
   artifacts:
@@ -88,7 +106,6 @@ build:release:cargo:armv7-unknown-linux-musleabihf:
 
 build:release:cargo:aarch64-unknown-linux-musl:
   extends: .build-cargo-shared-settings
-  image: messense/rust-musl-cross:aarch64-musl
   variables:
     TARGET: "aarch64-unknown-linux-musl"
   artifacts:
@@ -104,14 +121,17 @@ build:release:cargo:aarch64-unknown-linux-musl:
   cache:
     key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--debug"
   script:
-    - "time cargo build --target $TARGET"
+    # cross-compile conduit for target
+    - 'time time cross build --target="$TARGET" --locked'
     - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'
+    # print information about linking for debugging
+    - "file conduit-debug-$TARGET" # print file information
+    - 'readelf --dynamic conduit-debug-$TARGET | sed -e "/NEEDED/q1"' # ensure statically linked
   artifacts:
     expire_in: 4 weeks
 
 build:debug:cargo:x86_64-unknown-linux-musl:
   extends: ".cargo-debug-shared-settings"
-  image: messense/rust-musl-cross:x86_64-musl
   variables:
     TARGET: "x86_64-unknown-linux-musl"
   artifacts:
@@ -136,9 +156,6 @@ build:debug:cargo:x86_64-unknown-linux-musl:
     - "build:release:cargo:armv7-unknown-linux-musleabihf"
     - "build:release:cargo:aarch64-unknown-linux-musl"
   variables:
-    DOCKER_HOST: tcp://docker:2375/
-    DOCKER_TLS_CERTDIR: ""
-    DOCKER_DRIVER: overlay2
     PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64"
     DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile"
   cache:
@@ -210,24 +227,20 @@ docker:master:dockerhub:
 test:cargo:
   stage: "test"
   needs: []
-  image: "rust:latest"
+  image: "registry.gitlab.com/jfowl/conduit-containers/rust-with-tools:latest"
   tags: ["docker"]
   variables:
     CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
   interruptible: true
   before_script:
-    #    - mkdir -p $CARGO_HOME
-    - apt-get update -yqq
-    - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config libclang-dev
     - rustup component add clippy rustfmt
-    - curl "https://faulty-storage.de/gitlab-report" --output ./gitlab-report && chmod +x ./gitlab-report
     # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
-    - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi
+    - if [ -n "${SCCACHE_ENDPOINT}" ]; then export RUSTC_WRAPPER=/usr/local/cargo/bin/sccache; fi
   script:
     - rustc --version && cargo --version # Print version info for debugging
     - cargo fmt --all -- --check
-    - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | ./gitlab-report -p test > $CI_PROJECT_DIR/report.xml"
+    - "cargo test --color always --workspace --verbose --locked --no-fail-fast -- -Z unstable-options --format json | gitlab-report -p test > $CI_PROJECT_DIR/report.xml"
-    - "cargo clippy --color always --verbose --message-format=json | ./gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json"
+    - "cargo clippy --color always --verbose --message-format=json | gitlab-report -p clippy > $CI_PROJECT_DIR/gl-code-quality-report.json"
   artifacts:
     when: always
     reports:

CROSS_COMPILE.md

@@ -1,11 +0,0 @@
-Install docker:
-
-```
-$ sudo apt install docker
-$ sudo usermod -aG docker $USER
-$ exec sudo su -l $USER
-$ sudo systemctl start docker
-$ cargo install cross
-$ cross build --release --target armv7-unknown-linux-musleabihf
-```
-The cross-compiled binary is at target/armv7-unknown-linux-musleabihf/release/conduit

Cargo.lock

@@ -2174,7 +2174,7 @@ dependencies = [
 [[package]]
 name = "ruma"
 version = "0.4.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "assign",
  "js_int",
@@ -2195,7 +2195,7 @@ dependencies = [
 [[package]]
 name = "ruma-api"
 version = "0.18.5"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "bytes",
  "http",
@@ -2211,7 +2211,7 @@ dependencies = [
 [[package]]
 name = "ruma-api-macros"
 version = "0.18.5"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "proc-macro-crate",
  "proc-macro2",
@@ -2222,7 +2222,7 @@ dependencies = [
 [[package]]
 name = "ruma-appservice-api"
 version = "0.4.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "ruma-api",
  "ruma-common",
@@ -2236,7 +2236,7 @@ dependencies = [
 [[package]]
 name = "ruma-client-api"
 version = "0.12.3"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "assign",
  "bytes",
@@ -2256,7 +2256,7 @@ dependencies = [
 [[package]]
 name = "ruma-common"
 version = "0.6.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "indexmap",
  "js_int",
@@ -2271,7 +2271,7 @@ dependencies = [
 [[package]]
 name = "ruma-events"
 version = "0.24.6"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "indoc",
  "js_int",
@@ -2288,7 +2288,7 @@ dependencies = [
 [[package]]
 name = "ruma-events-macros"
 version = "0.24.6"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "proc-macro-crate",
  "proc-macro2",
@@ -2299,7 +2299,7 @@ dependencies = [
 [[package]]
 name = "ruma-federation-api"
 version = "0.3.1"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "js_int",
  "ruma-api",
@@ -2314,7 +2314,7 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers"
 version = "0.20.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "percent-encoding",
  "rand 0.8.4",
@@ -2329,7 +2329,7 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers-macros"
 version = "0.20.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "quote",
  "ruma-identifiers-validation",
@@ -2339,7 +2339,7 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers-validation"
 version = "0.5.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "thiserror",
 ]
@@ -2347,7 +2347,7 @@ dependencies = [
 [[package]]
 name = "ruma-identity-service-api"
 version = "0.3.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "js_int",
  "ruma-api",
@@ -2360,7 +2360,7 @@ dependencies = [
 [[package]]
 name = "ruma-push-gateway-api"
 version = "0.3.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "js_int",
  "ruma-api",
@@ -2375,7 +2375,7 @@ dependencies = [
 [[package]]
 name = "ruma-serde"
 version = "0.5.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "base64 0.13.0",
  "bytes",
@@ -2390,7 +2390,7 @@ dependencies = [
 [[package]]
 name = "ruma-serde-macros"
 version = "0.5.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "proc-macro-crate",
  "proc-macro2",
@@ -2401,7 +2401,7 @@ dependencies = [
 [[package]]
 name = "ruma-signatures"
 version = "0.9.0"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "base64 0.13.0",
  "ed25519-dalek",
@@ -2418,7 +2418,7 @@ dependencies = [
 [[package]]
 name = "ruma-state-res"
 version = "0.4.1"
-source = "git+https://github.com/ruma/ruma?rev=08d60b3d376b63462f769d4b9bd3bbfb560d501a#08d60b3d376b63462f769d4b9bd3bbfb560d501a"
+source = "git+https://github.com/ruma/ruma?rev=82becb86c837570224964425929d1b5305784435#82becb86c837570224964425929d1b5305784435"
 dependencies = [
  "itertools",
  "js_int",

Cargo.toml

@@ -20,7 +20,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request
 
 # Used for matrix spec type definitions and helpers
 #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
-ruma = { git = "https://github.com/ruma/ruma", rev = "08d60b3d376b63462f769d4b9bd3bbfb560d501a", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+ruma = { git = "https://github.com/ruma/ruma", rev = "82becb86c837570224964425929d1b5305784435", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
 #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
 #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
 

Cross.toml

@@ -0,0 +1,23 @@
+[build.env]
+# CI uses an S3 endpoint to store sccache artifacts, so their config needs to
+# be available in the cross container as well
+passthrough = [
+    "RUSTC_WRAPPER",
+    "AWS_ACCESS_KEY_ID",
+    "AWS_SECRET_ACCESS_KEY",
+    "SCCACHE_BUCKET",
+    "SCCACHE_ENDPOINT",
+    "SCCACHE_S3_USE_SSL",
+]
+
+[target.aarch64-unknown-linux-musl]
+image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-aarch64-unknown-linux-musl:latest"
+
+[target.arm-unknown-linux-musleabihf]
+image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-arm-unknown-linux-musleabihf:latest"
+
+[target.armv7-unknown-linux-musleabihf]
+image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-armv7-unknown-linux-musleabihf:latest"
+
+[target.x86_64-unknown-linux-musl]
+image = "registry.gitlab.com/jfowl/conduit-containers/rust-cross-x86_64-unknown-linux-musl:latest"

DEPLOY.md

@@ -29,7 +29,11 @@ $ sudo wget -O /usr/local/bin/matrix-conduit <url>
 $ sudo chmod +x /usr/local/bin/matrix-conduit
 ```
 
-Alternatively, you may compile the binary yourself using
+Alternatively, you may compile the binary yourself
+
+```bash
+$ sudo apt install libclang-dev build-essential
+```
 
 ```bash
 $ cargo build --release
@@ -37,7 +41,7 @@ $ cargo build --release
 
 Note that this currently requires Rust 1.50.
 
-If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](CROSS_COMPILE.md).
+If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](cross/README.md).
 
 ## Adding a Conduit user
 

Dockerfile

@@ -1,9 +1,9 @@
 # syntax=docker/dockerfile:1
-FROM docker.io/rust:1.58-alpine AS builder
+FROM docker.io/rust:1.58-bullseye AS builder
 WORKDIR /usr/src/conduit
 
 # Install required packages to build Conduit and it's dependencies
-RUN apk add musl-dev
+RUN apt update && apt -y install libclang-dev
 
 # == Build dependencies without our own code separately for caching ==
 #
@@ -26,28 +26,27 @@ COPY src src
 # Builds conduit and places the binary at /usr/src/conduit/target/release/conduit
 RUN touch src/main.rs && touch src/lib.rs && cargo build --release
 
-
-
-
 # ---------------------------------------------------------------------------------------------------------------
 # Stuff below this line actually ends up in the resulting docker image
 # ---------------------------------------------------------------------------------------------------------------
-FROM docker.io/alpine:3.15.0 AS runner
+FROM docker.io/debian:bullseye-slim AS runner
 
 # Standard port on which Conduit launches.
 # You still need to map the port when using the docker command or docker-compose.
 EXPOSE 6167
 
-# Note from @jfowl: I would like to remove this in the future and just have the Docker version be configured with envs.
+# Note from @jfowl: I would like to remove the config file in the future and just have the Docker version be configured with envs.
-ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"
+ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" \
+    CONDUIT_PORT=6167
 
 # Conduit needs:
 #   ca-certificates: for https
-#   libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big.
+#   iproute2 & wget: for the healthcheck script
-RUN apk add --no-cache \
+RUN apt update && apt -y install \
     ca-certificates \
-    libgcc
+    iproute2 \
-
+    wget \
+    && rm -rf /var/lib/apt/lists/*
 
 # Created directory for the database and media files
 RUN mkdir -p /srv/conduit/.local/share/conduit
@@ -59,20 +58,20 @@ HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh
 # Copy over the actual Conduit binary from the builder stage
 COPY --from=builder /usr/src/conduit/target/release/conduit /srv/conduit/conduit
 
-# Improve security: Don't run stuff as root, that does not need to run as root:
+# Improve security: Don't run stuff as root, that does not need to run as root
-# Add www-data user and group with UID 82, as used by alpine
+# Add 'conduit' user and group (100:82). The UID:GID choice is to be compatible
-# https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install
+# with previous, Alpine-based containers, where the user and group were both
+# named 'www-data'.
 RUN set -x ; \
-    addgroup -Sg 82 www-data 2>/dev/null ; \
+    groupadd -r -g 82 conduit ; \
-    adduser -S -D -H -h /srv/conduit -G www-data -g www-data www-data 2>/dev/null ; \
+    useradd -r -M -d /srv/conduit -o -u 100 -g conduit conduit && exit 0 ; exit 1
-    addgroup www-data www-data 2>/dev/null && exit 0 ; exit 1
 
-# Change ownership of Conduit files to www-data user and group
+# Change ownership of Conduit files to conduit user and group and make the healthcheck executable:
-RUN chown -cR www-data:www-data /srv/conduit
+RUN chown -cR conduit:conduit /srv/conduit && \
-RUN chmod +x /srv/conduit/healthcheck.sh
+    chmod +x /srv/conduit/healthcheck.sh
 
-# Change user to www-data
+# Change user to conduit, no root permissions afterwards:
-USER www-data
+USER conduit
 # Set container home directory
 WORKDIR /srv/conduit
 

conduit-example.toml

@@ -22,6 +22,7 @@ database_backend = "rocksdb"
 # The port Conduit will be running on. You need to set up a reverse proxy in
 # your web server (e.g. apache or nginx), so all requests to /_matrix on port
 # 443 and 8448 will be forwarded to the Conduit instance running on this port
+# Docker users: Don't change this, you'll need to map an external port to this.
 port = 6167
 
 # Max size for uploads

cross/README.md

@@ -0,0 +1,37 @@
+## Cross compilation
+
+The `cross` folder contains a set of convenience scripts (`build.sh` and `test.sh`) for cross-compiling Conduit.
+
+Currently supported targets are
+
+- aarch64-unknown-linux-musl
+- arm-unknown-linux-musleabihf
+- armv7-unknown-linux-musleabihf
+- x86\_64-unknown-linux-musl
+
+### Install prerequisites
+#### Docker
+[Installation guide](https://docs.docker.com/get-docker/).
+```sh
+$ sudo apt install docker
+$ sudo systemctl start docker
+$ sudo usermod -aG docker $USER
+$ newgrp docker
+```
+
+#### Cross
+[Installation guide](https://github.com/rust-embedded/cross/#installation).
+```sh
+$ cargo install cross
+```
+
+### Buiding Conduit
+```sh
+$ TARGET=armv7-unknown-linux-musleabihf ./cross/build.sh --release
+```
+The cross-compiled binary is at `target/armv7-unknown-linux-musleabihf/release/conduit`
+
+### Testing Conduit
+```sh
+$ TARGET=armv7-unknown-linux-musleabihf ./cross/test.sh --release
+```

docker/ci-binaries-packaging.Dockerfile

@@ -9,19 +9,21 @@
 
 FROM docker.io/alpine:3.15.0 AS runner
 
+
 # Standard port on which Conduit launches.
 # You still need to map the port when using the docker command or docker-compose.
 EXPOSE 6167
 
-# Note from @jfowl: I would like to remove this in the future and just have the Docker version be configured with envs. 
+# Note from @jfowl: I would like to remove the config file in the future and just have the Docker version be configured with envs. 
-ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"
+ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml" \
+    CONDUIT_PORT=6167
 
 # Conduit needs:
 #   ca-certificates: for https
-#   libgcc: Apparently this is needed, even if I (@jfowl) don't know exactly why. But whatever, it's not that big.
+#   iproute2: for `ss` for the healthcheck script
 RUN apk add --no-cache \
     ca-certificates \
-    libgcc
+    iproute2
 
 
 ARG CREATED

docker/healthcheck.sh

@@ -1,8 +1,9 @@
 #!/bin/sh
 
-# If the port is not specified as env var, take it from the config file
+# If the config file does not contain a default port and the CONDUIT_PORT env is not set, create
-if [ -z ${CONDUIT_PORT} ]; then
+# try to get port from process list
-  CONDUIT_PORT=$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*')
+if [ -z "${CONDUIT_PORT}" ]; then
+  CONDUIT_PORT=$(ss -tlpn | grep conduit | grep -m1 -o ':[0-9]*' | grep -m1 -o '[0-9]*')
 fi
 
 # The actual health check.

src/client_server/account.rs

@@ -60,7 +60,8 @@ pub async fn get_register_available_route(
     body: Ruma<get_username_availability::Request<'_>>,
 ) -> ConduitResult<get_username_availability::Response> {
     // Validate user id
-    let user_id = UserId::parse_with_server_name(body.username.clone(), db.globals.server_name())
+    let user_id =
+        UserId::parse_with_server_name(body.username.to_lowercase(), db.globals.server_name())
             .ok()
             .filter(|user_id| {
                 !user_id.is_historical() && user_id.server_name() == db.globals.server_name()

src/client_server/media.rs

@@ -4,7 +4,10 @@ use crate::{
 };
 use ruma::api::client::{
     error::ErrorKind,
-    r0::media::{create_content, get_content, get_content_thumbnail, get_media_config},
+    r0::media::{
+        create_content, get_content, get_content_as_filename, get_content_thumbnail,
+        get_media_config,
+    },
 };
 use std::convert::TryInto;
 
@@ -71,7 +74,39 @@ pub async fn create_content_route(
     .into())
 }
 
-/// # `POST /_matrix/media/r0/download/{serverName}/{mediaId}`
+pub async fn get_remote_content(
+    db: &DatabaseGuard,
+    mxc: &str,
+    server_name: &ruma::ServerName,
+    media_id: &str,
+) -> Result<get_content::Response, Error> {
+    let content_response = db
+        .sending
+        .send_federation_request(
+            &db.globals,
+            server_name,
+            get_content::Request {
+                allow_remote: false,
+                server_name,
+                media_id,
+            },
+        )
+        .await?;
+
+    db.media
+        .create(
+            mxc.to_string(),
+            &db.globals,
+            &content_response.content_disposition.as_deref(),
+            &content_response.content_type.as_deref(),
+            &content_response.file,
+        )
+        .await?;
+
+    Ok(content_response)
+}
+
+/// # `GET /_matrix/media/r0/download/{serverName}/{mediaId}`
 ///
 /// Load media from our server or over federation.
 ///
@@ -100,36 +135,58 @@ pub async fn get_content_route(
         }
         .into())
     } else if &*body.server_name != db.globals.server_name() && body.allow_remote {
-        let get_content_response = db
+        let remote_content_response =
-            .sending
+            get_remote_content(&db, &mxc, &body.server_name, &body.media_id).await?;
-            .send_federation_request(
+        Ok(remote_content_response.into())
-                &db.globals,
-                &body.server_name,
-                get_content::Request {
-                    allow_remote: false,
-                    server_name: &body.server_name,
-                    media_id: &body.media_id,
-                },
-            )
-            .await?;
-
-        db.media
-            .create(
-                mxc,
-                &db.globals,
-                &get_content_response.content_disposition.as_deref(),
-                &get_content_response.content_type.as_deref(),
-                &get_content_response.file,
-            )
-            .await?;
-
-        Ok(get_content_response.into())
     } else {
         Err(Error::BadRequest(ErrorKind::NotFound, "Media not found."))
     }
 }
 
-/// # `POST /_matrix/media/r0/thumbnail/{serverName}/{mediaId}`
+/// # `GET /_matrix/media/r0/download/{serverName}/{mediaId}/{fileName}`
+///
+/// Load media from our server or over federation, permitting desired filename.
+///
+/// - Only allows federation if `allow_remote` is true
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/media/r0/download/<_>/<_>/<_>", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_content_as_filename_route(
+    db: DatabaseGuard,
+    body: Ruma<get_content_as_filename::Request<'_>>,
+) -> ConduitResult<get_content_as_filename::Response> {
+    let mxc = format!("mxc://{}/{}", body.server_name, body.media_id);
+
+    if let Some(FileMeta {
+        content_disposition: _,
+        content_type,
+        file,
+    }) = db.media.get(&db.globals, &mxc).await?
+    {
+        Ok(get_content_as_filename::Response {
+            file,
+            content_type,
+            content_disposition: Some(format!("inline; filename={}", body.filename)),
+        }
+        .into())
+    } else if &*body.server_name != db.globals.server_name() && body.allow_remote {
+        let remote_content_response =
+            get_remote_content(&db, &mxc, &body.server_name, &body.media_id).await?;
+
+        Ok(get_content_as_filename::Response {
+            content_disposition: Some(format!("inline: filename={}", body.filename)),
+            content_type: remote_content_response.content_type,
+            file: remote_content_response.file,
+        }
+        .into())
+    } else {
+        Err(Error::BadRequest(ErrorKind::NotFound, "Media not found."))
+    }
+}
+
+/// # `GET /_matrix/media/r0/thumbnail/{serverName}/{mediaId}`
 ///
 /// Load media thumbnail from our server or over federation.
 ///
@@ -150,7 +207,7 @@ pub async fn get_content_thumbnail_route(
     }) = db
         .media
         .get_thumbnail(
-            mxc.clone(),
+            &mxc,
             &db.globals,
             body.width
                 .try_into()

src/client_server/membership.rs

@@ -655,7 +655,7 @@ async fn join_room_by_id_helper(
 
         db.rooms.get_or_create_shortroomid(room_id, &db.globals)?;
 
-        let pdu = PduEvent::from_id_val(event_id, join_event.clone())
+        let parsed_pdu = PduEvent::from_id_val(event_id, join_event.clone())
             .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?;
 
         let mut state = HashMap::new();
@@ -695,14 +695,15 @@ async fn join_room_by_id_helper(
         }
 
         let incoming_shortstatekey = db.rooms.get_or_create_shortstatekey(
-            &pdu.kind,
+            &parsed_pdu.kind,
-            pdu.state_key
+            parsed_pdu
+                .state_key
                 .as_ref()
                 .expect("Pdu is a membership state event"),
             &db.globals,
         )?;
 
-        state.insert(incoming_shortstatekey, pdu.event_id.clone());
+        state.insert(incoming_shortstatekey, parsed_pdu.event_id.clone());
 
         let create_shortstatekey = db
             .rooms
@@ -738,12 +739,12 @@ async fn join_room_by_id_helper(
 
         // We append to state before appending the pdu, so we don't have a moment in time with the
         // pdu without it's state. This is okay because append_pdu can't fail.
-        let statehashid = db.rooms.append_to_state(&pdu, &db.globals)?;
+        let statehashid = db.rooms.append_to_state(&parsed_pdu, &db.globals)?;
 
         db.rooms.append_pdu(
-            &pdu,
+            &parsed_pdu,
-            utils::to_canonical_object(&pdu).expect("Pdu is valid canonical object"),
+            join_event,
-            iter::once(&*pdu.event_id),
+            iter::once(&*parsed_pdu.event_id),
             db,
         )?;
 

src/client_server/room.rs

@@ -344,10 +344,13 @@ pub async fn create_room_route(
 
     // 6. Events listed in initial_state
     for event in &body.initial_state {
-        let pdu_builder = PduBuilder::from(event.deserialize().map_err(|e| {
+        let mut pdu_builder = event.deserialize_as::<PduBuilder>().map_err(|e| {
             warn!("Invalid initial state event: {:?}", e);
             Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event.")
-        })?);
+        })?;
+
+        // Implicit state key defaults to ""
+        pdu_builder.state_key.get_or_insert_with(|| "".to_owned());
 
         // Silently skip encryption events if they are not allowed
         if pdu_builder.event_type == EventType::RoomEncryption && !db.globals.allow_encryption() {

src/database.rs

@@ -49,6 +49,8 @@ pub struct Config {
     database_path: String,
     #[serde(default = "default_db_cache_capacity_mb")]
     db_cache_capacity_mb: f64,
+    #[serde(default = "default_conduit_cache_capacity_modifier")]
+    conduit_cache_capacity_modifier: f64,
     #[serde(default = "default_rocksdb_max_open_files")]
     rocksdb_max_open_files: i32,
     #[serde(default = "default_pdu_cache_capacity")]
@@ -129,8 +131,12 @@ fn default_db_cache_capacity_mb() -> f64 {
     10.0
 }
 
+fn default_conduit_cache_capacity_modifier() -> f64 {
+    1.0
+}
+
 fn default_rocksdb_max_open_files() -> i32 {
-    512
+    20
 }
 
 fn default_pdu_cache_capacity() -> u32 {
@@ -361,15 +367,27 @@ impl Database {
                         .try_into()
                         .expect("pdu cache capacity fits into usize"),
                 )),
-                auth_chain_cache: Mutex::new(LruCache::new(1_000_000)),
+                auth_chain_cache: Mutex::new(LruCache::new(
-                shorteventid_cache: Mutex::new(LruCache::new(1_000_000)),
+                    (100_000.0 * config.conduit_cache_capacity_modifier) as usize,
-                eventidshort_cache: Mutex::new(LruCache::new(1_000_000)),
+                )),
-                shortstatekey_cache: Mutex::new(LruCache::new(1_000_000)),
+                shorteventid_cache: Mutex::new(LruCache::new(
-                statekeyshort_cache: Mutex::new(LruCache::new(1_000_000)),
+                    (100_000.0 * config.conduit_cache_capacity_modifier) as usize,
+                )),
+                eventidshort_cache: Mutex::new(LruCache::new(
+                    (100_000.0 * config.conduit_cache_capacity_modifier) as usize,
+                )),
+                shortstatekey_cache: Mutex::new(LruCache::new(
+                    (100_000.0 * config.conduit_cache_capacity_modifier) as usize,
+                )),
+                statekeyshort_cache: Mutex::new(LruCache::new(
+                    (100_000.0 * config.conduit_cache_capacity_modifier) as usize,
+                )),
                 our_real_users_cache: RwLock::new(HashMap::new()),
                 appservice_in_room_cache: RwLock::new(HashMap::new()),
                 lazy_load_waiting: Mutex::new(HashMap::new()),
-                stateinfo_cache: Mutex::new(LruCache::new(1000)),
+                stateinfo_cache: Mutex::new(LruCache::new(
+                    (100.0 * config.conduit_cache_capacity_modifier) as usize,
+                )),
             },
             account_data: account_data::AccountData {
                 roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?,

src/database/admin.rs

@@ -262,7 +262,10 @@ fn process_admin_command(
                 let parsed_config = serde_yaml::from_str::<serde_yaml::Value>(&appservice_config);
                 match parsed_config {
                     Ok(yaml) => match db.appservice.register_appservice(yaml) {
-                        Ok(()) => RoomMessageEventContent::text_plain("Appservice registered."),
+                        Ok(id) => RoomMessageEventContent::text_plain(format!(
+                            "Appservice registered with ID: {}.",
+                            id
+                        )),
                         Err(e) => RoomMessageEventContent::text_plain(format!(
                             "Failed to register appservice: {}",
                             e

src/database/appservice.rs

@@ -12,7 +12,9 @@ pub struct Appservice {
 }
 
 impl Appservice {
-    pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<()> {
+    /// Registers an appservice and returns the ID to the caller
+    ///
+    pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<String> {
         // TODO: Rumaify
         let id = yaml.get("id").unwrap().as_str().unwrap();
         self.id_appserviceregistrations.insert(
@@ -22,9 +24,9 @@ impl Appservice {
         self.cached_registrations
             .write()
             .unwrap()
-            .insert(id.to_owned(), yaml);
+            .insert(id.to_owned(), yaml.to_owned());
 
-        Ok(())
+        Ok(id.to_owned())
     }
 
     /// Remove an appservice registration

src/database/key_backups.rs

@@ -7,7 +7,6 @@ use ruma::{
     serde::Raw,
     RoomId, UserId,
 };
-use serde_json::json;
 use std::{collections::BTreeMap, sync::Arc};
 
 use super::abstraction::Tree;
@@ -212,13 +211,13 @@ impl KeyBackups {
         &self,
         user_id: &UserId,
         version: &str,
-    ) -> Result<BTreeMap<Box<RoomId>, Raw<RoomKeyBackup>>> {
+    ) -> Result<BTreeMap<Box<RoomId>, RoomKeyBackup>> {
         let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(version.as_bytes());
         prefix.push(0xff);
 
-        let mut rooms = BTreeMap::<Box<RoomId>, Raw<RoomKeyBackup>>::new();
+        let mut rooms = BTreeMap::<Box<RoomId>, RoomKeyBackup>::new();
 
         for result in self
             .backupkeyid_backup
@@ -244,7 +243,7 @@ impl KeyBackups {
                     Error::bad_database("backupkeyid_backup room_id is invalid room id.")
                 })?;
 
-                let key_data: serde_json::Value = serde_json::from_slice(&value).map_err(|_| {
+                let key_data = serde_json::from_slice(&value).map_err(|_| {
                     Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")
                 })?;
 
@@ -252,25 +251,13 @@ impl KeyBackups {
             })
         {
             let (room_id, session_id, key_data) = result?;
-            let room_key_backup = rooms.entry(room_id).or_insert_with(|| {
+            rooms
-                Raw::new(&RoomKeyBackup {
+                .entry(room_id)
+                .or_insert_with(|| RoomKeyBackup {
                     sessions: BTreeMap::new(),
                 })
-                .expect("RoomKeyBackup serialization")
+                .sessions
-            });
+                .insert(session_id, key_data);
-
-            let mut object = room_key_backup
-                .deserialize_as::<serde_json::Map<String, serde_json::Value>>()
-                .map_err(|_| Error::bad_database("RoomKeyBackup is not an object"))?;
-
-            let sessions = object.entry("session").or_insert_with(|| json!({}));
-            if let serde_json::Value::Object(unsigned_object) = sessions {
-                unsigned_object.insert(session_id, key_data);
-            }
-
-            *room_key_backup = Raw::from_json(
-                serde_json::value::to_raw_value(&object).expect("Value => RawValue serialization"),
-            );
         }
 
         Ok(rooms)

src/database/media.rs

@@ -171,7 +171,7 @@ impl Media {
     /// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards.
     pub async fn get_thumbnail(
         &self,
-        mxc: String,
+        mxc: &str,
         globals: &Globals,
         width: u32,
         height: u32,

src/database/sending.rs

@@ -480,6 +480,26 @@ impl Sending {
         hash.as_ref().to_owned()
     }
 
+    /// Cleanup event data
+    /// Used for instance after we remove an appservice registration
+    ///
+    #[tracing::instrument(skip(self))]
+    pub fn cleanup_events(&self, key_id: &str) -> Result<()> {
+        let mut prefix = b"+".to_vec();
+        prefix.extend_from_slice(key_id.as_bytes());
+        prefix.push(0xff);
+
+        for (key, _) in self.servercurrentevent_data.scan_prefix(prefix.clone()) {
+            self.servercurrentevent_data.remove(&key).unwrap();
+        }
+
+        for (key, _) in self.servernameevent_data.scan_prefix(prefix.clone()) {
+            self.servernameevent_data.remove(&key).unwrap();
+        }
+
+        Ok(())
+    }
+
     #[tracing::instrument(skip(db, events, kind))]
     async fn handle_events(
         kind: OutgoingKind,
@@ -520,8 +540,15 @@ impl Sending {
                     &db.globals,
                     db.appservice
                         .get_registration(server.as_str())
-                        .unwrap()
+                        .map_err(|e| (kind.clone(), e))?
-                        .unwrap(), // TODO: handle error
+                        .ok_or_else(|| {
+                            (
+                                kind.clone(),
+                                Error::bad_database(
+                                    "[Appservice] Could not load registration from db.",
+                                ),
+                            )
+                        })?,
                     appservice::event::push_events::v1::Request {
                         events: &pdu_jsons,
                         txn_id: (&*base64::encode_config(

src/main.rs

@@ -129,6 +129,7 @@ fn setup_rocket(config: Figment, data: Arc<RwLock<Database>>) -> rocket::Rocket<
                 client_server::send_event_to_device_route,
                 client_server::get_media_config_route,
                 client_server::create_content_route,
+                client_server::get_content_as_filename_route,
                 client_server::get_content_route,
                 client_server::get_content_thumbnail_route,
                 client_server::get_devices_route,

src/pdu.rs

@@ -1,9 +1,8 @@
 use crate::Error;
 use ruma::{
     events::{
-        room::member::RoomMemberEventContent, AnyEphemeralRoomEvent, AnyInitialStateEvent,
+        room::member::RoomMemberEventContent, AnyEphemeralRoomEvent, AnyRoomEvent, AnyStateEvent,
-        AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent,
+        AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent,
-        EventType, StateEvent,
     },
     serde::{CanonicalJsonObject, CanonicalJsonValue, Raw},
     state_res, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, UInt, UserId,
@@ -361,17 +360,3 @@ pub struct PduBuilder {
     pub state_key: Option<String>,
     pub redacts: Option<Arc<EventId>>,
 }
-
-/// Direct conversion prevents loss of the empty `state_key` that ruma requires.
-impl From<AnyInitialStateEvent> for PduBuilder {
-    fn from(event: AnyInitialStateEvent) -> Self {
-        Self {
-            event_type: EventType::from(event.event_type()),
-            content: to_raw_value(&event.content())
-                .expect("AnyStateEventContent came from JSON and can thus turn back into JSON."),
-            unsigned: None,
-            state_key: Some(event.state_key().to_owned()),
-            redacts: None,
-        }
-    }
-}