diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 00000000..a236cc0f --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,847 @@ +name: CI and Artifacts + +on: + pull_request: + push: + paths-ignore: + - '.gitlab-ci.yml' + - '.gitignore' + - 'renovate.json' + - 'debian/**' + - 'docker/**' + branches: + - main + tags: + - '*' + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +concurrency: + group: ${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +env: + # Required to make some things output color + TERM: ansi + # Publishing to my nix binary cache + ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} + # conduwuit.cachix.org + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + # Just in case incremental is still being set to true, speeds up CI + CARGO_INCREMENTAL: 0 + # Custom nix binary cache if fork is being used + ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} + ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} + # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps + NIX_CONFIG: | + show-trace = true + extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net https://nix-community.cachix.org https://crane.cachix.org + extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk= + experimental-features = nix-command flakes + extra-experimental-features = nix-command flakes + accept-flake-config = true + WEB_UPLOAD_SSH_USERNAME: ${{ secrets.WEB_UPLOAD_SSH_USERNAME }} + GH_REF_NAME: ${{ github.ref_name }} + WEBSERVER_DIR_NAME: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} + DOCKER_HOST: "tcp://docker:2375" + DOCKER_TLS_CERTDIR: "" + +permissions: {} + +jobs: + dind-test: + name: dind-${{ matrix.runner }} + strategy: + matrix: + runner: + - jade + - tom + - ubuntu-22.04 + runs-on: ${{ matrix.runner }} + services: + dind: + env: + DOCKER_HOST: unix:///var/run/dind.socket + DOCKER_TLS_CERTDIR: "" + image: docker:dind + options: >- + --tty + --restart always + steps: + - name: curl + run: |- + sleep 30 + curl dind:2375 || true + tests: + name: Test + runs-on: [tom, nix] + container: + image: catthehacker/ubuntu:rust-latest + volumes: + - /nix/store:/nix/store + - /nix/var/nix/db:/nix/var/nix/db + steps: + - name: Setup SSH web publish + env: + web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' + run: | + mkdir -p -v ~/.ssh + + echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + + chmod 600 ~/.ssh/id_ed25519 + + cat >>~/.ssh/config <> "$GITHUB_ENV" + + - name: Sync repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Tag comparison check + if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }} + run: | + # Tag mismatch with latest repo tag check to prevent potential downgrades + LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) + + if [ ${LATEST_TAG} != ${GH_REF_NAME} ]; then + echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' + echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' >> $GITHUB_STEP_SUMMARY + exit 1 + fi + + - uses: https://github.com/cachix/install-nix-action@v31.1.0 + + - name: Nom + if: ${{ env.ACT_EXEC }} + run: | + echo "Installing nix output monitor" + nix profile install nixpkgs#nix-output-monitor -Lv + sudo cp $(which nom) /usr/bin/nom-nix + file $(which nom) + lsblk + + - name: Prepare build environment + shell: bash + run: | + echo $SHELL + df -h /nix/var/nix/db + du -had 1 /nix/ + if ! type nom &> /dev/null; then + sudo cp $(which nix) /usr/bin/nom-or-nix + fi + echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + nix profile install -v --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv nixpkgs#nodejs nixpkgs#iproute2 nixpkgs#nettools nixpkgs#bind + ip a + direnv allow + + if type nom &> /dev/null; then + nom develop .#all-features --command true + else + nix develop .#all-features --command true + fi + + - name: Cache CI dependencies + run: | + bin/nix-build-and-cache ci + bin/nix-build-and-cache just '.#devShells.x86_64-linux.default' + bin/nix-build-and-cache just '.#devShells.x86_64-linux.all-features' + bin/nix-build-and-cache just '.#devShells.x86_64-linux.dynamic' + + # use rust-cache + - uses: https://github.com/Swatinem/rust-cache@v2 + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + #if: ${{ !startsWith(github.ref, 'refs/tags/') }} + with: + cache-all-crates: "true" + cache-on-failure: "true" + cache-targets: "true" + + - name: Run CI tests + shell: bash + env: + CARGO_PROFILE: "test" + run: | + direnv exec . engage > >(tee -a test_output.log) + + - name: Build complement image + run: | + bin/nix-build-and-cache just .#complement + cp -v -f result complement_oci_image.tar.gz + + + - name: Upload Complement OCI image + uses: https://data.forgejo.org/forgejo/upload-artifact@v4 + env: + with: + name: complement_oci_image.tar.gz + path: complement_oci_image.tar.gz + if-no-files-found: error + compression-level: 0 + + + complement: + name: complement + runs-on: [dind, nix] + needs: tests + services: + dind: + image: docker:dind + options: >- + --privileged + env: + DOCKER_TLS_CERTDIR: "" + DOCKER_HOST: unix:///var/run/dind.socket + container: + image: catthehacker/ubuntu:act-latest + volumes: + - /nix/store:/nix/store + - /nix/var/nix/db:/nix/var/nix/db + steps: + - name: Sync repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: https://github.com/cachix/install-nix-action@v31.1.0 + + - name: Prepare build environment + shell: bash + run: | + echo $SHELL + df -h /nix/var/nix/db + du -had 1 /nix/ + if ! type nom &> /dev/null; then + sudo cp $(which nix) /usr/bin/nom-or-nix + fi + echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + nix profile install -v --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv nixpkgs#nodejs nixpkgs#iproute2 nixpkgs#nettools nixpkgs#bind + ip a + direnv allow + + if type nom &> /dev/null; then + nom develop .#all-features --command true + else + nix develop .#all-features --command true + fi + + - name: Download artifacts + uses: https://data.forgejo.org/forgejo/download-artifact@v4 + with: + pattern: "complement_oci_image.tar.gz" + + - name: Run Complement tests + shell: bash + env: + DOCKER_HOST: tcp://dind:2375 + CARGO_PROFILE: "test" + run: | + addr=$(ip -brief address show eth0 | awk '{print $3}' | awk -F/ '{print $1}') + export COMPLEMENT_HS_PORT_BINDING_IP=$(dig +short dind) + export COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT=dind + # the nix devshell sets $COMPLEMENT_SRC, so "/dev/null" is no-op + direnv exec . bin/complement "/dev/null" complement_test_logs.jsonl complement_test_results.jsonl > >(tee -a test_output.log) + netstat -tnlp + + - name: Upload Complement logs + uses: https://data.forgejo.org/forgejo/upload-artifact@v4 + with: + name: complement_test_logs.jsonl + path: complement_test_logs.jsonl + if-no-files-found: error + + - name: Upload Complement results + uses: https://data.forgejo.org/forgejo/upload-artifact@v4 + with: + name: complement_test_results.jsonl + path: complement_test_results.jsonl + if-no-files-found: error + + - name: Diff Complement results with checked-in repo results + run: | + diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_diff_output.log) + + - name: Update Job Summary + env: + GH_JOB_STATUS: ${{ job.status }} + if: success() || failure() + run: | + if [ ${GH_JOB_STATUS} == 'success' ]; then + echo '# ✅ CI completed suwuccessfully' >> $GITHUB_STEP_SUMMARY + else + echo '# ❌ CI failed (last 100 lines of output)' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + tail -n 100 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + echo '# Complement diff results (last 100 lines)' >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + tail -n 100 complement_diff_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi + + build: + name: Build + runs-on: [tom, nix] + container: + image: catthehacker/ubuntu:js-latest + volumes: + - /nix/store:/nix/store + - /nix/var/nix/db:/nix/var/nix/db + strategy: + matrix: + include: + - target: aarch64-linux-musl + - target: x86_64-linux-musl + steps: + - name: Sync repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + #- uses: nixbuild/nix-quick-install-action@master + - uses: https://github.com/cachix/install-nix-action@v31.1.0 + + - name: Setup SSH web publish + env: + web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' + run: | + mkdir -p -v ~/.ssh + + echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + + chmod 600 ~/.ssh/id_ed25519 + + cat >>~/.ssh/config <> "$GITHUB_ENV" + + - name: Prepare build environment + run: | + echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + direnv allow + nix develop .#all-features --command true --impure + + # use rust-cache + - uses: https://github.com/Swatinem/rust-cache@v2 + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + #if: ${{ !startsWith(github.ref, 'refs/tags/') }} + with: + cache-all-crates: "true" + cache-on-failure: "true" + cache-targets: "true" + + - name: Build static ${{ matrix.target }}-all-features + shell: bash + run: | + if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl" + fi + + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features + + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}.deb + mv -v target/release/conduwuit static-${{ matrix.target }} + mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb + + - name: Build static x86_64-linux-musl-all-features-x86_64-haswell-optimised + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-x86_64-linux-musl-all-features-x86_64-haswell-optimised + + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb + mv -v target/release/conduwuit static-x86_64-linux-musl-x86_64-haswell-optimised + mv -v target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb x86_64-linux-musl-x86_64-haswell-optimised.deb + + # quick smoke test of the x86_64 static release binary + - name: Quick smoke test the x86_64 static release binary + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + # GH actions default runners are x86_64 only + if file result/bin/conduwuit | grep x86-64; then + result/bin/conduwuit --version + result/bin/conduwuit --help + result/bin/conduwuit -Oserver_name="'$(date -u +%s).local'" -Odatabase_path="'/tmp/$(date -u +%s)'" --execute "server admin-notice awawawawawawawawawawa" --execute "server memory-usage" --execute "server shutdown" + fi + + - name: Build static debug ${{ matrix.target }}-all-features + shell: bash + run: | + if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl" + fi + + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug + + # > warning: dev profile is not supported and will be a hard error in the future. cargo-deb is for making releases, and it doesn't make sense to use it with dev profiles. + # so we need to coerce cargo-deb into thinking this is a release binary + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}-debug.deb + mv -v target/release/conduwuit static-${{ matrix.target }}-debug + mv -v target/release/${{ matrix.target }}-debug.deb ${{ matrix.target }}-debug.deb + + # quick smoke test of the x86_64 static debug binary + - name: Run x86_64 static debug binary + run: | + # GH actions default runners are x86_64 only + if file result/bin/conduwuit | grep x86-64; then + result/bin/conduwuit --version + fi + + # check validity of produced deb package, invalid debs will error on these commands + - name: Validate produced deb package + run: | + # List contents + dpkg-deb --contents ${{ matrix.target }}.deb + dpkg-deb --contents ${{ matrix.target }}-debug.deb + # List info + dpkg-deb --info ${{ matrix.target }}.deb + dpkg-deb --info ${{ matrix.target }}-debug.deb + + - name: Upload static-x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub + uses: forgejo/upload-artifact@v4 + if: ${{ matrix.target == 'x86_64-linux-musl' }} + with: + name: static-x86_64-linux-musl-x86_64-haswell-optimised + path: static-x86_64-linux-musl-x86_64-haswell-optimised + if-no-files-found: error + + - name: Upload static-${{ matrix.target }}-all-features to GitHub + uses: forgejo/upload-artifact@v4 + with: + name: static-${{ matrix.target }} + path: static-${{ matrix.target }} + if-no-files-found: error + + - name: Upload static deb ${{ matrix.target }}-all-features to GitHub + uses: forgejo/upload-artifact@v4 + with: + name: deb-${{ matrix.target }} + path: ${{ matrix.target }}.deb + if-no-files-found: error + compression-level: 0 + + - name: Upload static-x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + if [ ! -z $SSH_WEBSITE ]; then + chmod +x static-x86_64-linux-musl-x86_64-haswell-optimised + scp static-x86_64-linux-musl-x86_64-haswell-optimised website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-x86_64-linux-musl-x86_64-haswell-optimised + fi + + - name: Upload static-${{ matrix.target }}-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + chmod +x static-${{ matrix.target }} + scp static-${{ matrix.target }} website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }} + fi + + - name: Upload static deb x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp x86_64-linux-musl-x86_64-haswell-optimised.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/x86_64-linux-musl-x86_64-haswell-optimised.deb + fi + + - name: Upload static deb ${{ matrix.target }}-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp ${{ matrix.target }}.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}.deb + fi + + - name: Upload static-${{ matrix.target }}-debug-all-features to GitHub + uses: forgejo/upload-artifact@v4 + with: + name: static-${{ matrix.target }}-debug + path: static-${{ matrix.target }}-debug + if-no-files-found: error + + - name: Upload static deb ${{ matrix.target }}-debug-all-features to GitHub + uses: forgejo/upload-artifact@v4 + with: + name: deb-${{ matrix.target }}-debug + path: ${{ matrix.target }}-debug.deb + if-no-files-found: error + compression-level: 0 + + - name: Upload static-${{ matrix.target }}-debug-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp static-${{ matrix.target }}-debug website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }}-debug + fi + + - name: Upload static deb ${{ matrix.target }}-debug-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp ${{ matrix.target }}-debug.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}-debug.deb + fi + + - name: Build OCI image ${{ matrix.target }}-all-features + run: | + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features + + cp -v -f result oci-image-${{ matrix.target }}.tar.gz + + - name: Build OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + bin/nix-build-and-cache just .#oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised + + cp -v -f result oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + + - name: Build debug OCI image ${{ matrix.target }}-all-features + run: | + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug + + cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz + + - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub + if: ${{ matrix.target == 'x86_64-linux-musl' }} + uses: forgejo/upload-artifact@v4 + with: + name: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised + path: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + if-no-files-found: error + compression-level: 0 + - name: Upload OCI image ${{ matrix.target }}-all-features to GitHub + uses: forgejo/upload-artifact@v4 + with: + name: oci-image-${{ matrix.target }} + path: oci-image-${{ matrix.target }}.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Upload OCI image ${{ matrix.target }}-debug-all-features to GitHub + uses: forgejo/upload-artifact@v4 + with: + name: oci-image-${{ matrix.target }}-debug + path: oci-image-${{ matrix.target }}-debug.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz to webserver + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + fi + + - name: Upload OCI image ${{ matrix.target }}-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp oci-image-${{ matrix.target }}.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}.tar.gz + fi + + - name: Upload OCI image ${{ matrix.target }}-debug-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp oci-image-${{ matrix.target }}-debug.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}-debug.tar.gz + fi + + variables: + outputs: + github_repository: ${{ steps.var.outputs.github_repository }} + runs-on: self-hosted + steps: + - name: Setting global variables + uses: https://github.com/actions/github-script@v7 + id: var + with: + script: | + core.setOutput('github_repository', '${{ github.repository }}'.toLowerCase()) + docker: + name: Docker publish + runs-on: self-hosted + needs: [build, variables, tests] + permissions: + packages: write + contents: read + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' + env: + DOCKER_HUB_REPO: docker.io/${{ needs.variables.outputs.github_repository }} + GHCR_REPO: ghcr.io/${{ needs.variables.outputs.github_repository }} + GLCR_REPO: registry.gitlab.com/conduwuit/conduwuit + UNIQUE_TAG: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} + BRANCH_TAG: ${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} + + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} + GHCR_ENABLED: "${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) && 'true' || 'false' }}" + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Docker Hub + if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} + uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ vars.DOCKER_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitLab Container Registry + if: ${{ (vars.GITLAB_USERNAME != '') && (env.GITLAB_TOKEN != '') }} + uses: docker/login-action@v3 + with: + registry: registry.gitlab.com + username: ${{ vars.GITLAB_USERNAME }} + password: ${{ secrets.GITLAB_TOKEN }} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + pattern: "oci*" + + - name: Move OCI images into position + run: | + mv -v oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised/*.tar.gz oci-image-amd64-haswell-optimised.tar.gz + mv -v oci-image-x86_64-linux-musl/*.tar.gz oci-image-amd64.tar.gz + mv -v oci-image-aarch64-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz + mv -v oci-image-x86_64-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz + mv -v oci-image-aarch64-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz + + - name: Load and push amd64 haswell image + run: | + docker load -i oci-image-amd64-haswell-optimised.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker push ${GHCR_REPO}:${UNIQUE_TAG}-haswell + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker push ${GLCR_REPO}:${UNIQUE_TAG}-haswell + fi + + - name: Load and push amd64 image + run: | + docker load -i oci-image-amd64.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + fi + + - name: Load and push arm64 image + run: | + docker load -i oci-image-arm64v8.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 + fi + + - name: Load and push amd64 debug image + run: | + docker load -i oci-image-amd64-debug.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + + - name: Load and push arm64 debug image + run: | + docker load -i oci-image-arm64v8-debug.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + fi + + - name: Create Docker haswell manifests + run: | + # Dockerhub Container Registry + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + fi + # GitHub Container Registry + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell + fi + # GitLab Container Registry + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell + fi + + - name: Create Docker combined manifests + run: | + # Dockerhub Container Registry + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + fi + # GitHub Container Registry + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${GHCR_REPO}:${BRANCH_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + fi + # GitLab Container Registry + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${GLCR_REPO}:${BRANCH_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + fi + + - name: Create Docker combined debug manifests + run: | + # Dockerhub Container Registry + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + fi + # GitHub Container Registry + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + # GitLab Container Registry + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + + - name: Push manifests to Docker registries + run: | + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG} + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG} + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell + fi + if [ $GHCR_ENABLED = "true" ]; then + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG} + docker manifest push ${GHCR_REPO}:${BRANCH_TAG} + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-debug + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-haswell + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG} + docker manifest push ${GLCR_REPO}:${BRANCH_TAG} + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-debug + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-haswell + fi + + - name: Add Image Links to Job Summary + run: | + if [ ! -z $DOCKERHUB_TOKEN ]; then + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY + fi + if [ $GHCR_ENABLED = "true" ]; then + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY + fi + if [ ! -z $GITLAB_TOKEN ]; then + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..4d30d252 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,742 @@ +name: CI and Artifacts + +on: + pull_request: + push: + paths-ignore: + - '.gitlab-ci.yml' + - '.gitignore' + - 'renovate.json' + - 'debian/**' + - 'docker/**' + branches: + - main + tags: + - '*' + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +concurrency: + group: ${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +env: + # Required to make some things output color + TERM: ansi + # Publishing to my nix binary cache + ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} + # conduwuit.cachix.org + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + # Just in case incremental is still being set to true, speeds up CI + CARGO_INCREMENTAL: 0 + # Custom nix binary cache if fork is being used + ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} + ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} + # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps + NIX_CONFIG: | + show-trace = true + extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net https://nix-community.cachix.org https://crane.cachix.org + extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk= + experimental-features = nix-command flakes + extra-experimental-features = nix-command flakes + accept-flake-config = true + WEB_UPLOAD_SSH_USERNAME: ${{ secrets.WEB_UPLOAD_SSH_USERNAME }} + GH_REF_NAME: ${{ github.ref_name }} + WEBSERVER_DIR_NAME: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} + +permissions: {} + +jobs: + tests: + name: Test + runs-on: self-hosted + steps: + - name: Setup SSH web publish + env: + web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' + run: | + mkdir -p -v ~/.ssh + + echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + + chmod 600 ~/.ssh/id_ed25519 + + cat >>~/.ssh/config <> "$GITHUB_ENV" + + - name: Sync repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Tag comparison check + if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }} + run: | + # Tag mismatch with latest repo tag check to prevent potential downgrades + LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) + + if [ ${LATEST_TAG} != ${GH_REF_NAME} ]; then + echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' + echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' >> $GITHUB_STEP_SUMMARY + exit 1 + fi + + #- uses: nixbuild/nix-quick-install-action@master + - uses: cachix/install-nix-action@v31.1.0 + + - name: Nom + if: ${{ env.ACT }} + run: | + echo "aaaa" + nix profile install nixpkgs#nix-output-monitor + sudo cp $(which nom) /usr/bin/nom-nix + file $(which nom) + lsblk + + #- uses: https://github.com/cachix/install-nix-action@v27 + # if: ${{ ! env.ACT }} + + - name: Prepare build environment + run: | + if ! type nom &> /dev/null; then + sudo cp $(which nix) /usr/bin/nom-or-nix + fi + echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + direnv allow + set -x + file $(which nom-nix) + nom develop .#all-features --command true + + - name: Cache CI dependencies + run: | + bin/nix-build-and-cache ci + bin/nix-build-and-cache just '.#devShells.x86_64-linux.default' + bin/nix-build-and-cache just '.#devShells.x86_64-linux.all-features' + bin/nix-build-and-cache just '.#devShells.x86_64-linux.dynamic' + + # use rust-cache + - uses: Swatinem/rust-cache@v2 + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + #if: ${{ !startsWith(github.ref, 'refs/tags/') }} + with: + cache-all-crates: "true" + cache-on-failure: "true" + cache-targets: "true" + + - name: Run CI tests + env: + CARGO_PROFILE: "test" + run: | + direnv exec . engage > >(tee -a test_output.log) + + - name: Run Complement tests + env: + CARGO_PROFILE: "test" + run: | + # the nix devshell sets $COMPLEMENT_SRC, so "/dev/null" is no-op + direnv exec . bin/complement "/dev/null" complement_test_logs.jsonl complement_test_results.jsonl > >(tee -a test_output.log) + cp -v -f result complement_oci_image.tar.gz + + - name: Upload Complement OCI image + uses: actions/upload-artifact@v4 + with: + name: complement_oci_image.tar.gz + path: complement_oci_image.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Upload Complement logs + uses: actions/upload-artifact@v4 + with: + name: complement_test_logs.jsonl + path: complement_test_logs.jsonl + if-no-files-found: error + + - name: Upload Complement results + uses: actions/upload-artifact@v4 + with: + name: complement_test_results.jsonl + path: complement_test_results.jsonl + if-no-files-found: error + + - name: Diff Complement results with checked-in repo results + run: | + diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_diff_output.log) + + - name: Update Job Summary + env: + GH_JOB_STATUS: ${{ job.status }} + if: success() || failure() + run: | + if [ ${GH_JOB_STATUS} == 'success' ]; then + echo '# ✅ CI completed suwuccessfully' >> $GITHUB_STEP_SUMMARY + else + echo '# ❌ CI failed (last 100 lines of output)' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + tail -n 100 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + echo '# Complement diff results (last 100 lines)' >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + tail -n 100 complement_diff_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi + + build: + name: Build + runs-on: self-hosted + strategy: + matrix: + include: + - target: aarch64-linux-musl + - target: x86_64-linux-musl + steps: + - name: Sync repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + #- uses: nixbuild/nix-quick-install-action@master + - uses: cachix/install-nix-action@v31.1.0 + + - name: Setup SSH web publish + env: + web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' + run: | + mkdir -p -v ~/.ssh + + echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + + chmod 600 ~/.ssh/id_ed25519 + + cat >>~/.ssh/config <> "$GITHUB_ENV" + + - name: Prepare build environment + run: | + echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + direnv allow + nix develop .#all-features --command true --impure + + # use rust-cache + - uses: Swatinem/rust-cache@v2 + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + #if: ${{ !startsWith(github.ref, 'refs/tags/') }} + with: + cache-all-crates: "true" + cache-on-failure: "true" + cache-targets: "true" + + - name: Build static ${{ matrix.target }}-all-features + run: | + if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl" + fi + + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features + + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}.deb + mv -v target/release/conduwuit static-${{ matrix.target }} + mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb + + - name: Build static x86_64-linux-musl-all-features-x86_64-haswell-optimised + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-x86_64-linux-musl-all-features-x86_64-haswell-optimised + + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb + mv -v target/release/conduwuit static-x86_64-linux-musl-x86_64-haswell-optimised + mv -v target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb x86_64-linux-musl-x86_64-haswell-optimised.deb + + # quick smoke test of the x86_64 static release binary + - name: Quick smoke test the x86_64 static release binary + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + # GH actions default runners are x86_64 only + if file result/bin/conduwuit | grep x86-64; then + result/bin/conduwuit --version + result/bin/conduwuit --help + result/bin/conduwuit -Oserver_name="'$(date -u +%s).local'" -Odatabase_path="'/tmp/$(date -u +%s)'" --execute "server admin-notice awawawawawawawawawawa" --execute "server memory-usage" --execute "server shutdown" + fi + + - name: Build static debug ${{ matrix.target }}-all-features + run: | + if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl" + fi + + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug + + # > warning: dev profile is not supported and will be a hard error in the future. cargo-deb is for making releases, and it doesn't make sense to use it with dev profiles. + # so we need to coerce cargo-deb into thinking this is a release binary + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}-debug.deb + mv -v target/release/conduwuit static-${{ matrix.target }}-debug + mv -v target/release/${{ matrix.target }}-debug.deb ${{ matrix.target }}-debug.deb + + # quick smoke test of the x86_64 static debug binary + - name: Run x86_64 static debug binary + run: | + # GH actions default runners are x86_64 only + if file result/bin/conduwuit | grep x86-64; then + result/bin/conduwuit --version + fi + + # check validity of produced deb package, invalid debs will error on these commands + - name: Validate produced deb package + run: | + # List contents + dpkg-deb --contents ${{ matrix.target }}.deb + dpkg-deb --contents ${{ matrix.target }}-debug.deb + # List info + dpkg-deb --info ${{ matrix.target }}.deb + dpkg-deb --info ${{ matrix.target }}-debug.deb + + - name: Upload static-x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub + uses: actions/upload-artifact@v4 + if: ${{ matrix.target == 'x86_64-linux-musl' }} + with: + name: static-x86_64-linux-musl-x86_64-haswell-optimised + path: static-x86_64-linux-musl-x86_64-haswell-optimised + if-no-files-found: error + + - name: Upload static-${{ matrix.target }}-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: static-${{ matrix.target }} + path: static-${{ matrix.target }} + if-no-files-found: error + + - name: Upload static deb ${{ matrix.target }}-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: deb-${{ matrix.target }} + path: ${{ matrix.target }}.deb + if-no-files-found: error + compression-level: 0 + + - name: Upload static-x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + if [ ! -z $SSH_WEBSITE ]; then + chmod +x static-x86_64-linux-musl-x86_64-haswell-optimised + scp static-x86_64-linux-musl-x86_64-haswell-optimised website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-x86_64-linux-musl-x86_64-haswell-optimised + fi + + - name: Upload static-${{ matrix.target }}-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + chmod +x static-${{ matrix.target }} + scp static-${{ matrix.target }} website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }} + fi + + - name: Upload static deb x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp x86_64-linux-musl-x86_64-haswell-optimised.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/x86_64-linux-musl-x86_64-haswell-optimised.deb + fi + + - name: Upload static deb ${{ matrix.target }}-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp ${{ matrix.target }}.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}.deb + fi + + - name: Upload static-${{ matrix.target }}-debug-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: static-${{ matrix.target }}-debug + path: static-${{ matrix.target }}-debug + if-no-files-found: error + + - name: Upload static deb ${{ matrix.target }}-debug-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: deb-${{ matrix.target }}-debug + path: ${{ matrix.target }}-debug.deb + if-no-files-found: error + compression-level: 0 + + - name: Upload static-${{ matrix.target }}-debug-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp static-${{ matrix.target }}-debug website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }}-debug + fi + + - name: Upload static deb ${{ matrix.target }}-debug-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp ${{ matrix.target }}-debug.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}-debug.deb + fi + + - name: Build OCI image ${{ matrix.target }}-all-features + run: | + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features + + cp -v -f result oci-image-${{ matrix.target }}.tar.gz + + - name: Build OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + bin/nix-build-and-cache just .#oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised + + cp -v -f result oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + + - name: Build debug OCI image ${{ matrix.target }}-all-features + run: | + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug + + cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz + + - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub + if: ${{ matrix.target == 'x86_64-linux-musl' }} + uses: actions/upload-artifact@v4 + with: + name: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised + path: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + if-no-files-found: error + compression-level: 0 + - name: Upload OCI image ${{ matrix.target }}-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: oci-image-${{ matrix.target }} + path: oci-image-${{ matrix.target }}.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Upload OCI image ${{ matrix.target }}-debug-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: oci-image-${{ matrix.target }}-debug + path: oci-image-${{ matrix.target }}-debug.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz to webserver + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + fi + + - name: Upload OCI image ${{ matrix.target }}-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp oci-image-${{ matrix.target }}.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}.tar.gz + fi + + - name: Upload OCI image ${{ matrix.target }}-debug-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp oci-image-${{ matrix.target }}-debug.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}-debug.tar.gz + fi + + variables: + outputs: + github_repository: ${{ steps.var.outputs.github_repository }} + runs-on: self-hosted + steps: + - name: Setting global variables + uses: actions/github-script@v7 + id: var + with: + script: | + core.setOutput('github_repository', '${{ github.repository }}'.toLowerCase()) + docker: + name: Docker publish + runs-on: self-hosted + needs: [build, variables, tests] + permissions: + packages: write + contents: read + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' + env: + DOCKER_HUB_REPO: docker.io/${{ needs.variables.outputs.github_repository }} + GHCR_REPO: ghcr.io/${{ needs.variables.outputs.github_repository }} + GLCR_REPO: registry.gitlab.com/conduwuit/conduwuit + UNIQUE_TAG: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} + BRANCH_TAG: ${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} + + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} + GHCR_ENABLED: "${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) && 'true' || 'false' }}" + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Docker Hub + if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} + uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ vars.DOCKER_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitLab Container Registry + if: ${{ (vars.GITLAB_USERNAME != '') && (env.GITLAB_TOKEN != '') }} + uses: docker/login-action@v3 + with: + registry: registry.gitlab.com + username: ${{ vars.GITLAB_USERNAME }} + password: ${{ secrets.GITLAB_TOKEN }} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + pattern: "oci*" + + - name: Move OCI images into position + run: | + mv -v oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised/*.tar.gz oci-image-amd64-haswell-optimised.tar.gz + mv -v oci-image-x86_64-linux-musl/*.tar.gz oci-image-amd64.tar.gz + mv -v oci-image-aarch64-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz + mv -v oci-image-x86_64-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz + mv -v oci-image-aarch64-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz + + - name: Load and push amd64 haswell image + run: | + docker load -i oci-image-amd64-haswell-optimised.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker push ${GHCR_REPO}:${UNIQUE_TAG}-haswell + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker push ${GLCR_REPO}:${UNIQUE_TAG}-haswell + fi + + - name: Load and push amd64 image + run: | + docker load -i oci-image-amd64.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + fi + + - name: Load and push arm64 image + run: | + docker load -i oci-image-arm64v8.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 + fi + + - name: Load and push amd64 debug image + run: | + docker load -i oci-image-amd64-debug.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + + - name: Load and push arm64 debug image + run: | + docker load -i oci-image-arm64v8-debug.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + fi + + - name: Create Docker haswell manifests + run: | + # Dockerhub Container Registry + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + fi + # GitHub Container Registry + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell + fi + # GitLab Container Registry + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell + fi + + - name: Create Docker combined manifests + run: | + # Dockerhub Container Registry + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + fi + # GitHub Container Registry + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${GHCR_REPO}:${BRANCH_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + fi + # GitLab Container Registry + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${GLCR_REPO}:${BRANCH_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + fi + + - name: Create Docker combined debug manifests + run: | + # Dockerhub Container Registry + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + fi + # GitHub Container Registry + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + # GitLab Container Registry + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + + - name: Push manifests to Docker registries + run: | + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG} + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG} + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell + fi + if [ $GHCR_ENABLED = "true" ]; then + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG} + docker manifest push ${GHCR_REPO}:${BRANCH_TAG} + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-debug + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-haswell + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG} + docker manifest push ${GLCR_REPO}:${BRANCH_TAG} + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-debug + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-haswell + fi + + - name: Add Image Links to Job Summary + run: | + if [ ! -z $DOCKERHUB_TOKEN ]; then + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY + fi + if [ $GHCR_ENABLED = "true" ]; then + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY + fi + if [ ! -z $GITLAB_TOKEN ]; then + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/docker-hub-description.yml b/.github/workflows/docker-hub-description.yml new file mode 100644 index 00000000..b4f142db --- /dev/null +++ b/.github/workflows/docker-hub-description.yml @@ -0,0 +1,41 @@ +name: Update Docker Hub Description + +on: + push: + branches: + - main + paths: + - README.md + - .github/workflows/docker-hub-description.yml + + workflow_dispatch: + +jobs: + dockerHubDescription: + runs-on: ubuntu-latest + if: ${{ (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' && (vars.DOCKER_USERNAME != '') }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setting variables + uses: actions/github-script@v7 + id: var + with: + script: | + const githubRepo = '${{ github.repository }}'.toLowerCase() + const repoId = githubRepo.split('/')[1] + + core.setOutput('github_repository', githubRepo) + const dockerRepo = '${{ vars.DOCKER_USERNAME }}'.toLowerCase() + '/' + repoId + core.setOutput('docker_repo', dockerRepo) + + - name: Docker Hub Description + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ vars.DOCKER_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: ${{ steps.var.outputs.docker_repo }} + short-description: ${{ github.event.repository.description }} + enable-url-completion: true diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 00000000..b5b4ff46 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,104 @@ +name: Documentation and GitHub Pages + +on: + pull_request: + push: + branches: + - main + tags: + - '*' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +env: + # Required to make some things output color + TERM: ansi + # Publishing to my nix binary cache + ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} + # conduwuit.cachix.org + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + # Custom nix binary cache if fork is being used + ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} + ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} + # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps + NIX_CONFIG: | + show-trace = true + extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net https://nix-community.cachix.org https://crane.cachix.org + extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk= + experimental-features = nix-command flakes + extra-experimental-features = nix-command flakes + accept-flake-config = true + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +permissions: {} + +jobs: + docs: + name: Documentation and GitHub Pages + runs-on: self-hosted + + permissions: + pages: write + id-token: write + + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Sync repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setup GitHub Pages + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') + uses: actions/configure-pages@v5 + + - name: Prepare build environment + run: | + echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + direnv allow + nix develop --command true + + - name: Cache CI dependencies + run: | + bin/nix-build-and-cache ci + + - name: Run lychee and markdownlint + run: | + direnv exec . engage just lints lychee + direnv exec . engage just lints markdownlint + + - name: Build documentation (book) + run: | + bin/nix-build-and-cache just .#book + + cp -r --dereference result public + chmod u+w -R public + + - name: Upload generated documentation (book) as normal artifact + uses: actions/upload-artifact@v4 + with: + name: public + path: public + if-no-files-found: error + # don't compress again + compression-level: 0 + + - name: Upload generated documentation (book) as GitHub Pages artifact + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') + uses: actions/upload-pages-artifact@v3 + with: + path: public + + - name: Deploy to GitHub Pages + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..cfe72d2a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,118 @@ +name: Upload Release Assets + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: 'Tag to release' + required: true + type: string + action_id: + description: 'Action ID of the CI run' + required: true + type: string + +permissions: {} + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: write + env: + GH_EVENT_NAME: ${{ github.event_name }} + GH_EVENT_INPUTS_ACTION_ID: ${{ github.event.inputs.action_id }} + GH_EVENT_INPUTS_TAG: ${{ github.event.inputs.tag }} + GH_REPOSITORY: ${{ github.repository }} + GH_SHA: ${{ github.sha }} + GH_TAG: ${{ github.event.release.tag_name }} + + steps: + - name: get latest ci id + id: get_ci_id + env: + GH_TOKEN: ${{ github.token }} + run: | + if [ "${GH_EVENT_NAME}" == "workflow_dispatch" ]; then + id="${GH_EVENT_INPUTS_ACTION_ID}" + tag="${GH_EVENT_INPUTS_TAG}" + else + # get all runs of the ci workflow + json=$(gh api "repos/${GH_REPOSITORY}/actions/workflows/ci.yml/runs") + + # find first run that is github sha and status is completed + id=$(echo "$json" | jq ".workflow_runs[] | select(.head_sha == \"${GH_SHA}\" and .status == \"completed\") | .id" | head -n 1) + + if [ ! "$id" ]; then + echo "No completed runs found" + echo "ci_id=0" >> "$GITHUB_OUTPUT" + exit 0 + fi + + tag="${GH_TAG}" + fi + + echo "ci_id=$id" >> "$GITHUB_OUTPUT" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + + - name: get latest ci artifacts + if: steps.get_ci_id.outputs.ci_id != 0 + uses: actions/download-artifact@v4 + env: + GH_TOKEN: ${{ github.token }} + with: + merge-multiple: true + run-id: ${{ steps.get_ci_id.outputs.ci_id }} + github-token: ${{ github.token }} + + - run: | + ls + + - name: upload release assets + if: steps.get_ci_id.outputs.ci_id != 0 + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ steps.get_ci_id.outputs.tag }} + run: | + for file in $(find . -type f); do + case "$file" in + *json*) echo "Skipping $file...";; + *) echo "Uploading $file..."; gh release upload $TAG "$file" --clobber --repo="${GH_REPOSITORY}" || echo "Something went wrong, skipping.";; + esac + done + + - name: upload release assets to website + if: steps.get_ci_id.outputs.ci_id != 0 + env: + TAG: ${{ steps.get_ci_id.outputs.tag }} + run: | + mkdir -p -v ~/.ssh + + echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + + chmod 600 ~/.ssh/id_ed25519 + + cat >>~/.ssh/config < /dev/null; then echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi + # Accept flake config from "untrusted" users + - if command -v nix > /dev/null; then echo "accept-flake-config = true" >> /etc/nix/nix.conf; fi + + # Add conduwuit binary cache + - if command -v nix > /dev/null; then echo "extra-substituters = https://attic.kennel.juneis.dog/conduwuit" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE=" >> /etc/nix/nix.conf; fi + + - if command -v nix > /dev/null; then echo "extra-substituters = https://attic.kennel.juneis.dog/conduit" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk=" >> /etc/nix/nix.conf; fi + + # Add alternate binary cache + - if command -v nix > /dev/null && [ -n "$ATTIC_ENDPOINT" ]; then echo "extra-substituters = $ATTIC_ENDPOINT" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null && [ -n "$ATTIC_PUBLIC_KEY" ]; then echo "extra-trusted-public-keys = $ATTIC_PUBLIC_KEY" >> /etc/nix/nix.conf; fi + + # Add crane binary cache + - if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi + + # Add nix-community binary cache + - if command -v nix > /dev/null; then echo "extra-substituters = https://nix-community.cachix.org" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" >> /etc/nix/nix.conf; fi + + - if command -v nix > /dev/null; then echo "extra-substituters = https://aseipp-nix-cache.freetls.fastly.net" >> /etc/nix/nix.conf; fi + + # Install direnv and nix-direnv + - if command -v nix > /dev/null; then nix-env -iA nixpkgs.direnv nixpkgs.nix-direnv; fi + + # Allow .envrc + - if command -v nix > /dev/null; then direnv allow; fi + + # Set CARGO_HOME to a cacheable path + - export CARGO_HOME="$(git rev-parse --show-toplevel)/.gitlab-ci.d/cargo" + +ci: + stage: ci + image: nixos/nix:2.24.9 + script: + # Cache CI dependencies + - ./bin/nix-build-and-cache ci + + - direnv exec . engage + cache: + key: nix + paths: + - target + - .gitlab-ci.d + rules: + # CI on upstream runners (only available for maintainers) + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $IS_UPSTREAM_CI == "true" + # Manual CI on unprotected branches that are not MRs + - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_REF_PROTECTED == "false" + when: manual + # Manual CI on forks + - if: $IS_UPSTREAM_CI != "true" + when: manual + - if: $CI + interruptible: true + +artifacts: + stage: artifacts + image: nixos/nix:2.24.9 + script: + - ./bin/nix-build-and-cache just .#static-x86_64-linux-musl + - cp result/bin/conduit x86_64-linux-musl + + - mkdir -p target/release + - cp result/bin/conduit target/release + - direnv exec . cargo deb --no-build --no-strip + - mv target/debian/*.deb x86_64-linux-musl.deb + + # Since the OCI image package is based on the binary package, this has the + # fun side effect of uploading the normal binary too. Conduit users who are + # deploying with Nix can leverage this fact by adding our binary cache to + # their systems. + # + # Note that although we have an `oci-image-x86_64-linux-musl` + # output, we don't build it because it would be largely redundant to this + # one since it's all containerized anyway. + - ./bin/nix-build-and-cache just .#oci-image + - cp result oci-image-amd64.tar.gz + + - ./bin/nix-build-and-cache just .#static-aarch64-linux-musl + - cp result/bin/conduit aarch64-linux-musl + + - ./bin/nix-build-and-cache just .#oci-image-aarch64-linux-musl + - cp result oci-image-arm64v8.tar.gz + + - ./bin/nix-build-and-cache just .#book + # We can't just copy the symlink, we need to dereference it https://gitlab.com/gitlab-org/gitlab/-/issues/19746 + - cp -r --dereference result public + artifacts: + paths: + - x86_64-linux-musl + - aarch64-linux-musl + - x86_64-linux-musl.deb + - oci-image-amd64.tar.gz + - oci-image-arm64v8.tar.gz + - public + rules: + # CI required for all MRs + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + # Optional CI on forks + - if: $IS_UPSTREAM_CI != "true" + when: manual + allow_failure: true + - if: $CI + interruptible: true + +pages: + stage: publish + dependencies: + - artifacts + only: + - next + script: + - "true" + artifacts: + paths: + - public diff --git a/.gitlab/merge_request_templates/MR.md b/.gitlab/merge_request_templates/MR.md new file mode 100644 index 00000000..4210554b --- /dev/null +++ b/.gitlab/merge_request_templates/MR.md @@ -0,0 +1,8 @@ + + + +----------------------------------------------------------------------------- + +- [ ] I ran `cargo fmt`, `cargo clippy`, and `cargo test` +- [ ] I agree to release my code and all other changes of this MR under the Apache-2.0 license + diff --git a/.gitlab/route-map.yml b/.gitlab/route-map.yml new file mode 100644 index 00000000..cf31bd18 --- /dev/null +++ b/.gitlab/route-map.yml @@ -0,0 +1,3 @@ +# Docs: Map markdown to html files +- source: /docs/(.+)\.md/ + public: '\1.html' diff --git a/flake.lock b/flake.lock index 1f87b9b6..fc37d100 100644 --- a/flake.lock +++ b/flake.lock @@ -80,18 +80,18 @@ "complement": { "flake": false, "locked": { - "lastModified": 1741891349, - "narHash": "sha256-YvrzOWcX7DH1drp5SGa+E/fc7wN3hqFtPbqPjZpOu1Q=", - "owner": "girlbossceo", - "repo": "complement", - "rev": "e587b3df569cba411aeac7c20b6366d03c143745", - "type": "github" + "lastModified": 1745202855, + "narHash": "sha256-Jr4625Gp5SzL1teCAEkIBiwIhWBZ3UlPXa//I+6Ncyk=", + "ref": "morguldir/hs-ip", + "rev": "d44afcb142a0d1e3d877e3f2e38115910f7f062c", + "revCount": 860, + "type": "git", + "url": "https://forgejo.ellis.link/continuwuation/complement" }, "original": { - "owner": "girlbossceo", - "ref": "main", - "repo": "complement", - "type": "github" + "ref": "morguldir/hs-ip", + "type": "git", + "url": "https://forgejo.ellis.link/continuwuation/complement" } }, "crane": { diff --git a/flake.nix b/flake.nix index 49e860ed..8fbb851b 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ inputs = { attic.url = "github:zhaofengli/attic?ref=main"; cachix.url = "github:cachix/cachix?ref=master"; - complement = { url = "github:girlbossceo/complement?ref=main"; flake = false; }; + complement = { url = "git+https://forgejo.ellis.link/continuwuation/complement?ref=morguldir/hs-ip"; flake = false; }; crane = { url = "github:ipetkov/crane?ref=master"; }; fenix = { url = "github:nix-community/fenix?ref=main"; inputs.nixpkgs.follows = "nixpkgs"; }; flake-compat = { url = "github:edolstra/flake-compat?ref=master"; flake = false; };