From 9ce78ffae2947d2297877f3e05891a2075c5a53d Mon Sep 17 00:00:00 2001 From: morguldir Date: Tue, 15 Apr 2025 20:09:25 +0200 Subject: [PATCH 1/9] Revert "ci: Delete all old CI files" This reverts commit 35bffa5970e311a2b2ffa7c23ecd77121c36dfbe. --- .github/workflows/ci.yml | 717 +++++++++++++++++++ .github/workflows/docker-hub-description.yml | 41 ++ .github/workflows/documentation.yml | 104 +++ .github/workflows/release.yml | 118 +++ .gitlab-ci.yml | 152 ++++ .gitlab/merge_request_templates/MR.md | 8 + .gitlab/route-map.yml | 3 + 7 files changed, 1143 insertions(+) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/docker-hub-description.yml create mode 100644 .github/workflows/documentation.yml create mode 100644 .github/workflows/release.yml create mode 100644 .gitlab-ci.yml create mode 100644 .gitlab/merge_request_templates/MR.md create mode 100644 .gitlab/route-map.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..5043f23b --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,717 @@ +name: CI and Artifacts + +on: + pull_request: + push: + paths-ignore: + - '.gitlab-ci.yml' + - '.gitignore' + - 'renovate.json' + - 'debian/**' + - 'docker/**' + branches: + - main + tags: + - '*' + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +concurrency: + group: ${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +env: + # Required to make some things output color + TERM: ansi + # Publishing to my nix binary cache + ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} + # conduwuit.cachix.org + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + # Just in case incremental is still being set to true, speeds up CI + CARGO_INCREMENTAL: 0 + # Custom nix binary cache if fork is being used + ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} + ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} + # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps + NIX_CONFIG: | + show-trace = true + extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net https://nix-community.cachix.org https://crane.cachix.org + extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk= + experimental-features = nix-command flakes + extra-experimental-features = nix-command flakes + accept-flake-config = true + WEB_UPLOAD_SSH_USERNAME: ${{ secrets.WEB_UPLOAD_SSH_USERNAME }} + GH_REF_NAME: ${{ github.ref_name }} + WEBSERVER_DIR_NAME: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} + +permissions: {} + +jobs: + tests: + name: Test + runs-on: self-hosted + steps: + - name: Setup SSH web publish + env: + web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' + run: | + mkdir -p -v ~/.ssh + + echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + + chmod 600 ~/.ssh/id_ed25519 + + cat >>~/.ssh/config <> "$GITHUB_ENV" + + - name: Sync repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Tag comparison check + if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }} + run: | + # Tag mismatch with latest repo tag check to prevent potential downgrades + LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) + + if [ ${LATEST_TAG} != ${GH_REF_NAME} ]; then + echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' + echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' >> $GITHUB_STEP_SUMMARY + exit 1 + fi + + - name: Prepare build environment + run: | + echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + direnv allow + nix develop .#all-features --command true + + - name: Cache CI dependencies + run: | + bin/nix-build-and-cache ci + bin/nix-build-and-cache just '.#devShells.x86_64-linux.default' + bin/nix-build-and-cache just '.#devShells.x86_64-linux.all-features' + bin/nix-build-and-cache just '.#devShells.x86_64-linux.dynamic' + + # use rust-cache + - uses: Swatinem/rust-cache@v2 + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + #if: ${{ !startsWith(github.ref, 'refs/tags/') }} + with: + cache-all-crates: "true" + cache-on-failure: "true" + cache-targets: "true" + + - name: Run CI tests + env: + CARGO_PROFILE: "test" + run: | + direnv exec . engage > >(tee -a test_output.log) + + - name: Run Complement tests + env: + CARGO_PROFILE: "test" + run: | + # the nix devshell sets $COMPLEMENT_SRC, so "/dev/null" is no-op + direnv exec . bin/complement "/dev/null" complement_test_logs.jsonl complement_test_results.jsonl > >(tee -a test_output.log) + cp -v -f result complement_oci_image.tar.gz + + - name: Upload Complement OCI image + uses: actions/upload-artifact@v4 + with: + name: complement_oci_image.tar.gz + path: complement_oci_image.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Upload Complement logs + uses: actions/upload-artifact@v4 + with: + name: complement_test_logs.jsonl + path: complement_test_logs.jsonl + if-no-files-found: error + + - name: Upload Complement results + uses: actions/upload-artifact@v4 + with: + name: complement_test_results.jsonl + path: complement_test_results.jsonl + if-no-files-found: error + + - name: Diff Complement results with checked-in repo results + run: | + diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_diff_output.log) + + - name: Update Job Summary + env: + GH_JOB_STATUS: ${{ job.status }} + if: success() || failure() + run: | + if [ ${GH_JOB_STATUS} == 'success' ]; then + echo '# ✅ CI completed suwuccessfully' >> $GITHUB_STEP_SUMMARY + else + echo '# ❌ CI failed (last 100 lines of output)' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + tail -n 100 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + echo '# Complement diff results (last 100 lines)' >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + tail -n 100 complement_diff_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi + + build: + name: Build + runs-on: self-hosted + strategy: + matrix: + include: + - target: aarch64-linux-musl + - target: x86_64-linux-musl + steps: + - name: Sync repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setup SSH web publish + env: + web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' + run: | + mkdir -p -v ~/.ssh + + echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + + chmod 600 ~/.ssh/id_ed25519 + + cat >>~/.ssh/config <> "$GITHUB_ENV" + + - name: Prepare build environment + run: | + echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + direnv allow + nix develop .#all-features --command true --impure + + # use rust-cache + - uses: Swatinem/rust-cache@v2 + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + #if: ${{ !startsWith(github.ref, 'refs/tags/') }} + with: + cache-all-crates: "true" + cache-on-failure: "true" + cache-targets: "true" + + - name: Build static ${{ matrix.target }}-all-features + run: | + if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl" + fi + + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features + + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}.deb + mv -v target/release/conduwuit static-${{ matrix.target }} + mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb + + - name: Build static x86_64-linux-musl-all-features-x86_64-haswell-optimised + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-x86_64-linux-musl-all-features-x86_64-haswell-optimised + + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb + mv -v target/release/conduwuit static-x86_64-linux-musl-x86_64-haswell-optimised + mv -v target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb x86_64-linux-musl-x86_64-haswell-optimised.deb + + # quick smoke test of the x86_64 static release binary + - name: Quick smoke test the x86_64 static release binary + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + # GH actions default runners are x86_64 only + if file result/bin/conduwuit | grep x86-64; then + result/bin/conduwuit --version + result/bin/conduwuit --help + result/bin/conduwuit -Oserver_name="'$(date -u +%s).local'" -Odatabase_path="'/tmp/$(date -u +%s)'" --execute "server admin-notice awawawawawawawawawawa" --execute "server memory-usage" --execute "server shutdown" + fi + + - name: Build static debug ${{ matrix.target }}-all-features + run: | + if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl" + fi + + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug + + # > warning: dev profile is not supported and will be a hard error in the future. cargo-deb is for making releases, and it doesn't make sense to use it with dev profiles. + # so we need to coerce cargo-deb into thinking this is a release binary + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}-debug.deb + mv -v target/release/conduwuit static-${{ matrix.target }}-debug + mv -v target/release/${{ matrix.target }}-debug.deb ${{ matrix.target }}-debug.deb + + # quick smoke test of the x86_64 static debug binary + - name: Run x86_64 static debug binary + run: | + # GH actions default runners are x86_64 only + if file result/bin/conduwuit | grep x86-64; then + result/bin/conduwuit --version + fi + + # check validity of produced deb package, invalid debs will error on these commands + - name: Validate produced deb package + run: | + # List contents + dpkg-deb --contents ${{ matrix.target }}.deb + dpkg-deb --contents ${{ matrix.target }}-debug.deb + # List info + dpkg-deb --info ${{ matrix.target }}.deb + dpkg-deb --info ${{ matrix.target }}-debug.deb + + - name: Upload static-x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub + uses: actions/upload-artifact@v4 + if: ${{ matrix.target == 'x86_64-linux-musl' }} + with: + name: static-x86_64-linux-musl-x86_64-haswell-optimised + path: static-x86_64-linux-musl-x86_64-haswell-optimised + if-no-files-found: error + + - name: Upload static-${{ matrix.target }}-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: static-${{ matrix.target }} + path: static-${{ matrix.target }} + if-no-files-found: error + + - name: Upload static deb ${{ matrix.target }}-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: deb-${{ matrix.target }} + path: ${{ matrix.target }}.deb + if-no-files-found: error + compression-level: 0 + + - name: Upload static-x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + if [ ! -z $SSH_WEBSITE ]; then + chmod +x static-x86_64-linux-musl-x86_64-haswell-optimised + scp static-x86_64-linux-musl-x86_64-haswell-optimised website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-x86_64-linux-musl-x86_64-haswell-optimised + fi + + - name: Upload static-${{ matrix.target }}-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + chmod +x static-${{ matrix.target }} + scp static-${{ matrix.target }} website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }} + fi + + - name: Upload static deb x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp x86_64-linux-musl-x86_64-haswell-optimised.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/x86_64-linux-musl-x86_64-haswell-optimised.deb + fi + + - name: Upload static deb ${{ matrix.target }}-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp ${{ matrix.target }}.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}.deb + fi + + - name: Upload static-${{ matrix.target }}-debug-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: static-${{ matrix.target }}-debug + path: static-${{ matrix.target }}-debug + if-no-files-found: error + + - name: Upload static deb ${{ matrix.target }}-debug-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: deb-${{ matrix.target }}-debug + path: ${{ matrix.target }}-debug.deb + if-no-files-found: error + compression-level: 0 + + - name: Upload static-${{ matrix.target }}-debug-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp static-${{ matrix.target }}-debug website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }}-debug + fi + + - name: Upload static deb ${{ matrix.target }}-debug-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp ${{ matrix.target }}-debug.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}-debug.deb + fi + + - name: Build OCI image ${{ matrix.target }}-all-features + run: | + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features + + cp -v -f result oci-image-${{ matrix.target }}.tar.gz + + - name: Build OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + bin/nix-build-and-cache just .#oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised + + cp -v -f result oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + + - name: Build debug OCI image ${{ matrix.target }}-all-features + run: | + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug + + cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz + + - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub + if: ${{ matrix.target == 'x86_64-linux-musl' }} + uses: actions/upload-artifact@v4 + with: + name: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised + path: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + if-no-files-found: error + compression-level: 0 + - name: Upload OCI image ${{ matrix.target }}-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: oci-image-${{ matrix.target }} + path: oci-image-${{ matrix.target }}.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Upload OCI image ${{ matrix.target }}-debug-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: oci-image-${{ matrix.target }}-debug + path: oci-image-${{ matrix.target }}-debug.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz to webserver + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + fi + + - name: Upload OCI image ${{ matrix.target }}-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp oci-image-${{ matrix.target }}.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}.tar.gz + fi + + - name: Upload OCI image ${{ matrix.target }}-debug-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp oci-image-${{ matrix.target }}-debug.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}-debug.tar.gz + fi + + variables: + outputs: + github_repository: ${{ steps.var.outputs.github_repository }} + runs-on: self-hosted + steps: + - name: Setting global variables + uses: actions/github-script@v7 + id: var + with: + script: | + core.setOutput('github_repository', '${{ github.repository }}'.toLowerCase()) + docker: + name: Docker publish + runs-on: self-hosted + needs: [build, variables, tests] + permissions: + packages: write + contents: read + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' + env: + DOCKER_HUB_REPO: docker.io/${{ needs.variables.outputs.github_repository }} + GHCR_REPO: ghcr.io/${{ needs.variables.outputs.github_repository }} + GLCR_REPO: registry.gitlab.com/conduwuit/conduwuit + UNIQUE_TAG: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} + BRANCH_TAG: ${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} + + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} + GHCR_ENABLED: "${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) && 'true' || 'false' }}" + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Docker Hub + if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} + uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ vars.DOCKER_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitLab Container Registry + if: ${{ (vars.GITLAB_USERNAME != '') && (env.GITLAB_TOKEN != '') }} + uses: docker/login-action@v3 + with: + registry: registry.gitlab.com + username: ${{ vars.GITLAB_USERNAME }} + password: ${{ secrets.GITLAB_TOKEN }} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + pattern: "oci*" + + - name: Move OCI images into position + run: | + mv -v oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised/*.tar.gz oci-image-amd64-haswell-optimised.tar.gz + mv -v oci-image-x86_64-linux-musl/*.tar.gz oci-image-amd64.tar.gz + mv -v oci-image-aarch64-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz + mv -v oci-image-x86_64-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz + mv -v oci-image-aarch64-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz + + - name: Load and push amd64 haswell image + run: | + docker load -i oci-image-amd64-haswell-optimised.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker push ${GHCR_REPO}:${UNIQUE_TAG}-haswell + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker push ${GLCR_REPO}:${UNIQUE_TAG}-haswell + fi + + - name: Load and push amd64 image + run: | + docker load -i oci-image-amd64.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + fi + + - name: Load and push arm64 image + run: | + docker load -i oci-image-arm64v8.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 + fi + + - name: Load and push amd64 debug image + run: | + docker load -i oci-image-amd64-debug.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + + - name: Load and push arm64 debug image + run: | + docker load -i oci-image-arm64v8-debug.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + fi + + - name: Create Docker haswell manifests + run: | + # Dockerhub Container Registry + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + fi + # GitHub Container Registry + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell + fi + # GitLab Container Registry + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell + fi + + - name: Create Docker combined manifests + run: | + # Dockerhub Container Registry + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + fi + # GitHub Container Registry + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${GHCR_REPO}:${BRANCH_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + fi + # GitLab Container Registry + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${GLCR_REPO}:${BRANCH_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + fi + + - name: Create Docker combined debug manifests + run: | + # Dockerhub Container Registry + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + fi + # GitHub Container Registry + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + # GitLab Container Registry + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + + - name: Push manifests to Docker registries + run: | + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG} + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG} + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell + fi + if [ $GHCR_ENABLED = "true" ]; then + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG} + docker manifest push ${GHCR_REPO}:${BRANCH_TAG} + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-debug + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-haswell + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG} + docker manifest push ${GLCR_REPO}:${BRANCH_TAG} + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-debug + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-haswell + fi + + - name: Add Image Links to Job Summary + run: | + if [ ! -z $DOCKERHUB_TOKEN ]; then + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY + fi + if [ $GHCR_ENABLED = "true" ]; then + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY + fi + if [ ! -z $GITLAB_TOKEN ]; then + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/docker-hub-description.yml b/.github/workflows/docker-hub-description.yml new file mode 100644 index 00000000..b4f142db --- /dev/null +++ b/.github/workflows/docker-hub-description.yml @@ -0,0 +1,41 @@ +name: Update Docker Hub Description + +on: + push: + branches: + - main + paths: + - README.md + - .github/workflows/docker-hub-description.yml + + workflow_dispatch: + +jobs: + dockerHubDescription: + runs-on: ubuntu-latest + if: ${{ (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' && (vars.DOCKER_USERNAME != '') }} + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setting variables + uses: actions/github-script@v7 + id: var + with: + script: | + const githubRepo = '${{ github.repository }}'.toLowerCase() + const repoId = githubRepo.split('/')[1] + + core.setOutput('github_repository', githubRepo) + const dockerRepo = '${{ vars.DOCKER_USERNAME }}'.toLowerCase() + '/' + repoId + core.setOutput('docker_repo', dockerRepo) + + - name: Docker Hub Description + uses: peter-evans/dockerhub-description@v4 + with: + username: ${{ vars.DOCKER_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + repository: ${{ steps.var.outputs.docker_repo }} + short-description: ${{ github.event.repository.description }} + enable-url-completion: true diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml new file mode 100644 index 00000000..b5b4ff46 --- /dev/null +++ b/.github/workflows/documentation.yml @@ -0,0 +1,104 @@ +name: Documentation and GitHub Pages + +on: + pull_request: + push: + branches: + - main + tags: + - '*' + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +env: + # Required to make some things output color + TERM: ansi + # Publishing to my nix binary cache + ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} + # conduwuit.cachix.org + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + # Custom nix binary cache if fork is being used + ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} + ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} + # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps + NIX_CONFIG: | + show-trace = true + extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net https://nix-community.cachix.org https://crane.cachix.org + extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= cache.lix.systems:aBnZUw8zA7H35Cz2RyKFVs3H4PlGTLawyY5KRbvJR8o= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk= + experimental-features = nix-command flakes + extra-experimental-features = nix-command flakes + accept-flake-config = true + +# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued. +# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete. +concurrency: + group: "pages" + cancel-in-progress: false + +permissions: {} + +jobs: + docs: + name: Documentation and GitHub Pages + runs-on: self-hosted + + permissions: + pages: write + id-token: write + + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + + steps: + - name: Sync repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Setup GitHub Pages + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') + uses: actions/configure-pages@v5 + + - name: Prepare build environment + run: | + echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + direnv allow + nix develop --command true + + - name: Cache CI dependencies + run: | + bin/nix-build-and-cache ci + + - name: Run lychee and markdownlint + run: | + direnv exec . engage just lints lychee + direnv exec . engage just lints markdownlint + + - name: Build documentation (book) + run: | + bin/nix-build-and-cache just .#book + + cp -r --dereference result public + chmod u+w -R public + + - name: Upload generated documentation (book) as normal artifact + uses: actions/upload-artifact@v4 + with: + name: public + path: public + if-no-files-found: error + # don't compress again + compression-level: 0 + + - name: Upload generated documentation (book) as GitHub Pages artifact + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') + uses: actions/upload-pages-artifact@v3 + with: + path: public + + - name: Deploy to GitHub Pages + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main') && (github.event_name != 'pull_request') + id: deployment + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..cfe72d2a --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,118 @@ +name: Upload Release Assets + +on: + release: + types: [published] + workflow_dispatch: + inputs: + tag: + description: 'Tag to release' + required: true + type: string + action_id: + description: 'Action ID of the CI run' + required: true + type: string + +permissions: {} + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: write + env: + GH_EVENT_NAME: ${{ github.event_name }} + GH_EVENT_INPUTS_ACTION_ID: ${{ github.event.inputs.action_id }} + GH_EVENT_INPUTS_TAG: ${{ github.event.inputs.tag }} + GH_REPOSITORY: ${{ github.repository }} + GH_SHA: ${{ github.sha }} + GH_TAG: ${{ github.event.release.tag_name }} + + steps: + - name: get latest ci id + id: get_ci_id + env: + GH_TOKEN: ${{ github.token }} + run: | + if [ "${GH_EVENT_NAME}" == "workflow_dispatch" ]; then + id="${GH_EVENT_INPUTS_ACTION_ID}" + tag="${GH_EVENT_INPUTS_TAG}" + else + # get all runs of the ci workflow + json=$(gh api "repos/${GH_REPOSITORY}/actions/workflows/ci.yml/runs") + + # find first run that is github sha and status is completed + id=$(echo "$json" | jq ".workflow_runs[] | select(.head_sha == \"${GH_SHA}\" and .status == \"completed\") | .id" | head -n 1) + + if [ ! "$id" ]; then + echo "No completed runs found" + echo "ci_id=0" >> "$GITHUB_OUTPUT" + exit 0 + fi + + tag="${GH_TAG}" + fi + + echo "ci_id=$id" >> "$GITHUB_OUTPUT" + echo "tag=$tag" >> "$GITHUB_OUTPUT" + + - name: get latest ci artifacts + if: steps.get_ci_id.outputs.ci_id != 0 + uses: actions/download-artifact@v4 + env: + GH_TOKEN: ${{ github.token }} + with: + merge-multiple: true + run-id: ${{ steps.get_ci_id.outputs.ci_id }} + github-token: ${{ github.token }} + + - run: | + ls + + - name: upload release assets + if: steps.get_ci_id.outputs.ci_id != 0 + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ steps.get_ci_id.outputs.tag }} + run: | + for file in $(find . -type f); do + case "$file" in + *json*) echo "Skipping $file...";; + *) echo "Uploading $file..."; gh release upload $TAG "$file" --clobber --repo="${GH_REPOSITORY}" || echo "Something went wrong, skipping.";; + esac + done + + - name: upload release assets to website + if: steps.get_ci_id.outputs.ci_id != 0 + env: + TAG: ${{ steps.get_ci_id.outputs.tag }} + run: | + mkdir -p -v ~/.ssh + + echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + + chmod 600 ~/.ssh/id_ed25519 + + cat >>~/.ssh/config < /dev/null; then echo "experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-experimental-features = nix-command flakes" >> /etc/nix/nix.conf; fi + # Accept flake config from "untrusted" users + - if command -v nix > /dev/null; then echo "accept-flake-config = true" >> /etc/nix/nix.conf; fi + + # Add conduwuit binary cache + - if command -v nix > /dev/null; then echo "extra-substituters = https://attic.kennel.juneis.dog/conduwuit" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE=" >> /etc/nix/nix.conf; fi + + - if command -v nix > /dev/null; then echo "extra-substituters = https://attic.kennel.juneis.dog/conduit" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk=" >> /etc/nix/nix.conf; fi + + # Add alternate binary cache + - if command -v nix > /dev/null && [ -n "$ATTIC_ENDPOINT" ]; then echo "extra-substituters = $ATTIC_ENDPOINT" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null && [ -n "$ATTIC_PUBLIC_KEY" ]; then echo "extra-trusted-public-keys = $ATTIC_PUBLIC_KEY" >> /etc/nix/nix.conf; fi + + # Add crane binary cache + - if command -v nix > /dev/null; then echo "extra-substituters = https://crane.cachix.org" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk=" >> /etc/nix/nix.conf; fi + + # Add nix-community binary cache + - if command -v nix > /dev/null; then echo "extra-substituters = https://nix-community.cachix.org" >> /etc/nix/nix.conf; fi + - if command -v nix > /dev/null; then echo "extra-trusted-public-keys = nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs=" >> /etc/nix/nix.conf; fi + + - if command -v nix > /dev/null; then echo "extra-substituters = https://aseipp-nix-cache.freetls.fastly.net" >> /etc/nix/nix.conf; fi + + # Install direnv and nix-direnv + - if command -v nix > /dev/null; then nix-env -iA nixpkgs.direnv nixpkgs.nix-direnv; fi + + # Allow .envrc + - if command -v nix > /dev/null; then direnv allow; fi + + # Set CARGO_HOME to a cacheable path + - export CARGO_HOME="$(git rev-parse --show-toplevel)/.gitlab-ci.d/cargo" + +ci: + stage: ci + image: nixos/nix:2.24.9 + script: + # Cache CI dependencies + - ./bin/nix-build-and-cache ci + + - direnv exec . engage + cache: + key: nix + paths: + - target + - .gitlab-ci.d + rules: + # CI on upstream runners (only available for maintainers) + - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $IS_UPSTREAM_CI == "true" + # Manual CI on unprotected branches that are not MRs + - if: $CI_PIPELINE_SOURCE != "merge_request_event" && $CI_COMMIT_REF_PROTECTED == "false" + when: manual + # Manual CI on forks + - if: $IS_UPSTREAM_CI != "true" + when: manual + - if: $CI + interruptible: true + +artifacts: + stage: artifacts + image: nixos/nix:2.24.9 + script: + - ./bin/nix-build-and-cache just .#static-x86_64-linux-musl + - cp result/bin/conduit x86_64-linux-musl + + - mkdir -p target/release + - cp result/bin/conduit target/release + - direnv exec . cargo deb --no-build --no-strip + - mv target/debian/*.deb x86_64-linux-musl.deb + + # Since the OCI image package is based on the binary package, this has the + # fun side effect of uploading the normal binary too. Conduit users who are + # deploying with Nix can leverage this fact by adding our binary cache to + # their systems. + # + # Note that although we have an `oci-image-x86_64-linux-musl` + # output, we don't build it because it would be largely redundant to this + # one since it's all containerized anyway. + - ./bin/nix-build-and-cache just .#oci-image + - cp result oci-image-amd64.tar.gz + + - ./bin/nix-build-and-cache just .#static-aarch64-linux-musl + - cp result/bin/conduit aarch64-linux-musl + + - ./bin/nix-build-and-cache just .#oci-image-aarch64-linux-musl + - cp result oci-image-arm64v8.tar.gz + + - ./bin/nix-build-and-cache just .#book + # We can't just copy the symlink, we need to dereference it https://gitlab.com/gitlab-org/gitlab/-/issues/19746 + - cp -r --dereference result public + artifacts: + paths: + - x86_64-linux-musl + - aarch64-linux-musl + - x86_64-linux-musl.deb + - oci-image-amd64.tar.gz + - oci-image-arm64v8.tar.gz + - public + rules: + # CI required for all MRs + - if: $CI_PIPELINE_SOURCE == "merge_request_event" + # Optional CI on forks + - if: $IS_UPSTREAM_CI != "true" + when: manual + allow_failure: true + - if: $CI + interruptible: true + +pages: + stage: publish + dependencies: + - artifacts + only: + - next + script: + - "true" + artifacts: + paths: + - public diff --git a/.gitlab/merge_request_templates/MR.md b/.gitlab/merge_request_templates/MR.md new file mode 100644 index 00000000..4210554b --- /dev/null +++ b/.gitlab/merge_request_templates/MR.md @@ -0,0 +1,8 @@ + + + +----------------------------------------------------------------------------- + +- [ ] I ran `cargo fmt`, `cargo clippy`, and `cargo test` +- [ ] I agree to release my code and all other changes of this MR under the Apache-2.0 license + diff --git a/.gitlab/route-map.yml b/.gitlab/route-map.yml new file mode 100644 index 00000000..cf31bd18 --- /dev/null +++ b/.gitlab/route-map.yml @@ -0,0 +1,3 @@ +# Docs: Map markdown to html files +- source: /docs/(.+)\.md/ + public: '\1.html' From 5516ff93d1b6439b3d0d1d564ba294057d48af61 Mon Sep 17 00:00:00 2001 From: morguldir Date: Wed, 16 Apr 2025 00:38:58 +0200 Subject: [PATCH 2/9] ci: allow running with docker again, add forgejo specific ci.yml --- .forgejo/workflows/ci.yml | 747 ++++++++++++++++++++++++++++++++++++++ .github/workflows/ci.yml | 27 +- 2 files changed, 773 insertions(+), 1 deletion(-) create mode 100644 .forgejo/workflows/ci.yml diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml new file mode 100644 index 00000000..3978a894 --- /dev/null +++ b/.forgejo/workflows/ci.yml @@ -0,0 +1,747 @@ +name: CI and Artifacts + +on: + pull_request: + push: + paths-ignore: + - '.gitlab-ci.yml' + - '.gitignore' + - 'renovate.json' + - 'debian/**' + - 'docker/**' + branches: + - main + tags: + - '*' + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +concurrency: + group: ${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +env: + # Required to make some things output color + TERM: ansi + # Publishing to my nix binary cache + ATTIC_TOKEN: ${{ secrets.ATTIC_TOKEN }} + # conduwuit.cachix.org + CACHIX_AUTH_TOKEN: ${{ secrets.CACHIX_AUTH_TOKEN }} + # Just in case incremental is still being set to true, speeds up CI + CARGO_INCREMENTAL: 0 + # Custom nix binary cache if fork is being used + ATTIC_ENDPOINT: ${{ vars.ATTIC_ENDPOINT }} + ATTIC_PUBLIC_KEY: ${{ vars.ATTIC_PUBLIC_KEY }} + # Get error output from nix that we can actually use, and use our binary caches for the earlier CI steps + NIX_CONFIG: | + show-trace = true + extra-substituters = https://attic.kennel.juneis.dog/conduwuit https://attic.kennel.juneis.dog/conduit https://conduwuit.cachix.org https://aseipp-nix-cache.freetls.fastly.net https://nix-community.cachix.org https://crane.cachix.org + extra-trusted-public-keys = conduit:eEKoUwlQGDdYmAI/Q/0slVlegqh/QmAvQd7HBSm21Wk= conduwuit:BbycGUgTISsltcmH0qNjFR9dbrQNYgdIAcmViSGoVTE= conduwuit.cachix.org-1:MFRm6jcnfTf0jSAbmvLfhO3KBMt4px+1xaereWXp8Xg= nix-community.cachix.org-1:mB9FSh9qf2dCimDSUo8Zy7bkq5CX+/rkCWyvRCYg3Fs= crane.cachix.org-1:8Scfpmn9w+hGdXH/Q9tTLiYAE/2dnJYRJP7kl80GuRk= + experimental-features = nix-command flakes + extra-experimental-features = nix-command flakes + accept-flake-config = true + WEB_UPLOAD_SSH_USERNAME: ${{ secrets.WEB_UPLOAD_SSH_USERNAME }} + GH_REF_NAME: ${{ github.ref_name }} + WEBSERVER_DIR_NAME: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} + +permissions: {} + +jobs: + tests: + name: Test + runs-on: self-hosted + volumes: + - /nix/store:/nix/store + steps: + - name: Setup SSH web publish + env: + web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' + run: | + mkdir -p -v ~/.ssh + + echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + + chmod 600 ~/.ssh/id_ed25519 + + cat >>~/.ssh/config <> "$GITHUB_ENV" + + - name: Sync repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Tag comparison check + if: ${{ startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') }} + run: | + # Tag mismatch with latest repo tag check to prevent potential downgrades + LATEST_TAG=$(git describe --tags `git rev-list --tags --max-count=1`) + + if [ ${LATEST_TAG} != ${GH_REF_NAME} ]; then + echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' + echo '# WARNING: Attempting to run this workflow for a tag that is not the latest repo tag. Aborting.' >> $GITHUB_STEP_SUMMARY + exit 1 + fi + + #- uses: nixbuild/nix-quick-install-action@master + - uses: https://github.com/cachix/install-nix-action@v31.1.0 + if: ${{ env.ACT }} + + - name: Nom + if: ${{ env.ACT }} + run: | + echo "aaaa" + nix profile install nixpkgs#nix-output-monitor + sudo cp $(which nom) /usr/bin/nom-nix + file $(which nom) + lsblk + + #- uses: https://github.com/cachix/install-nix-action@v27 + # if: ${{ ! env.ACT }} + + - name: Prepare build environment + run: | + if ! type nom &> /dev/null; then + sudo cp $(which nix) /usr/bin/nom-or-nix + fi + echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + direnv allow + set -x + file $(which nom-nix) + nom develop .#all-features --command true + + - name: Cache CI dependencies + run: | + bin/nix-build-and-cache ci + bin/nix-build-and-cache just '.#devShells.x86_64-linux.default' + bin/nix-build-and-cache just '.#devShells.x86_64-linux.all-features' + bin/nix-build-and-cache just '.#devShells.x86_64-linux.dynamic' + + # use rust-cache + - uses: https://github.com/Swatinem/rust-cache@v2 + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + #if: ${{ !startsWith(github.ref, 'refs/tags/') }} + with: + cache-all-crates: "true" + cache-on-failure: "true" + cache-targets: "true" + + - name: Run CI tests + env: + CARGO_PROFILE: "test" + run: | + direnv exec . engage > >(tee -a test_output.log) + + - name: Run Complement tests + env: + CARGO_PROFILE: "test" + run: | + # the nix devshell sets $COMPLEMENT_SRC, so "/dev/null" is no-op + direnv exec . bin/complement "/dev/null" complement_test_logs.jsonl complement_test_results.jsonl > >(tee -a test_output.log) + cp -v -f result complement_oci_image.tar.gz + + - name: Upload Complement OCI image + uses: actions/upload-artifact@v4 + with: + name: complement_oci_image.tar.gz + path: complement_oci_image.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Upload Complement logs + uses: actions/upload-artifact@v4 + with: + name: complement_test_logs.jsonl + path: complement_test_logs.jsonl + if-no-files-found: error + + - name: Upload Complement results + uses: actions/upload-artifact@v4 + with: + name: complement_test_results.jsonl + path: complement_test_results.jsonl + if-no-files-found: error + + - name: Diff Complement results with checked-in repo results + run: | + diff -u --color=always tests/test_results/complement/test_results.jsonl complement_test_results.jsonl > >(tee -a complement_diff_output.log) + + - name: Update Job Summary + env: + GH_JOB_STATUS: ${{ job.status }} + if: success() || failure() + run: | + if [ ${GH_JOB_STATUS} == 'success' ]; then + echo '# ✅ CI completed suwuccessfully' >> $GITHUB_STEP_SUMMARY + else + echo '# ❌ CI failed (last 100 lines of output)' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + tail -n 100 test_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + echo '# Complement diff results (last 100 lines)' >> $GITHUB_STEP_SUMMARY + echo '```diff' >> $GITHUB_STEP_SUMMARY + tail -n 100 complement_diff_output.log | sed 's/\x1b\[[0-9;]*m//g' >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + fi + + build: + name: Build + runs-on: self-hosted + volumes: + - /nix/store:/nix/store + strategy: + matrix: + include: + - target: aarch64-linux-musl + - target: x86_64-linux-musl + steps: + - name: Sync repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + #- uses: nixbuild/nix-quick-install-action@master + - uses: https://github.com/cachix/install-nix-action@v31.1.0 + + - name: Setup SSH web publish + env: + web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && (env.web_upload_ssh_private_key != '') && github.event.pull_request.user.login != 'renovate[bot]' + run: | + mkdir -p -v ~/.ssh + + echo "${{ secrets.WEB_UPLOAD_SSH_KNOWN_HOSTS }}" >> ~/.ssh/known_hosts + echo "${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }}" >> ~/.ssh/id_ed25519 + + chmod 600 ~/.ssh/id_ed25519 + + cat >>~/.ssh/config <> "$GITHUB_ENV" + + - name: Prepare build environment + run: | + echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + direnv allow + nix develop .#all-features --command true --impure + + # use rust-cache + - uses: https://github.com/Swatinem/rust-cache@v2 + # we want a fresh-state when we do releases/tags to avoid potential cache poisoning attacks impacting + # releases and tags + #if: ${{ !startsWith(github.ref, 'refs/tags/') }} + with: + cache-all-crates: "true" + cache-on-failure: "true" + cache-targets: "true" + + - name: Build static ${{ matrix.target }}-all-features + run: | + if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl" + fi + + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features + + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}.deb + mv -v target/release/conduwuit static-${{ matrix.target }} + mv -v target/release/${{ matrix.target }}.deb ${{ matrix.target }}.deb + + - name: Build static x86_64-linux-musl-all-features-x86_64-haswell-optimised + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-x86_64-linux-musl-all-features-x86_64-haswell-optimised + + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb + mv -v target/release/conduwuit static-x86_64-linux-musl-x86_64-haswell-optimised + mv -v target/release/x86_64-linux-musl-x86_64-haswell-optimised.deb x86_64-linux-musl-x86_64-haswell-optimised.deb + + # quick smoke test of the x86_64 static release binary + - name: Quick smoke test the x86_64 static release binary + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + # GH actions default runners are x86_64 only + if file result/bin/conduwuit | grep x86-64; then + result/bin/conduwuit --version + result/bin/conduwuit --help + result/bin/conduwuit -Oserver_name="'$(date -u +%s).local'" -Odatabase_path="'/tmp/$(date -u +%s)'" --execute "server admin-notice awawawawawawawawawawa" --execute "server memory-usage" --execute "server shutdown" + fi + + - name: Build static debug ${{ matrix.target }}-all-features + run: | + if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="x86_64-unknown-linux-musl" + elif [[ ${{ matrix.target }} == "aarch64-linux-musl" ]] + then + CARGO_DEB_TARGET_TUPLE="aarch64-unknown-linux-musl" + fi + + SOURCE_DATE_EPOCH=$(git log -1 --pretty=%ct) + + bin/nix-build-and-cache just .#static-${{ matrix.target }}-all-features-debug + + # > warning: dev profile is not supported and will be a hard error in the future. cargo-deb is for making releases, and it doesn't make sense to use it with dev profiles. + # so we need to coerce cargo-deb into thinking this is a release binary + mkdir -v -p target/release/ + mkdir -v -p target/$CARGO_DEB_TARGET_TUPLE/release/ + cp -v -f result/bin/conduwuit target/release/conduwuit + cp -v -f result/bin/conduwuit target/$CARGO_DEB_TARGET_TUPLE/release/conduwuit + direnv exec . cargo deb --verbose --no-build --no-strip -p conduwuit --target=$CARGO_DEB_TARGET_TUPLE --output target/release/${{ matrix.target }}-debug.deb + mv -v target/release/conduwuit static-${{ matrix.target }}-debug + mv -v target/release/${{ matrix.target }}-debug.deb ${{ matrix.target }}-debug.deb + + # quick smoke test of the x86_64 static debug binary + - name: Run x86_64 static debug binary + run: | + # GH actions default runners are x86_64 only + if file result/bin/conduwuit | grep x86-64; then + result/bin/conduwuit --version + fi + + # check validity of produced deb package, invalid debs will error on these commands + - name: Validate produced deb package + run: | + # List contents + dpkg-deb --contents ${{ matrix.target }}.deb + dpkg-deb --contents ${{ matrix.target }}-debug.deb + # List info + dpkg-deb --info ${{ matrix.target }}.deb + dpkg-deb --info ${{ matrix.target }}-debug.deb + + - name: Upload static-x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub + uses: actions/upload-artifact@v4 + if: ${{ matrix.target == 'x86_64-linux-musl' }} + with: + name: static-x86_64-linux-musl-x86_64-haswell-optimised + path: static-x86_64-linux-musl-x86_64-haswell-optimised + if-no-files-found: error + + - name: Upload static-${{ matrix.target }}-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: static-${{ matrix.target }} + path: static-${{ matrix.target }} + if-no-files-found: error + + - name: Upload static deb ${{ matrix.target }}-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: deb-${{ matrix.target }} + path: ${{ matrix.target }}.deb + if-no-files-found: error + compression-level: 0 + + - name: Upload static-x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + if [ ! -z $SSH_WEBSITE ]; then + chmod +x static-x86_64-linux-musl-x86_64-haswell-optimised + scp static-x86_64-linux-musl-x86_64-haswell-optimised website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-x86_64-linux-musl-x86_64-haswell-optimised + fi + + - name: Upload static-${{ matrix.target }}-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + chmod +x static-${{ matrix.target }} + scp static-${{ matrix.target }} website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }} + fi + + - name: Upload static deb x86_64-linux-musl-all-features-x86_64-haswell-optimised to webserver + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp x86_64-linux-musl-x86_64-haswell-optimised.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/x86_64-linux-musl-x86_64-haswell-optimised.deb + fi + + - name: Upload static deb ${{ matrix.target }}-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp ${{ matrix.target }}.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}.deb + fi + + - name: Upload static-${{ matrix.target }}-debug-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: static-${{ matrix.target }}-debug + path: static-${{ matrix.target }}-debug + if-no-files-found: error + + - name: Upload static deb ${{ matrix.target }}-debug-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: deb-${{ matrix.target }}-debug + path: ${{ matrix.target }}-debug.deb + if-no-files-found: error + compression-level: 0 + + - name: Upload static-${{ matrix.target }}-debug-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp static-${{ matrix.target }}-debug website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/static-${{ matrix.target }}-debug + fi + + - name: Upload static deb ${{ matrix.target }}-debug-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp ${{ matrix.target }}-debug.deb website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/${{ matrix.target }}-debug.deb + fi + + - name: Build OCI image ${{ matrix.target }}-all-features + run: | + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features + + cp -v -f result oci-image-${{ matrix.target }}.tar.gz + + - name: Build OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + bin/nix-build-and-cache just .#oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised + + cp -v -f result oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + + - name: Build debug OCI image ${{ matrix.target }}-all-features + run: | + bin/nix-build-and-cache just .#oci-image-${{ matrix.target }}-all-features-debug + + cp -v -f result oci-image-${{ matrix.target }}-debug.tar.gz + + - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub + if: ${{ matrix.target == 'x86_64-linux-musl' }} + uses: actions/upload-artifact@v4 + with: + name: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised + path: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + if-no-files-found: error + compression-level: 0 + - name: Upload OCI image ${{ matrix.target }}-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: oci-image-${{ matrix.target }} + path: oci-image-${{ matrix.target }}.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Upload OCI image ${{ matrix.target }}-debug-all-features to GitHub + uses: actions/upload-artifact@v4 + with: + name: oci-image-${{ matrix.target }}-debug + path: oci-image-${{ matrix.target }}-debug.tar.gz + if-no-files-found: error + compression-level: 0 + + - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz to webserver + if: ${{ matrix.target == 'x86_64-linux-musl' }} + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz + fi + + - name: Upload OCI image ${{ matrix.target }}-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp oci-image-${{ matrix.target }}.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}.tar.gz + fi + + - name: Upload OCI image ${{ matrix.target }}-debug-all-features to webserver + run: | + if [ ! -z $SSH_WEBSITE ]; then + scp oci-image-${{ matrix.target }}-debug.tar.gz website:/var/www/girlboss.ceo/~strawberry/conduwuit/ci-bins/${WEBSERVER_DIR_NAME}/oci-image-${{ matrix.target }}-debug.tar.gz + fi + + variables: + outputs: + github_repository: ${{ steps.var.outputs.github_repository }} + runs-on: self-hosted + steps: + - name: Setting global variables + uses: https://github.com/actions/github-script@v7 + id: var + with: + script: | + core.setOutput('github_repository', '${{ github.repository }}'.toLowerCase()) + docker: + name: Docker publish + runs-on: self-hosted + needs: [build, variables, tests] + permissions: + packages: write + contents: read + if: (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main' || (github.event.pull_request.draft != true)) && github.event.pull_request.user.login != 'renovate[bot]' + env: + DOCKER_HUB_REPO: docker.io/${{ needs.variables.outputs.github_repository }} + GHCR_REPO: ghcr.io/${{ needs.variables.outputs.github_repository }} + GLCR_REPO: registry.gitlab.com/conduwuit/conduwuit + UNIQUE_TAG: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} + BRANCH_TAG: ${{ (startsWith(github.ref, 'refs/tags/v') && !endsWith(github.ref, '-rc') && 'latest') || (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }} + + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} + GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }} + GHCR_ENABLED: "${{ (github.event_name != 'pull_request' || github.event.pull_request.head.repo.fork == false) && 'true' || 'false' }}" + steps: + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Login to Docker Hub + if: ${{ (vars.DOCKER_USERNAME != '') && (env.DOCKERHUB_TOKEN != '') }} + uses: docker/login-action@v3 + with: + registry: docker.io + username: ${{ vars.DOCKER_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Login to GitLab Container Registry + if: ${{ (vars.GITLAB_USERNAME != '') && (env.GITLAB_TOKEN != '') }} + uses: docker/login-action@v3 + with: + registry: registry.gitlab.com + username: ${{ vars.GITLAB_USERNAME }} + password: ${{ secrets.GITLAB_TOKEN }} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + pattern: "oci*" + + - name: Move OCI images into position + run: | + mv -v oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised/*.tar.gz oci-image-amd64-haswell-optimised.tar.gz + mv -v oci-image-x86_64-linux-musl/*.tar.gz oci-image-amd64.tar.gz + mv -v oci-image-aarch64-linux-musl/*.tar.gz oci-image-arm64v8.tar.gz + mv -v oci-image-x86_64-linux-musl-debug/*.tar.gz oci-image-amd64-debug.tar.gz + mv -v oci-image-aarch64-linux-musl-debug/*.tar.gz oci-image-arm64v8-debug.tar.gz + + - name: Load and push amd64 haswell image + run: | + docker load -i oci-image-amd64-haswell-optimised.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker push ${GHCR_REPO}:${UNIQUE_TAG}-haswell + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker push ${GLCR_REPO}:${UNIQUE_TAG}-haswell + fi + + - name: Load and push amd64 image + run: | + docker load -i oci-image-amd64.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + fi + + - name: Load and push arm64 image + run: | + docker load -i oci-image-arm64v8.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 + docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 + fi + + - name: Load and push amd64 debug image + run: | + docker load -i oci-image-amd64-debug.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker push ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + + - name: Load and push arm64 debug image + run: | + docker load -i oci-image-arm64v8-debug.tar.gz + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug + fi + if [ $GHCR_ENABLED = "true" ]; then + docker tag $(docker images -q conduwuit:main) ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker tag $(docker images -q conduwuit:main) ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + docker push ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug + fi + + - name: Create Docker haswell manifests + run: | + # Dockerhub Container Registry + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + fi + # GitHub Container Registry + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-haswell --amend ${GHCR_REPO}:${UNIQUE_TAG}-haswell + fi + # GitLab Container Registry + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-haswell --amend ${GLCR_REPO}:${UNIQUE_TAG}-haswell + fi + + - name: Create Docker combined manifests + run: | + # Dockerhub Container Registry + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG} --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64 + fi + # GitHub Container Registry + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${GHCR_REPO}:${BRANCH_TAG} --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64 + fi + # GitLab Container Registry + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + docker manifest create ${GLCR_REPO}:${BRANCH_TAG} --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8 --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64 + fi + + - name: Create Docker combined debug manifests + run: | + # Dockerhub Container Registry + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest create ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-amd64-debug + fi + # GitHub Container Registry + if [ $GHCR_ENABLED = "true" ]; then + docker manifest create ${GHCR_REPO}:${UNIQUE_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${GHCR_REPO}:${BRANCH_TAG}-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GHCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + # GitLab Container Registry + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest create ${GLCR_REPO}:${UNIQUE_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + docker manifest create ${GLCR_REPO}:${BRANCH_TAG}-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-arm64v8-debug --amend ${GLCR_REPO}:${UNIQUE_TAG}-amd64-debug + fi + + - name: Push manifests to Docker registries + run: | + if [ ! -z $DOCKERHUB_TOKEN ]; then + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG} + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG} + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-debug + docker manifest push ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${DOCKER_HUB_REPO}:${BRANCH_TAG}-haswell + fi + if [ $GHCR_ENABLED = "true" ]; then + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG} + docker manifest push ${GHCR_REPO}:${BRANCH_TAG} + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-debug + docker manifest push ${GHCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${GHCR_REPO}:${BRANCH_TAG}-haswell + fi + if [ ! -z $GITLAB_TOKEN ]; then + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG} + docker manifest push ${GLCR_REPO}:${BRANCH_TAG} + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-debug + docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-debug + docker manifest push ${GLCR_REPO}:${UNIQUE_TAG}-haswell + docker manifest push ${GLCR_REPO}:${BRANCH_TAG}-haswell + fi + + - name: Add Image Links to Job Summary + run: | + if [ ! -z $DOCKERHUB_TOKEN ]; then + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${DOCKER_HUB_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY + fi + if [ $GHCR_ENABLED = "true" ]; then + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GHCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY + fi + if [ ! -z $GITLAB_TOKEN ]; then + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-debug\`" >> $GITHUB_STEP_SUMMARY + echo "- \`docker pull ${GLCR_REPO}:${UNIQUE_TAG}-haswell\`" >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5043f23b..4d30d252 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -100,11 +100,32 @@ jobs: exit 1 fi + #- uses: nixbuild/nix-quick-install-action@master + - uses: cachix/install-nix-action@v31.1.0 + + - name: Nom + if: ${{ env.ACT }} + run: | + echo "aaaa" + nix profile install nixpkgs#nix-output-monitor + sudo cp $(which nom) /usr/bin/nom-nix + file $(which nom) + lsblk + + #- uses: https://github.com/cachix/install-nix-action@v27 + # if: ${{ ! env.ACT }} + - name: Prepare build environment run: | + if ! type nom &> /dev/null; then + sudo cp $(which nix) /usr/bin/nom-or-nix + fi echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow - nix develop .#all-features --command true + set -x + file $(which nom-nix) + nom develop .#all-features --command true - name: Cache CI dependencies run: | @@ -196,6 +217,9 @@ jobs: with: persist-credentials: false + #- uses: nixbuild/nix-quick-install-action@master + - uses: cachix/install-nix-action@v31.1.0 + - name: Setup SSH web publish env: web_upload_ssh_private_key: ${{ secrets.WEB_UPLOAD_SSH_PRIVATE_KEY }} @@ -227,6 +251,7 @@ jobs: - name: Prepare build environment run: | echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + nix profile install --impure --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow nix develop .#all-features --command true --impure From 5fef27759c0af4cf4432537e20d55d672b975ead Mon Sep 17 00:00:00 2001 From: morguldir Date: Wed, 16 Apr 2025 01:23:55 +0200 Subject: [PATCH 3/9] ci: use improved not-nexy runners --- .forgejo/workflows/ci.yml | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 3978a894..f96ee57e 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -49,9 +49,11 @@ permissions: {} jobs: tests: name: Test - runs-on: self-hosted - volumes: - - /nix/store:/nix/store + runs-on: [ubuntu-latest, dind, nix] + container: + volumes: + - /nix/store:/nix/store + - /nix/var/nix/db:/nix/var/nix/db steps: - name: Setup SSH web publish env: @@ -102,12 +104,10 @@ jobs: exit 1 fi - #- uses: nixbuild/nix-quick-install-action@master - uses: https://github.com/cachix/install-nix-action@v31.1.0 - if: ${{ env.ACT }} - name: Nom - if: ${{ env.ACT }} + if: ${{ env.ACT_EXEC }} run: | echo "aaaa" nix profile install nixpkgs#nix-output-monitor @@ -115,20 +115,23 @@ jobs: file $(which nom) lsblk - #- uses: https://github.com/cachix/install-nix-action@v27 - # if: ${{ ! env.ACT }} - - name: Prepare build environment run: | + du -shc /nix/* + #du -shc /root/.profile/* if ! type nom &> /dev/null; then sudo cp $(which nix) /usr/bin/nom-or-nix fi echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow - set -x - file $(which nom-nix) - nom develop .#all-features --command true + + du -shc /nix/* + if type nom &> /dev/null; then + nom develop .#all-features --command true + else + nix develop .#all-features --command true + fi - name: Cache CI dependencies run: | @@ -208,9 +211,11 @@ jobs: build: name: Build - runs-on: self-hosted - volumes: - - /nix/store:/nix/store + runs-on: [not-nexy, nix, dind] + container: + volumes: + - /nix/store:/nix/store + - /nix/var/nix/db:/nix/var/nix/db strategy: matrix: include: From a67e21081987f0a11b5c577355003d70dc1d25ca Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 19 Apr 2025 03:58:53 +0200 Subject: [PATCH 4/9] switch image and use host network for complement --- .forgejo/workflows/ci.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index f96ee57e..66842259 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -49,11 +49,13 @@ permissions: {} jobs: tests: name: Test - runs-on: [ubuntu-latest, dind, nix] + runs-on: [tom, dind, nix] container: + network: host + image: catthehacker/ubuntu:act-latest volumes: - - /nix/store:/nix/store - - /nix/var/nix/db:/nix/var/nix/db + - /nix/store:/nix/store + - /nix/var/nix/db:/nix/var/nix/db steps: - name: Setup SSH web publish env: @@ -117,9 +119,10 @@ jobs: - name: Prepare build environment run: | + df -h /nix/var/nix/db du -shc /nix/* #du -shc /root/.profile/* - if ! type nom &> /dev/null; then + if ! command -v nom &> /dev/null; then sudo cp $(which nix) /usr/bin/nom-or-nix fi echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" @@ -127,7 +130,7 @@ jobs: direnv allow du -shc /nix/* - if type nom &> /dev/null; then + if command -v nom &> /dev/null; then nom develop .#all-features --command true else nix develop .#all-features --command true @@ -211,8 +214,9 @@ jobs: build: name: Build - runs-on: [not-nexy, nix, dind] + runs-on: [tom, nix, dind] container: + image: catthehacker/ubuntu:act-latest volumes: - /nix/store:/nix/store - /nix/var/nix/db:/nix/var/nix/db From 0c0c1229fe1c9a83d4dd5116732400a35a61d07b Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 19 Apr 2025 05:16:42 +0200 Subject: [PATCH 5/9] use js image and switch to command -v --- .forgejo/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 66842259..c529c1c9 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: runs-on: [tom, dind, nix] container: network: host - image: catthehacker/ubuntu:act-latest + image: catthehacker/ubuntu:js-latest volumes: - /nix/store:/nix/store - /nix/var/nix/db:/nix/var/nix/db @@ -216,7 +216,7 @@ jobs: name: Build runs-on: [tom, nix, dind] container: - image: catthehacker/ubuntu:act-latest + image: catthehacker/ubuntu:js-latest volumes: - /nix/store:/nix/store - /nix/var/nix/db:/nix/var/nix/db From 4121482770a764941ea9408d62b2d241671fc4cd Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 19 Apr 2025 06:00:29 +0200 Subject: [PATCH 6/9] use bash when setting up build environment --- .forgejo/workflows/ci.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index c529c1c9..03f67cfd 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -112,25 +112,25 @@ jobs: if: ${{ env.ACT_EXEC }} run: | echo "aaaa" - nix profile install nixpkgs#nix-output-monitor + nix profile install nixpkgs#nix-output-monitor -v sudo cp $(which nom) /usr/bin/nom-nix file $(which nom) lsblk - name: Prepare build environment + shell: bash run: | + echo $SHELL df -h /nix/var/nix/db - du -shc /nix/* - #du -shc /root/.profile/* - if ! command -v nom &> /dev/null; then + du -had 1 /nix/ + if ! type nom &> /dev/null; then sudo cp $(which nix) /usr/bin/nom-or-nix fi echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - nix profile install --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + nix profile install -v --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv direnv allow - du -shc /nix/* - if command -v nom &> /dev/null; then + if type nom &> /dev/null; then nom develop .#all-features --command true else nix develop .#all-features --command true @@ -154,6 +154,7 @@ jobs: cache-targets: "true" - name: Run CI tests + shell: bash env: CARGO_PROFILE: "test" run: | @@ -280,6 +281,7 @@ jobs: cache-targets: "true" - name: Build static ${{ matrix.target }}-all-features + shell: bash run: | if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] then @@ -329,6 +331,7 @@ jobs: fi - name: Build static debug ${{ matrix.target }}-all-features + shell: bash run: | if [[ ${{ matrix.target }} == "x86_64-linux-musl" ]] then From 23cc6e7f125fce84ea784a79654bda5bee28df2e Mon Sep 17 00:00:00 2001 From: morguldir Date: Sat, 19 Apr 2025 06:18:19 +0200 Subject: [PATCH 7/9] use forgejo/upload-artifact --- .forgejo/workflows/ci.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index 03f67cfd..d5dd9cca 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -52,7 +52,7 @@ jobs: runs-on: [tom, dind, nix] container: network: host - image: catthehacker/ubuntu:js-latest + image: catthehacker/ubuntu:act-latest volumes: - /nix/store:/nix/store - /nix/var/nix/db:/nix/var/nix/db @@ -127,7 +127,7 @@ jobs: sudo cp $(which nix) /usr/bin/nom-or-nix fi echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - nix profile install -v --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv + nix profile install -v --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv nixpkgs#nodejs direnv allow if type nom &> /dev/null; then @@ -161,6 +161,7 @@ jobs: direnv exec . engage > >(tee -a test_output.log) - name: Run Complement tests + shell: bash env: CARGO_PROFILE: "test" run: | @@ -374,7 +375,7 @@ jobs: dpkg-deb --info ${{ matrix.target }}-debug.deb - name: Upload static-x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub - uses: actions/upload-artifact@v4 + uses: forgejo/upload-artifact@v4 if: ${{ matrix.target == 'x86_64-linux-musl' }} with: name: static-x86_64-linux-musl-x86_64-haswell-optimised @@ -382,14 +383,14 @@ jobs: if-no-files-found: error - name: Upload static-${{ matrix.target }}-all-features to GitHub - uses: actions/upload-artifact@v4 + uses: forgejo/upload-artifact@v4 with: name: static-${{ matrix.target }} path: static-${{ matrix.target }} if-no-files-found: error - name: Upload static deb ${{ matrix.target }}-all-features to GitHub - uses: actions/upload-artifact@v4 + uses: forgejo/upload-artifact@v4 with: name: deb-${{ matrix.target }} path: ${{ matrix.target }}.deb @@ -425,14 +426,14 @@ jobs: fi - name: Upload static-${{ matrix.target }}-debug-all-features to GitHub - uses: actions/upload-artifact@v4 + uses: forgejo/upload-artifact@v4 with: name: static-${{ matrix.target }}-debug path: static-${{ matrix.target }}-debug if-no-files-found: error - name: Upload static deb ${{ matrix.target }}-debug-all-features to GitHub - uses: actions/upload-artifact@v4 + uses: forgejo/upload-artifact@v4 with: name: deb-${{ matrix.target }}-debug path: ${{ matrix.target }}-debug.deb @@ -472,14 +473,14 @@ jobs: - name: Upload OCI image x86_64-linux-musl-all-features-x86_64-haswell-optimised to GitHub if: ${{ matrix.target == 'x86_64-linux-musl' }} - uses: actions/upload-artifact@v4 + uses: forgejo/upload-artifact@v4 with: name: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised path: oci-image-x86_64-linux-musl-all-features-x86_64-haswell-optimised.tar.gz if-no-files-found: error compression-level: 0 - name: Upload OCI image ${{ matrix.target }}-all-features to GitHub - uses: actions/upload-artifact@v4 + uses: forgejo/upload-artifact@v4 with: name: oci-image-${{ matrix.target }} path: oci-image-${{ matrix.target }}.tar.gz @@ -487,7 +488,7 @@ jobs: compression-level: 0 - name: Upload OCI image ${{ matrix.target }}-debug-all-features to GitHub - uses: actions/upload-artifact@v4 + uses: forgejo/upload-artifact@v4 with: name: oci-image-${{ matrix.target }}-debug path: oci-image-${{ matrix.target }}-debug.tar.gz From 304f44a11b99a68162a3d61b7b4db912315c10fb Mon Sep 17 00:00:00 2001 From: morguldir Date: Mon, 21 Apr 2025 05:47:13 +0200 Subject: [PATCH 8/9] run complement against docker:dind --- .forgejo/workflows/ci.yml | 54 ++++++++++++++++++++++++++++++++++----- flake.lock | 20 +++++++-------- flake.nix | 2 +- 3 files changed, 59 insertions(+), 17 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index d5dd9cca..adb8354e 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -43,15 +43,47 @@ env: WEB_UPLOAD_SSH_USERNAME: ${{ secrets.WEB_UPLOAD_SSH_USERNAME }} GH_REF_NAME: ${{ github.ref_name }} WEBSERVER_DIR_NAME: ${{ (github.head_ref != '' && format('merge-{0}-{1}', github.event.number, github.event.pull_request.user.login)) || github.ref_name }}-${{ github.sha }} + DOCKER_HOST: "tcp://docker:2375" + DOCKER_TLS_CERTDIR: "" permissions: {} jobs: + dind-test: + name: dind-${{ matrix.runner }} + strategy: + matrix: + runner: + - jade + - tom + - ubuntu-22.04 + runs-on: ${{ matrix.runner }} + services: + dind: + env: + DOCKER_HOST: unix:///var/run/dind.socket + DOCKER_TLS_CERTDIR: "" + image: docker:dind + options: >- + --tty + --restart always + steps: + - name: curl + run: |- + sleep 30 + curl dind:2375 || true tests: name: Test - runs-on: [tom, dind, nix] + runs-on: [tom, nix] + services: + dind: + image: docker:dind + options: >- + --privileged + env: + DOCKER_TLS_CERTDIR: "" + DOCKER_HOST: unix:///var/run/dind.socket container: - network: host image: catthehacker/ubuntu:act-latest volumes: - /nix/store:/nix/store @@ -111,8 +143,8 @@ jobs: - name: Nom if: ${{ env.ACT_EXEC }} run: | - echo "aaaa" - nix profile install nixpkgs#nix-output-monitor -v + echo "Installing nix output monitor" + nix profile install nixpkgs#nix-output-monitor -Lv sudo cp $(which nom) /usr/bin/nom-nix file $(which nom) lsblk @@ -127,7 +159,8 @@ jobs: sudo cp $(which nix) /usr/bin/nom-or-nix fi echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" - nix profile install -v --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv nixpkgs#nodejs + nix profile install -v --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv nixpkgs#nodejs nixpkgs#iproute2 nixpkgs#nettools nixpkgs#bind + ip a direnv allow if type nom &> /dev/null; then @@ -160,14 +193,23 @@ jobs: run: | direnv exec . engage > >(tee -a test_output.log) + - name: Build complement image + run: | + bin/nix-build-and-cache just .#complement + - name: Run Complement tests shell: bash env: + DOCKER_HOST: tcp://dind:2375 CARGO_PROFILE: "test" run: | + addr=$(ip -brief address show eth0 | awk '{print $3}' | awk -F/ '{print $1}') + export COMPLEMENT_HS_PORT_BINDING_IP=$(dig +short dind) + export COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT=dind # the nix devshell sets $COMPLEMENT_SRC, so "/dev/null" is no-op direnv exec . bin/complement "/dev/null" complement_test_logs.jsonl complement_test_results.jsonl > >(tee -a test_output.log) cp -v -f result complement_oci_image.tar.gz + netstat -tnlp - name: Upload Complement OCI image uses: actions/upload-artifact@v4 @@ -216,7 +258,7 @@ jobs: build: name: Build - runs-on: [tom, nix, dind] + runs-on: [tom, nix] container: image: catthehacker/ubuntu:js-latest volumes: diff --git a/flake.lock b/flake.lock index 1f87b9b6..fc37d100 100644 --- a/flake.lock +++ b/flake.lock @@ -80,18 +80,18 @@ "complement": { "flake": false, "locked": { - "lastModified": 1741891349, - "narHash": "sha256-YvrzOWcX7DH1drp5SGa+E/fc7wN3hqFtPbqPjZpOu1Q=", - "owner": "girlbossceo", - "repo": "complement", - "rev": "e587b3df569cba411aeac7c20b6366d03c143745", - "type": "github" + "lastModified": 1745202855, + "narHash": "sha256-Jr4625Gp5SzL1teCAEkIBiwIhWBZ3UlPXa//I+6Ncyk=", + "ref": "morguldir/hs-ip", + "rev": "d44afcb142a0d1e3d877e3f2e38115910f7f062c", + "revCount": 860, + "type": "git", + "url": "https://forgejo.ellis.link/continuwuation/complement" }, "original": { - "owner": "girlbossceo", - "ref": "main", - "repo": "complement", - "type": "github" + "ref": "morguldir/hs-ip", + "type": "git", + "url": "https://forgejo.ellis.link/continuwuation/complement" } }, "crane": { diff --git a/flake.nix b/flake.nix index 49e860ed..8fbb851b 100644 --- a/flake.nix +++ b/flake.nix @@ -2,7 +2,7 @@ inputs = { attic.url = "github:zhaofengli/attic?ref=main"; cachix.url = "github:cachix/cachix?ref=master"; - complement = { url = "github:girlbossceo/complement?ref=main"; flake = false; }; + complement = { url = "git+https://forgejo.ellis.link/continuwuation/complement?ref=morguldir/hs-ip"; flake = false; }; crane = { url = "github:ipetkov/crane?ref=master"; }; fenix = { url = "github:nix-community/fenix?ref=main"; inputs.nixpkgs.follows = "nixpkgs"; }; flake-compat = { url = "github:edolstra/flake-compat?ref=master"; flake = false; }; From 34c025097c540fdf910127aa176d8e0875eb756b Mon Sep 17 00:00:00 2001 From: morguldir Date: Sun, 27 Apr 2025 23:00:46 +0200 Subject: [PATCH 9/9] ci: upload complement image to run complement in a separate step --- .forgejo/workflows/ci.yml | 85 ++++++++++++++++++++++++++++++--------- 1 file changed, 65 insertions(+), 20 deletions(-) diff --git a/.forgejo/workflows/ci.yml b/.forgejo/workflows/ci.yml index adb8354e..a236cc0f 100644 --- a/.forgejo/workflows/ci.yml +++ b/.forgejo/workflows/ci.yml @@ -75,16 +75,8 @@ jobs: tests: name: Test runs-on: [tom, nix] - services: - dind: - image: docker:dind - options: >- - --privileged - env: - DOCKER_TLS_CERTDIR: "" - DOCKER_HOST: unix:///var/run/dind.socket container: - image: catthehacker/ubuntu:act-latest + image: catthehacker/ubuntu:rust-latest volumes: - /nix/store:/nix/store - /nix/var/nix/db:/nix/var/nix/db @@ -196,6 +188,68 @@ jobs: - name: Build complement image run: | bin/nix-build-and-cache just .#complement + cp -v -f result complement_oci_image.tar.gz + + + - name: Upload Complement OCI image + uses: https://data.forgejo.org/forgejo/upload-artifact@v4 + env: + with: + name: complement_oci_image.tar.gz + path: complement_oci_image.tar.gz + if-no-files-found: error + compression-level: 0 + + + complement: + name: complement + runs-on: [dind, nix] + needs: tests + services: + dind: + image: docker:dind + options: >- + --privileged + env: + DOCKER_TLS_CERTDIR: "" + DOCKER_HOST: unix:///var/run/dind.socket + container: + image: catthehacker/ubuntu:act-latest + volumes: + - /nix/store:/nix/store + - /nix/var/nix/db:/nix/var/nix/db + steps: + - name: Sync repository + uses: actions/checkout@v4 + with: + persist-credentials: false + + - uses: https://github.com/cachix/install-nix-action@v31.1.0 + + - name: Prepare build environment + shell: bash + run: | + echo $SHELL + df -h /nix/var/nix/db + du -had 1 /nix/ + if ! type nom &> /dev/null; then + sudo cp $(which nix) /usr/bin/nom-or-nix + fi + echo 'source $HOME/.nix-profile/share/nix-direnv/direnvrc' > "$HOME/.direnvrc" + nix profile install -v --inputs-from . nixpkgs#direnv nixpkgs#nix-direnv nixpkgs#nodejs nixpkgs#iproute2 nixpkgs#nettools nixpkgs#bind + ip a + direnv allow + + if type nom &> /dev/null; then + nom develop .#all-features --command true + else + nix develop .#all-features --command true + fi + + - name: Download artifacts + uses: https://data.forgejo.org/forgejo/download-artifact@v4 + with: + pattern: "complement_oci_image.tar.gz" - name: Run Complement tests shell: bash @@ -208,26 +262,17 @@ jobs: export COMPLEMENT_HOSTNAME_RUNNING_COMPLEMENT=dind # the nix devshell sets $COMPLEMENT_SRC, so "/dev/null" is no-op direnv exec . bin/complement "/dev/null" complement_test_logs.jsonl complement_test_results.jsonl > >(tee -a test_output.log) - cp -v -f result complement_oci_image.tar.gz netstat -tnlp - - name: Upload Complement OCI image - uses: actions/upload-artifact@v4 - with: - name: complement_oci_image.tar.gz - path: complement_oci_image.tar.gz - if-no-files-found: error - compression-level: 0 - - name: Upload Complement logs - uses: actions/upload-artifact@v4 + uses: https://data.forgejo.org/forgejo/upload-artifact@v4 with: name: complement_test_logs.jsonl path: complement_test_logs.jsonl if-no-files-found: error - name: Upload Complement results - uses: actions/upload-artifact@v4 + uses: https://data.forgejo.org/forgejo/upload-artifact@v4 with: name: complement_test_results.jsonl path: complement_test_results.jsonl