forgejo/ci: better nix-based runner

2023-12-05 21:46:00 +01:00 · 2023-12-05 21:46:00 +01:00 · cb73afe887
parent 394799d66f
commit cb73afe887
7 changed files with 220 additions and 51 deletions
--- a/.forgejo/workflows/build.yml
+++ b/.forgejo/workflows/build.yml
@ -8,41 +8,46 @@ on:
      - '.forgejo/workflows/build.yml'
 jobs:
  build-amd64-linux:
-    runs-on: ubuntu-latest-amd64
+    runs-on: nix-amd64
    steps:
-      - uses: actions/checkout@v3
-      - name: Install Nix
-        uses: https://github.com/DeterminateSystems/nix-installer-action@v4
-        with:
-          init: none
-          planner: linux
-          github-token: null
-      - name: Setup Attic
-        uses: https://github.com/ryanccn/attic-action@v0
-        with:
-          endpoint: ${{ secrets.ATTIC_ENDPOINT }}
-          cache: ${{ secrets.ATTIC_CACHE }}
-          token: ${{ secrets.ATTIC_TOKEN }}
-      - run: nix build '.#nixosConfigurations.tanker.pkgs.attic-server'
-      - run: nix build '.#nixosConfigurations.tanker.pkgs.attic-client'
-      - run: nix build '.#nixosConfigurations.tanker.config.services.nginx.package'
-      - run: nix build '.#nixosConfigurations.tanker.config.home-manager.users.daniel.programs.neovim.finalPackage'
+      - uses: actions/checkout@v4
+      - name: Cache Login
+        run: |
+          echo "Logging in ..."
+          attic login --set-default attic ${{ secrets.ATTIC_ENDPOINT }} ${{ secrets.ATTIC_TOKEN }}
+      - name: Build attic-server
+        run: |
+          nix build '.#nixosConfigurations.tanker.pkgs.attic-server'
+          attic push ${{ secrets.ATTIC_CACHE }} ./result
+      - name: Build attic-client
+        run: |
+          nix build '.#nixosConfigurations.tanker.pkgs.attic-client'
+          attic push ${{ secrets.ATTIC_CACHE }} ./result
+      - name: Build nginx
+        run: |
+          nix build '.#nixosConfigurations.tanker.config.services.nginx.package'
+          attic push ${{ secrets.ATTIC_CACHE }} ./result
+      - name: Build neovim
+        run: |
+          nix build '.#nixosConfigurations.tanker.config.home-manager.users.daniel.programs.neovim.finalPackage'
+          attic push ${{ secrets.ATTIC_CACHE }} ./result
  build-arm64-linux:
-    runs-on: ubuntu-latest-arm64
+    runs-on: nix-arm64
    steps:
-      - uses: actions/checkout@v3
-      - name: Install Nix
-        uses: https://github.com/DeterminateSystems/nix-installer-action@v4
-        with:
-          init: none
-          planner: linux
-          github-token: null
-      - name: Setup Attic
-        uses: https://github.com/ryanccn/attic-action@v0
-        with:
-          endpoint: ${{ secrets.ATTIC_ENDPOINT }}
-          cache: ${{ secrets.ATTIC_CACHE }}
-          token: ${{ secrets.ATTIC_TOKEN }}
-      - run: nix build '.#nixosConfigurations.argon.pkgs.attic-client'
-      - run: nix build '.#nixosConfigurations.argon.config.services.nginx.package'
-      - run: nix build '.#nixosConfigurations.argon.config.home-manager.users.daniel.programs.neovim.finalPackage'
+      - uses: actions/checkout@v4
+      - name: Cache Login
+        run: |
+          echo "Logging in ..."
+          attic login --set-default attic ${{ secrets.ATTIC_ENDPOINT }} ${{ secrets.ATTIC_TOKEN }}
+      - name: Build attic-client
+        run: |
+          nix build '.#nixosConfigurations.argon.pkgs.attic-client'
+          attic push ${{ secrets.ATTIC_CACHE }} ./result
+      - name: Build nginx
+        run: |
+          nix build '.#nixosConfigurations.argon.config.services.nginx.package'
+          attic push ${{ secrets.ATTIC_CACHE }} ./result
+      - name: Build neovim
+        run: |
+          nix build '.#nixosConfigurations.argon.config.home-manager.users.daniel.programs.neovim.finalPackage'
+          attic push ${{ secrets.ATTIC_CACHE }} ./result
--- a/agenix/hosts/argon/forgejo-actions/token.age
+++ b/agenix/hosts/argon/forgejo-actions/token.age
--- a/agenix/hosts/tanker/forgejo-actions/token.age
+++ b/agenix/hosts/tanker/forgejo-actions/token.age
@ -1,10 +1,12 @@
 age-encryption.org/v1
-> ssh-ed25519 MtGp6g jM+++wGqvWlj9wKwKdrS3d8xpwfLp1ks4GCh3sm/6zM
-QkBPJf0lBF617AFcko4KA/Aq6mi6eMWp/ye2Abf2fKk
-> ssh-ed25519 iO8/4g IGCdvdMB6PdiqxA9yamSUMCfH4Bk0JmtOuZt4WZrFGM
-cz19imzpQAkWv+iCoUzBfMRC5D0yusCMQkROrjBhoJ0
-> PT|e-grease cJG6UW4o
-4G+Rp2jt1sZbGLxuKl7DgX1wl1kaOhhEjkloCeaHg0lt6P7bmjcg++jh6hWs7MhO
-dMp8SKY9
--- VdFRpEBs74LJOqJNYwiGeb/wy/e7Wm+aFQnw3AI1pFw
-r‹åkòBúx–µvŠ™aob""éh&éÅîb{ñÌH´òã,’×q¢·Ã RSm½¤ûÃTûo	¾hB•gdµ
÷?GÒ¼n0©î
+-> ssh-ed25519 MtGp6g I3WQex/smit8a4Isd02PK5wjcjy2hUed3UJpd0y33So
+SyMzqODrDT5V8VEjp8ERWWa8f/LkJXOfB0I2v0/5xMk
+-> ssh-ed25519 iO8/4g 7HVKn8hYsADqVUoaJZQq+VsnLa3fwRsDNEuAe4HYBmM
+ZZrHbVUtM0gKgIhSVARW7VhB3VVMd3kqu4aClviJYjE
+-> XGZy\BA-grease (7F_&
+ogqkD7AyPLaMX7ZRC+MXIHSUR0pRYFGGpQzLse/J+Xfn+d8Fca+ORmdZ7hszVCYV
+2+vZ3YeW5undYzkMjJuVyZf5qS/S5Mbp
+--- rVstDRi//LFd7AO4GKNShRHipPHwPdl1B3pKqYoNZms
+Ä4AªhÜ!q¢aøtVµ!±Æòr![P+ÔDv·Ò'%F
¹Ê<C2B9>Ú$ü<>LŸPa
+cÔÀiôkZÁÄ%@`èã<C3A8>“Åè&¡
+å`[;
--- a/flake.lock
+++ b/flake.lock
@ -87,11 +87,11 @@
        ]
      },
      "locked": {
-        "lastModified": 1700927249,
-        "narHash": "sha256-iqmIWiEng890/ru7ZBf4nUezFPyRm2fjRTvuwwxqk2o=",
+        "lastModified": 1701775991,
+        "narHash": "sha256-/51DaSTzoW+wQfj5P9EnTbSxixDFjjhfnGdMKcSp+is=",
        "owner": "nix-community",
        "repo": "disko",
-        "rev": "3cb78c93e6a02f494aaf6aeb37481c27a2e2ee22",
+        "rev": "f84c3684900d11cf19f530070d32d55f0ed51374",
        "type": "github"
      },
      "original": {
--- a/system/hosts/argon.nix
+++ b/system/hosts/argon.nix
@ -22,7 +22,7 @@ in

    ../nixos/attic.nix

-    (import ../nixos/forgejo-runner.nix (args // { name = "argon"; tag = "ubuntu-latest-arm64"; }))
+    (import ../nixos/forgejo-runner.nix (args // { name = "argon"; tag = "ubuntu-latest-arm64"; nixTag = "arm64"; }))

    ../nixos/tailscale.nix

--- a/system/hosts/tanker.nix
+++ b/system/hosts/tanker.nix
@ -33,7 +33,7 @@ in
    ../nixos/fedifetcher.nix

    ../nixos/forgejo.nix
-    (import ../nixos/forgejo-runner.nix (args // { name = "tanker"; tag = "ubuntu-latest-amd64"; }))
+    (import ../nixos/forgejo-runner.nix (args // { name = "tanker"; tag = "ubuntu-latest-amd64"; nixTag = "amd64"; }))

    ../nixos/headscale.nix

--- a/system/nixos/forgejo-runner.nix
+++ b/system/nixos/forgejo-runner.nix
@ -1,14 +1,155 @@
-{ pkgs, config, name, tag, ... }:
+{ pkgs, config, name, tag, nixTag, ... }:
+
+# Based on: https://git.clan.lol/clan/clan-infra/src/branch/main/modules/web01/gitea/actions-runner.nix

 let
  forgejoUrl = "https://git.kempkens.io";
+
+  storeDeps = pkgs.runCommand "store-deps" { } ''
+    mkdir -p $out/bin
+    for dir in ${toString [ pkgs.attic-client pkgs.coreutils pkgs.findutils pkgs.gnugrep pkgs.gawk pkgs.git pkgs.nixVersions.stable pkgs.bash pkgs.jq pkgs.nodejs ]}; do
+      for bin in "$dir"/bin/*; do
+        ln -s "$bin" "$out/bin/$(basename "$bin")"
+      done
+    done
+
+    # Add SSL CA certs
+    mkdir -p $out/etc/ssl/certs
+    cp -a "${pkgs.cacert}/etc/ssl/certs/ca-bundle.crt" $out/etc/ssl/certs/ca-bundle.crt
+  '';
 in
 {
+  systemd.services = {
+    forgejo-runner-nix-image = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "podman.service" ];
+      requires = [ "podman.service" ];
+      path = [ config.virtualisation.podman.package pkgs.gnutar pkgs.shadow pkgs.getent ];
+      script = ''
+        set -eux -o pipefail
+        mkdir -p etc/nix
+
+        touch etc/passwd etc/group
+        groupid=$(cut -d: -f3 < <(getent group nix-ci-user))
+        userid=$(cut -d: -f3 < <(getent passwd nix-ci-user))
+        groupadd --prefix $(pwd) --gid "$groupid" nix-ci-user
+        emptypassword='$6$1ero.LwbisiU.h3D$GGmnmECbPotJoPQ5eoSTD6tTjKnSWZcjHoVTkxFLZP17W9hRi/XkmCiAMOfWruUwy8gMjINrBMNODc7cYEo4K.'
+        useradd --prefix $(pwd) -p "$emptypassword" -m -d /tmp -u "$userid" -g "$groupid" -G nix-ci-user nix-ci-user
+
+        cat <<NIX_CONFIG > etc/nix/nix.conf
+        accept-flake-config = true
+        experimental-features = nix-command flakes
+        NIX_CONFIG
+
+        cat <<NSSWITCH > etc/nsswitch.conf
+        passwd:    files mymachines systemd
+        group:     files mymachines systemd
+        shadow:    files
+
+        hosts:     files mymachines dns myhostname
+        networks:  files
+
+        ethers:    files
+        services:  files
+        protocols: files
+        rpc:       files
+        NSSWITCH
+
+        tar -cv . | tar -tvf -
+        tar -cv . | podman import - forgejo-runner-nix
+      '';
+      serviceConfig = {
+        RuntimeDirectory = "forgejo-runner-nix-image";
+        WorkingDirectory = "/run/forgejo-runner-nix-image";
+        Type = "oneshot";
+        RemainAfterExit = true;
+      };
+    };
+
+    forgejo-runner-nix = {
+      after = [ "forgejo-runner-nix-image.service" ];
+      requires = [ "forgejo-runner-nix-image.service" ];
+
+      serviceConfig = {
+        # Hardening (may overlap with DynamicUser=)
+        # The following options are only for optimizing output of systemd-analyze
+        AmbientCapabilities = "";
+        CapabilityBoundingSet = "";
+        # ProtectClock= adds DeviceAllow=char-rtc r
+        DeviceAllow = "";
+        NoNewPrivileges = true;
+        PrivateDevices = true;
+        PrivateMounts = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectHostname = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectSystem = "strict";
+        RemoveIPC = true;
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        UMask = "0066";
+        ProtectProc = "invisible";
+        SystemCallFilter = [
+          "~@clock"
+          "~@cpu-emulation"
+          "~@module"
+          "~@mount"
+          "~@obsolete"
+          "~@raw-io"
+          "~@reboot"
+          "~@swap"
+          # needed by go?
+          #"~@resources"
+          "~@privileged"
+          "~capset"
+          "~setdomainname"
+          "~sethostname"
+        ];
+        RestrictAddressFamilies = [ "AF_INET" "AF_INET6" "AF_UNIX" "AF_NETLINK" ];
+
+        # Needs network access
+        PrivateNetwork = false;
+        # Cannot be true due to Node
+        MemoryDenyWriteExecute = false;
+
+        # The more restrictive "pid" option makes `nix` commands in CI emit
+        # "GC Warning: Couldn't read /proc/stat"
+        # You may want to set this to "pid" if not using `nix` commands
+        ProcSubset = "all";
+        # Coverage programs for compiled code such as `cargo-tarpaulin` disable
+        # ASLR (address space layout randomization) which requires the
+        # `personality` syscall
+        # You may want to set this to `true` if not using coverage tooling on
+        # compiled code
+        LockPersonality = false;
+
+        # Note that this has some interactions with the User setting; so you may
+        # want to consult the systemd docs if using both.
+        DynamicUser = true;
+      };
+    };
+  };
+
+  users.users.nix-ci-user = {
+    group = "nix-ci-user";
+    description = "Used for running nix-based CI jobs";
+    home = "/var/empty";
+    isSystemUser = true;
+  };
+  users.groups.nix-ci-user = { };
+
  services.gitea-actions-runner = {
    package = pkgs.forgejo-actions-runner;

    instances = {
-      tanker = {
+      act = {
        enable = true;
        url = forgejoUrl;

@ -19,6 +160,27 @@ in
          "${tag}:docker://ghcr.io/catthehacker/ubuntu:act-latest"
        ];
      };
+
+      nix = {
+        enable = true;
+        url = forgejoUrl;
+
+        name = "${name}-nix";
+        tokenFile = config.age.secrets.forgejo-actions-token.path;
+
+        labels = [
+          "nix-${nixTag}:docker://forgejo-runner-nix"
+        ];
+
+        settings = {
+          container.options = "-e NIX_BUILD_SHELL=/bin/bash -e PAGER=cat -e PATH=/bin -e SSL_CERT_FILE=/etc/ssl/certs/ca-bundle.crt --device /dev/kvm -v /nix:/nix -v ${storeDeps}/bin:/bin -v ${storeDeps}/etc/ssl:/etc/ssl --user nix-ci-user --device=/dev/kvm";
+          container.valid_volumes = [
+            "/nix"
+            "${storeDeps}/bin"
+            "${storeDeps}/etc/ssl"
+          ];
+        };
+      };
    };
  };
 }