refactor(users/flokli): unify archeology and archivist

This merges the two directories together, and explains the two different
AWS profiles and what they're used for.

Change-Id: Ieaa09be5af02491056f5ad83b1d639e2de9a218b
Reviewed-on: https://cl.snix.dev/c/snix/+/30102
Autosubmit: Florian Klink <flokli@flokli.de>
Reviewed-by: Ryan Lahfa <masterancpp@gmail.com>
Tested-by: besadii

users/flokli/archeology/README.md

@@ -1,5 +0,0 @@
-# archeology
-
-This directory contains various scripts and helpers used for nix-archeology tasks.
-
-It's used from some of the archeology instances, as well as standalone.

users/flokli/archeology/default.nix

@@ -1,51 +0,0 @@
-{ depot, pkgs, ... }:
-
-let
-  clickhouseConfigAWS = builtins.toFile "clickhouse-local.xml" ''
-    <clickhouse>
-        <s3>
-          <use_environment_credentials>true</use_environment_credentials>
-        </s3>
-    </clickhouse>
-  '';
-  # clickhouse has a very odd AWS config concept.
-  # Configure it to be a bit more sane.
-  clickhouseLocalFixedAWS = pkgs.runCommand "clickhouse-local-fixed"
-    {
-      nativeBuildInputs = [ pkgs.makeWrapper ];
-    } ''
-    mkdir -p $out/bin
-    makeWrapper ${pkgs.clickhouse}/bin/clickhouse-local $out/bin/clickhouse-local \
-      --append-flags "-C ${clickhouseConfigAWS}"
-  '';
-in
-
-depot.nix.readTree.drvTargets {
-  inherit clickhouseLocalFixedAWS;
-  parse-bucket-logs = pkgs.runCommand "archeology-parse-bucket-logs"
-    {
-      nativeBuildInputs = [ pkgs.makeWrapper ];
-    } ''
-    mkdir -p $out/bin
-    makeWrapper ${(pkgs.writers.writeRust "parse-bucket-logs-unwrapped" {} ./parse_bucket_logs.rs)} $out/bin/archeology-parse-bucket-logs \
-      --prefix PATH : ${pkgs.lib.makeBinPath [ clickhouseLocalFixedAWS ]}
-  '';
-
-  shell = pkgs.mkShell {
-    name = "archeology-shell";
-    packages = with pkgs; [ awscli2 clickhouseLocalFixedAWS rust-analyzer rustc rustfmt ];
-
-    AWS_PROFILE = "sso";
-    AWS_CONFIG_FILE = pkgs.writeText "aws-config" ''
-      [sso-session nixos]
-      sso_region = eu-north-1
-      sso_start_url = https://nixos.awsapps.com/start
-      sso_registration_scopes = sso:account:access
-
-      [profile "sso"]
-      sso_session = nixos
-      sso_account_id = 080433136561
-      sso_role_name = archeologist
-    '';
-  };
-}

users/flokli/archivist/README.md

@@ -0,0 +1,23 @@
+# archivist
+
+This directory contains various scripts and helpers used for nix-archivist tasks.
+
+It's used from some of the archivist EC2 instance, as well as standalone.
+
+## AWS Profile setup
+There's 2 AWS Accounts, reachable via the nixos.awsapps.com SSO portal.
+
+### archeologist
+This is assuming the `archeologist` AWS role in the main NixOS account.
+
+### archivist
+This is a separate AWS Account, only for the archivist project. We can assume
+`AWSAdministratorAccess` in there.
+
+## Machine
+The `archivist-ec2` machine currently is deployed in the main NixOS account.
+
+It regularly processes S3 bucket logs and dumps them in parquet format into
+another bucket.
+In the future, we want to move this machine to the dedicated `archivist` AWS
+account.

users/flokli/archivist/default.nix

@@ -2,7 +2,41 @@
 , pkgs
 , ...
 }:
+
+let
+  clickhouseConfigAWS = builtins.toFile "clickhouse-local.xml" ''
+    <clickhouse>
+        <s3>
+          <use_environment_credentials>true</use_environment_credentials>
+        </s3>
+    </clickhouse>
+  '';
+  # clickhouse has a very odd AWS config concept.
+  # Configure it to be a bit more sane.
+  clickhouseLocalFixedAWS = pkgs.runCommand "clickhouse-local-fixed"
+    {
+      nativeBuildInputs = [ pkgs.makeWrapper ];
+    } ''
+    mkdir -p $out/bin
+    makeWrapper ${pkgs.clickhouse}/bin/clickhouse-local $out/bin/clickhouse-local \
+      --append-flags "-C ${clickhouseConfigAWS}"
+  '';
+
+in
 depot.nix.readTree.drvTargets {
+  inherit clickhouseLocalFixedAWS;
+
+  parse-bucket-logs = pkgs.runCommand "archivist-parse-bucket-logs"
+    {
+      nativeBuildInputs = [ pkgs.makeWrapper ];
+    } ''
+    mkdir -p $out/bin
+    makeWrapper ${(pkgs.writers.writeRust "parse-bucket-logs-unwrapped" {} ./parse_bucket_logs.rs)} $out/bin/archivist-parse-bucket-logs \
+      --prefix PATH : ${pkgs.lib.makeBinPath [ clickhouseLocalFixedAWS ]}
+  '';
+
+
+  # A shell, by default pointing us to the archivist SSO profile / account by default.
   shell = pkgs.mkShell {
     name = "archivist-shell";
     packages = with pkgs; [ awscli2 ];
@@ -14,15 +48,15 @@ depot.nix.readTree.drvTargets {
       sso_start_url = https://nixos.awsapps.com/start
       sso_registration_scopes = sso:account:access
 
-      [profile "archivist"]
-      sso_session = nixos
-      sso_account_id = 286553126452
-      sso_role_name = AWSAdministratorAccess
-
       [profile "archeologist"]
       sso_session = nixos
-      sso_account_id = 080433136561
+      sso_account_id = 080433136561 # nixos root
       sso_role_name = archeologist
+
+      [profile "archivist"]
+      sso_session = nixos
+      sso_account_id = 286553126452 # archivist
+      sso_role_name = AWSAdministratorAccess
     '';
   };
 }

users/flokli/nixos/archeology-ec2/OWNERS

@@ -1 +0,0 @@
-edef

users/flokli/nixos/archivist-ec2/configuration.nix

@@ -3,7 +3,7 @@
 {
   imports = [
     "${modulesPath}/virtualisation/amazon-image.nix"
-    ../profiles/archeology.nix
+    ../profiles/archivist.nix
   ];
 
   systemd.timers.parse-bucket-logs = {
@@ -12,7 +12,7 @@
   };
 
   systemd.services.parse-bucket-logs = {
-    path = [ depot.users.flokli.archeology.parse-bucket-logs ];
+    path = [ depot.users.flokli.archivist.parse-bucket-logs ];
     serviceConfig = {
       Type = "oneshot";
       ExecStart = (pkgs.writers.writePython3 "parse-bucket-logs-continuously"
@@ -25,10 +25,10 @@
   };
 
   environment.systemPackages = [
-    depot.users.flokli.archeology.parse-bucket-logs
+    depot.users.flokli.archivist.parse-bucket-logs
   ];
 
-  networking.hostName = "archeology-ec2";
+  networking.hostName = "archivist-ec2";
 
   system.stateVersion = "23.05"; # Did you read the comment?
 }

users/flokli/nixos/archivist-ec2/parse-bucket-logs-continuously.py

@@ -47,7 +47,7 @@ while True:
     # Invoke parse-bucket-logs script inside a tempdir and upload on success.
     with tempfile.TemporaryDirectory() as td:
         work_file_name = os.path.join(td, "output.parquet")
-        args = ["archeology-parse-bucket-logs", src, work_file_name]
+        args = ["archivist-parse-bucket-logs", src, work_file_name]
         subprocess.run(
             args,
             check=True  # throw exception if nonzero exit code

users/flokli/nixos/default.nix

@@ -1,4 +1,4 @@
-{ depot, pkgs, lib, ... }:
+{ depot, pkgs, ... }:
 
 let
   # assumes `name` is configured appropriately in your .ssh/config
@@ -11,13 +11,13 @@ let
 
 in
 depot.nix.readTree.drvTargets rec {
-  archeologyEc2System = (depot.ops.nixos.nixosFor ({ ... }: {
+  archivistEc2System = (depot.ops.nixos.nixosFor ({ ... }: {
     imports = [
-      ./archeology-ec2/configuration.nix
+      ./archivist-ec2/configuration.nix
     ];
   })).config.system.build.toplevel;
 
-  deploy-archeology-ec2 = (deployScript "archeology-ec2" archeologyEc2System);
+  deploy-archivist-ec2 = (deployScript "archivist-ec2" archivistEc2System);
 
   nixosTvixCacheSystem = (depot.ops.nixos.nixosFor ({ ... }: {
     imports = [
@@ -28,7 +28,7 @@ depot.nix.readTree.drvTargets rec {
   deploy-nixos-tvix-cache = (deployScript "root@nixos.tvix.store" nixosTvixCacheSystem);
 
   deps = (depot.nix.lazy-deps {
-    deploy-archeology-ec2.attr = "users.flokli.nixos.deploy-archeology-ec2";
+    deploy-archivist-ec2.attr = "users.flokli.nixos.deploy-archivist-ec2";
     aws.attr = "third_party.nixpkgs.awscli";
   });
 

users/flokli/nixos/profiles/archivist.nix

@@ -1,4 +1,4 @@
-# Set of unconditional config options applicable to all archeology machines.
+# Set of unconditional config options applicable to all archivist machines.
 
 { pkgs, ... }: