diff --git a/users/flokli/archeology/README.md b/users/flokli/archeology/README.md deleted file mode 100644 index e4cd9b84b..000000000 --- a/users/flokli/archeology/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# archeology - -This directory contains various scripts and helpers used for nix-archeology tasks. - -It's used from some of the archeology instances, as well as standalone. diff --git a/users/flokli/archeology/default.nix b/users/flokli/archeology/default.nix deleted file mode 100644 index 690944403..000000000 --- a/users/flokli/archeology/default.nix +++ /dev/null @@ -1,51 +0,0 @@ -{ depot, pkgs, ... }: - -let - clickhouseConfigAWS = builtins.toFile "clickhouse-local.xml" '' - - - true - - - ''; - # clickhouse has a very odd AWS config concept. - # Configure it to be a bit more sane. - clickhouseLocalFixedAWS = pkgs.runCommand "clickhouse-local-fixed" - { - nativeBuildInputs = [ pkgs.makeWrapper ]; - } '' - mkdir -p $out/bin - makeWrapper ${pkgs.clickhouse}/bin/clickhouse-local $out/bin/clickhouse-local \ - --append-flags "-C ${clickhouseConfigAWS}" - ''; -in - -depot.nix.readTree.drvTargets { - inherit clickhouseLocalFixedAWS; - parse-bucket-logs = pkgs.runCommand "archeology-parse-bucket-logs" - { - nativeBuildInputs = [ pkgs.makeWrapper ]; - } '' - mkdir -p $out/bin - makeWrapper ${(pkgs.writers.writeRust "parse-bucket-logs-unwrapped" {} ./parse_bucket_logs.rs)} $out/bin/archeology-parse-bucket-logs \ - --prefix PATH : ${pkgs.lib.makeBinPath [ clickhouseLocalFixedAWS ]} - ''; - - shell = pkgs.mkShell { - name = "archeology-shell"; - packages = with pkgs; [ awscli2 clickhouseLocalFixedAWS rust-analyzer rustc rustfmt ]; - - AWS_PROFILE = "sso"; - AWS_CONFIG_FILE = pkgs.writeText "aws-config" '' - [sso-session nixos] - sso_region = eu-north-1 - sso_start_url = https://nixos.awsapps.com/start - sso_registration_scopes = sso:account:access - - [profile "sso"] - sso_session = nixos - sso_account_id = 080433136561 - sso_role_name = archeologist - ''; - }; -} diff --git a/users/flokli/archivist/README.md b/users/flokli/archivist/README.md new file mode 100644 index 000000000..904c87a4e --- /dev/null +++ b/users/flokli/archivist/README.md @@ -0,0 +1,23 @@ +# archivist + +This directory contains various scripts and helpers used for nix-archivist tasks. + +It's used from some of the archivist EC2 instance, as well as standalone. + +## AWS Profile setup +There's 2 AWS Accounts, reachable via the nixos.awsapps.com SSO portal. + +### archeologist +This is assuming the `archeologist` AWS role in the main NixOS account. + +### archivist +This is a separate AWS Account, only for the archivist project. We can assume +`AWSAdministratorAccess` in there. + +## Machine +The `archivist-ec2` machine currently is deployed in the main NixOS account. + +It regularly processes S3 bucket logs and dumps them in parquet format into +another bucket. +In the future, we want to move this machine to the dedicated `archivist` AWS +account. diff --git a/users/flokli/archivist/default.nix b/users/flokli/archivist/default.nix index ef49c46db..43fc3ec1f 100644 --- a/users/flokli/archivist/default.nix +++ b/users/flokli/archivist/default.nix @@ -2,7 +2,41 @@ , pkgs , ... }: + +let + clickhouseConfigAWS = builtins.toFile "clickhouse-local.xml" '' + + + true + + + ''; + # clickhouse has a very odd AWS config concept. + # Configure it to be a bit more sane. + clickhouseLocalFixedAWS = pkgs.runCommand "clickhouse-local-fixed" + { + nativeBuildInputs = [ pkgs.makeWrapper ]; + } '' + mkdir -p $out/bin + makeWrapper ${pkgs.clickhouse}/bin/clickhouse-local $out/bin/clickhouse-local \ + --append-flags "-C ${clickhouseConfigAWS}" + ''; + +in depot.nix.readTree.drvTargets { + inherit clickhouseLocalFixedAWS; + + parse-bucket-logs = pkgs.runCommand "archivist-parse-bucket-logs" + { + nativeBuildInputs = [ pkgs.makeWrapper ]; + } '' + mkdir -p $out/bin + makeWrapper ${(pkgs.writers.writeRust "parse-bucket-logs-unwrapped" {} ./parse_bucket_logs.rs)} $out/bin/archivist-parse-bucket-logs \ + --prefix PATH : ${pkgs.lib.makeBinPath [ clickhouseLocalFixedAWS ]} + ''; + + + # A shell, by default pointing us to the archivist SSO profile / account by default. shell = pkgs.mkShell { name = "archivist-shell"; packages = with pkgs; [ awscli2 ]; @@ -14,15 +48,15 @@ depot.nix.readTree.drvTargets { sso_start_url = https://nixos.awsapps.com/start sso_registration_scopes = sso:account:access - [profile "archivist"] - sso_session = nixos - sso_account_id = 286553126452 - sso_role_name = AWSAdministratorAccess - [profile "archeologist"] sso_session = nixos - sso_account_id = 080433136561 + sso_account_id = 080433136561 # nixos root sso_role_name = archeologist + + [profile "archivist"] + sso_session = nixos + sso_account_id = 286553126452 # archivist + sso_role_name = AWSAdministratorAccess ''; }; } diff --git a/users/flokli/archeology/parse_bucket_logs.rs b/users/flokli/archivist/parse_bucket_logs.rs similarity index 100% rename from users/flokli/archeology/parse_bucket_logs.rs rename to users/flokli/archivist/parse_bucket_logs.rs diff --git a/users/flokli/nixos/archeology-ec2/OWNERS b/users/flokli/nixos/archeology-ec2/OWNERS deleted file mode 100644 index b9bc074a8..000000000 --- a/users/flokli/nixos/archeology-ec2/OWNERS +++ /dev/null @@ -1 +0,0 @@ -edef diff --git a/users/flokli/archeology/OWNERS b/users/flokli/nixos/archivist-ec2/OWNERS similarity index 100% rename from users/flokli/archeology/OWNERS rename to users/flokli/nixos/archivist-ec2/OWNERS diff --git a/users/flokli/nixos/archeology-ec2/configuration.nix b/users/flokli/nixos/archivist-ec2/configuration.nix similarity index 79% rename from users/flokli/nixos/archeology-ec2/configuration.nix rename to users/flokli/nixos/archivist-ec2/configuration.nix index f0fc0c5d0..b43cff837 100644 --- a/users/flokli/nixos/archeology-ec2/configuration.nix +++ b/users/flokli/nixos/archivist-ec2/configuration.nix @@ -3,7 +3,7 @@ { imports = [ "${modulesPath}/virtualisation/amazon-image.nix" - ../profiles/archeology.nix + ../profiles/archivist.nix ]; systemd.timers.parse-bucket-logs = { @@ -12,7 +12,7 @@ }; systemd.services.parse-bucket-logs = { - path = [ depot.users.flokli.archeology.parse-bucket-logs ]; + path = [ depot.users.flokli.archivist.parse-bucket-logs ]; serviceConfig = { Type = "oneshot"; ExecStart = (pkgs.writers.writePython3 "parse-bucket-logs-continuously" @@ -25,10 +25,10 @@ }; environment.systemPackages = [ - depot.users.flokli.archeology.parse-bucket-logs + depot.users.flokli.archivist.parse-bucket-logs ]; - networking.hostName = "archeology-ec2"; + networking.hostName = "archivist-ec2"; system.stateVersion = "23.05"; # Did you read the comment? } diff --git a/users/flokli/nixos/archeology-ec2/hardware-configuration.nix b/users/flokli/nixos/archivist-ec2/hardware-configuration.nix similarity index 100% rename from users/flokli/nixos/archeology-ec2/hardware-configuration.nix rename to users/flokli/nixos/archivist-ec2/hardware-configuration.nix diff --git a/users/flokli/nixos/archeology-ec2/parse-bucket-logs-continuously.py b/users/flokli/nixos/archivist-ec2/parse-bucket-logs-continuously.py similarity index 96% rename from users/flokli/nixos/archeology-ec2/parse-bucket-logs-continuously.py rename to users/flokli/nixos/archivist-ec2/parse-bucket-logs-continuously.py index f6ec8fb77..fb477a689 100644 --- a/users/flokli/nixos/archeology-ec2/parse-bucket-logs-continuously.py +++ b/users/flokli/nixos/archivist-ec2/parse-bucket-logs-continuously.py @@ -47,7 +47,7 @@ while True: # Invoke parse-bucket-logs script inside a tempdir and upload on success. with tempfile.TemporaryDirectory() as td: work_file_name = os.path.join(td, "output.parquet") - args = ["archeology-parse-bucket-logs", src, work_file_name] + args = ["archivist-parse-bucket-logs", src, work_file_name] subprocess.run( args, check=True # throw exception if nonzero exit code diff --git a/users/flokli/nixos/default.nix b/users/flokli/nixos/default.nix index 5ca6bbce1..92dffa0ec 100644 --- a/users/flokli/nixos/default.nix +++ b/users/flokli/nixos/default.nix @@ -1,4 +1,4 @@ -{ depot, pkgs, lib, ... }: +{ depot, pkgs, ... }: let # assumes `name` is configured appropriately in your .ssh/config @@ -11,13 +11,13 @@ let in depot.nix.readTree.drvTargets rec { - archeologyEc2System = (depot.ops.nixos.nixosFor ({ ... }: { + archivistEc2System = (depot.ops.nixos.nixosFor ({ ... }: { imports = [ - ./archeology-ec2/configuration.nix + ./archivist-ec2/configuration.nix ]; })).config.system.build.toplevel; - deploy-archeology-ec2 = (deployScript "archeology-ec2" archeologyEc2System); + deploy-archivist-ec2 = (deployScript "archivist-ec2" archivistEc2System); nixosTvixCacheSystem = (depot.ops.nixos.nixosFor ({ ... }: { imports = [ @@ -28,7 +28,7 @@ depot.nix.readTree.drvTargets rec { deploy-nixos-tvix-cache = (deployScript "root@nixos.tvix.store" nixosTvixCacheSystem); deps = (depot.nix.lazy-deps { - deploy-archeology-ec2.attr = "users.flokli.nixos.deploy-archeology-ec2"; + deploy-archivist-ec2.attr = "users.flokli.nixos.deploy-archivist-ec2"; aws.attr = "third_party.nixpkgs.awscli"; }); diff --git a/users/flokli/nixos/profiles/archeology.nix b/users/flokli/nixos/profiles/archivist.nix similarity index 95% rename from users/flokli/nixos/profiles/archeology.nix rename to users/flokli/nixos/profiles/archivist.nix index 995885646..329253cd7 100644 --- a/users/flokli/nixos/profiles/archeology.nix +++ b/users/flokli/nixos/profiles/archivist.nix @@ -1,4 +1,4 @@ -# Set of unconditional config options applicable to all archeology machines. +# Set of unconditional config options applicable to all archivist machines. { pkgs, ... }: