From a15760671d6328facbaf542019171571624374b0 Mon Sep 17 00:00:00 2001 From: Aspen Smith Date: Sat, 21 Sep 2024 13:30:44 -0400 Subject: [PATCH] feat(aspen/system): Move metrics to ogopogo, refresh Change-Id: I93ddc961b473e15febe22a16879875dbd926236a Reviewed-on: https://cl.tvl.fyi/c/depot/+/12501 Autosubmit: aspen Tested-by: BuildkiteCI Reviewed-by: aspen --- ops/machines/all-systems.nix | 1 - users/aspen/secrets/bbbg.age | Bin 598 -> 598 bytes users/aspen/secrets/buildkite-ssh-key.age | Bin 3833 -> 3833 bytes users/aspen/secrets/buildkite-token.age | Bin 483 -> 483 bytes users/aspen/secrets/cloudflare.age | Bin 409 -> 519 bytes users/aspen/secrets/ddclient-password.age | Bin 360 -> 360 bytes users/aspen/secrets/secrets.nix | 2 +- .../secrets/windtunnel-bot-github-token.age | 16 +- users/aspen/system/system/machines/lusca.nix | 1 + .../aspen/system/system/machines/mugwump.nix | 140 ------------- .../aspen/system/system/machines/ogopogo.nix | 3 +- users/aspen/system/system/modules/metrics.nix | 197 ++++++++++++++++++ .../system/modules/prometheus-exporter.nix | 31 +++ 13 files changed, 240 insertions(+), 151 deletions(-) create mode 100644 users/aspen/system/system/modules/metrics.nix create mode 100644 users/aspen/system/system/modules/prometheus-exporter.nix diff --git a/ops/machines/all-systems.nix b/ops/machines/all-systems.nix index 5df09fa0b..14a8b6b26 100644 --- a/ops/machines/all-systems.nix +++ b/ops/machines/all-systems.nix @@ -4,7 +4,6 @@ sanduny whitby nixery-01 - volgasprint-cache ]) ++ (with depot.users.tazjin.nixos; [ diff --git a/users/aspen/secrets/bbbg.age b/users/aspen/secrets/bbbg.age index d8294b047191113c4b2e4e646094c9b9ff94a291..379441b74f5c2ca9781738742b3fe9cd17e57231 100644 GIT binary patch delta 545 zcmcb{a*btzPJOOPVTns*Rk2ras-wPZP+3U0k$-4ql~Gwreo#qJs-;s_aaNdzxm&4m zI#*$#OP+zYnQKbApLb|Qm~U8)X?9_{zez|{nMqWpn}3+OMU}fjU~!(IE0?aFLUD11 zZfc5=si~o*f=`;0TT-Bct524>aY(XfcDPZMn_rMqm7AZRbD(98rJ;|fYm`rBx=D#^ zpoOEhuVJMLS6XUmg;{P?re#Hbx@&q!q?dQHYhgi-M_Op4M^<>LqhGkMpI4}UuwjPl z#E;_j9z_PGo)Jk#*#>Ea>26utd3i?RQ3W2kE|yhJ1%_ol*^xn|E;)u_fvzT8ZeCf5 z>6wQ4Kt-BsKD=PnS9UFZN1EG8+~} zFIvZ|Ed68U#EY{Xj(=U`m+;JZrp)K9o;7b+Mfv}}o1``CWSyciuZK<0&8oJ0HEixH z-fACy6tFS!NnHQN*^{IrLe;bXr?*JdwuToZrZlo9MKvk)i%)-7E-cI^H%Cw8%AMuu p%r=JpeIcgpQM*s5mF}I8GnK#O)NkuXRsAROeP5Pl8%#|}1pstx$YlTk delta 545 zcmcb{a*btzPJO1EQGRxoXSTO@Mp~3nx_N#|m4CKnYI3G=T1iGph;d++caC|1L193a zCs##gl9y*cABMSdT~Tnh+9;?VQ!JWOQnlxj(f6yq?u>2 zZ>4KmgmZW~S7>H&WS)0wT5yi9r@zNzTQ| zxs_&FiA4p8IR*ZH!7fE!Za&VQCT^K&`H3m|nSo{IIX)%9xyA-uy1Kdw##vz%<>8T) zPDYkKIof&7PR0>cp}yv(spYbQ`Ju%|+6E!TT&aF%1Y8pgCZ#k!-u*1e z#`^O9@FlxEUQAwS*54WH{-al1Qu5j(*+V7wyc)CEUnI4$WM96T9(^<;$o^$XTq&#L z>4vQ1ybD;1+*dg1ysu}A^*wvEHtMbC`j)!e7&CSq{9AR}w@m8Rju}ZWxnrk? z`rK1^7?8gG%~{roXD3}3w9t-|{jeu;BlF3#`7TF3g*}+LV|D34DW|KhGBuYX1wQPs r@iDBvz-`&8y|HTPuSdyCPULelX`MJYXYH}GHT);Ob4EWjE_n?A;;PkF diff --git a/users/aspen/secrets/buildkite-ssh-key.age b/users/aspen/secrets/buildkite-ssh-key.age index 062be3b9bd98e930b34b4c017a6e640cbf3d267a..61ad416385c6a4f6f8a4e2e2fa9bdcb6e959f1dd 100644 GIT binary patch literal 3833 zcmYdHPt{G$OD?J`D9Oyv)5|YP*Do{V(zR14F3!+RO))YxHMCT4F7R?n4^%MDH}}^M z)%UU}EOgHFaZicJa;x$)iwG!BvEGT2&v@r4`>ve~SvNaeyi1WgJEMm^h3(Rs zDj$x`&1p5ZQZHT|e)=ifGIoXme{Obv&5iz8?FAS1Hid4R)gJTB_=x&*=GteqTUsJh zek4dueEYSAS zJpqs97}s1*iuap0vtiQxl}x?4Kb9%AMmDpvIR4rv^sV9Vjk0X%h8+_R@GNH!(G2>1 z)%?-c^@-6M+##(c+txVz-63JSIBoU*X|0JV1)Nzzc2e`Qehbu!>P_qDabdX~SpJZg z<-LfQXF&A}xp|j*>NYKZZLi2)vSMf7Zg;hsh5JMDwN3k5`*++s_UJPIOwk*~{yzL^ zA9Z_9n&cG9%iNfs5%Or(G1J9$<}A^yPF&q{9nI8^14<*Ojl8IME6BrnaYJHR_w06!+YbN>w3vkXRIDxoRN5K zNekx}7S;#N|F5tzJ4c*J^}YP%(i4{1RT5m)J{DTGDqoiLby->OzEha*E4*n<)T#Gf zHb2a7*DlFtW4zQ?ZJceTy}op^`)bzSmq)GY913~*WW&F3u(%{|_M1NY%j2T1X{ujR zr}5Qnj!c{={-W7znuq4=B@upXhbKqAo758@JnzQ>fo05z<-sqqr^l+k)jSwroFg zz|^De=Ji8S>r~yD10TkQGVX1znW5~nx>;$-m6))N6UrayJHP%J7u7MnVea44(sz{? zSiG>Rd7a?zD1Py_o^RN;s5i&AdvD5)Ke~MJR^dlqzltngHKWY&&P3-eTRuNMaLR%) zML$n6`yT(Ke(v5(t@+u zTPCu8dT+4ly>FoVOz}ytCYc(i zE%X!eJuq*xg8q}`*%80)n*DVTnQAAtZqJ=VTPlxzku~|Q`q26RMTV5P*uIzf(>$Mr z)veyns`o4R=Cl*cIn04_&HrJZJ!^>bLT#1IrOK;im{}`E=x0E=~=5< zvGbZC0r#!6BPyyhUjILC)OzXbi4w0AvHS*cr<2x?v$7s&T|aJWWfOg`N;z|H;q#DB zQtuqyHr(n8c+bRg^7GpGDMANs{V2b*)$2iYePz$CC?VzJC47r=j_f*kvovr<;o0QI z7#GCD z6K-J2Kx0e0tn-^j^XXA%03mEguW;H5*3Oh7E z?US3t>8xj$+fOkTa|Q*!nmMOq_2KJlulJVxx^t+bCg{OC?#36*r88#7G8ricwM~g} z41cbYJzZn&>Q6Uyw~PP!@h+zEhTfsKr|NUp`pn~CDOmqt=A=)~iF4F^=N}WFzlcd^ z-sjk2=babYBztY=Ec;?~BSBp;T7S0snGbX3dWNY^X#2XyId0Om_t$S;Yj5ydQ04Si z>=%1Yge)7^^Psi44;x>+aJN|1pTLmx<3^BW^Nq)^%c6o`o%OG@?$ej`OS;-|XG6xp z-%dKevZiWXYT13I^y{}RZkyL6DMbY6>iUgLWFXF4tcnMuy zCLZWqsiCmom$q#>^ILP9IkTJ+*ELD z*1EI%9{RrgHIefzV{c=9(%DPh<|d3!4?NP|@;SS)_{DrSfdyY$1+VROV&$HMmqkHd!7Ha;L=Aw z?ZBTl!UcSaD>s(#9z81_=45_ICMsk1^mTFh3$}C5IpFlzt>hWYynUZU^`0ptc&v{) ztn$zLxpMKgc-0H%W(Hi^_4qA+&?cXUQFfB$#fvwK*D2Q=uQ_ST|NF`{ncd&s#x*lm zi^vr!xjer1d~MFX$7`c*+;F!vHe*{CbS_Cl?A(4A>*6U)k>Xc#;uHRu)>Jy!&pNS$ zW$m@oKP_ZC1bXIc{CNA&q`F$9h|lKx&KvvR+3_a{e{22NQQRAQrFHAa^V=JX|D5|# zDfTE%{X-sC!Nr&&8Ru%w4RQMYPTJ+_m8$1r*UY&0yw$$fktgZPqZ2KEkF472(zQ!) z?e`ltNxgRqWs6TPTl>z}&pYSZpY|X>C%Mdtrgn|9S2qr_Hw0r$jYh`1pO9ba(ZiD-R~kl1ZyuV8Yw3B$$-ovFcm$hRa*Ge{i%v z)hYTZS|U6(Iap%X1F=rU*OS}7PVJr_GV%K4#zM8?T8^2HmkeJYIJoNdedW~>AqQmR z`@Hq4Uxi0$#p@Q@a-Ruz_E}&n<*+$;t9s3~SuavOZEW^V>AE1f>RZO8w=Y~1=l@ES z6PXbo7Q%bttI_6w#VwN>?>?*I{9<-@_Nu8FF^oFwLU+6jo+Xx>o@xuI6ZeA>r_8A4kZv&#Qps=iCI{z@xn+vz~_qg+W_`q<}R*tF}`k>$(}A9-3F+w@v; zUrSEp0|N)E%!cqbhvF+sA3fBr54kn7a?1isv%94Z{(p`FSEYY-kp}is`qm@SN-2-KZM$0iCtYp>>-<}2YlR%CBy*|uD2 z35Rj(>%^}zzjMNhI{!Ub6fg6+TQbR;+xOvuFE-tWf2g)8tr43Z=dS-(JA+BP_>8aC z>%%|i%;rB>EnnYz>1fh@@tyMyM{e!1m@}(eJH;l^;!|1Jq}v{$#SNLO-xg1Pe=v}X zGvrDBjnX{5>511(mz=rw<6h~r;J(@WBjusn>9XkSu#W#+}Euyub=zn-AGQuLa`bCo?QcO>;2Cf74tactam z)Nr(bPUNawe&D6FfY;94lD97_jOE44>GI>4Ry{5bL7e~cdQCA%!+W)PYw;V zOm__mjB?KfN&5F}OhA(mg6TBAd%3CC%43EIHjTAlp|zBi}czs3apK z)6~==JvY(V%P7|}FyAOREF~)_qa58f_f%K&;y?upBa?DpUytbf-?S`O=Sq(<1LNYzFyrE6i>$oD z2zMv{47c1ur-F*&sAMi(U0sDL-wfA+$dnKhKgVGI(u&A*%RFO~+@#R#Vn2hTAOp9M zDi^Z=?*NY!%X}{GI}+Vz)<6EZW?t{COR2Y=eU51vsanHh@m2MVWQ9wUz%@O?Y4v%eCuYd)E@~*0o!g7Ox8`m}2<) zYJbY9OuM~{%#VjEepB%GGBL4jD|Bwno5_Afcj_6BIPq%EM?L=16TT_Wd;7?RY4To+ zsV;}+SGF9F`RAo3T;<$$P+8IC@OQJ7$=}vQ=UH?A{_%9lF7t9z{y475Y9*HgYOh@1 zYu%6#CzI{F_H;^n!30Kw5KrIPmFqkkeb1g3*t@p$w5#K$s*A6~66`IcUb(U=wscj7 z`PbYQwVJ%PwS}=V?3JrubYhQid$y4LlD&5mW=Nd!&OB@RWo2`nTkNYM-?fVuF<30& zj@e%K@Y>ofyJA*&vu8ELu6bgq|LckLS?!e94ZH8`DgIe0I&qfr{f0+NXSc5rWPe?B zA-G}7<>MDmDpxBl+{EitB)Qk5zhL?ft_ci%UhSbzrnQ-FoG8$8l=Z;36YatZ7vpX? zeP8ioiQ*kTzo4U5GxsmDa&-{)-^o{(v0Y#g_`tM$^GOoJ)s z-)zb8yjnhM<&=B|nIrCzyR59waU9+h6CHn5-eIMHzD1{6?CEJIXYo(>ax#*TGt~I-sj0nT;6)`y(nmBHn+w?mtk)Z za}~=~tE1gX`8(x)*YZE%mT!yNzOQo9Q_h{rE1zfQ#LD~YO)KJlr=zqwrRvAKSvlhE zZOdbn&xJ=fac7*{S><#^a{n$apVhYnKRMY?-FZbQbI-!k6(Ke$2blSu$9^lSc)4rl zwG##lTe>b;C;z;ib-$0@{_3Sv%g-AFc0_D5UA6M~ud2DH_q9|^{Cbl|dMm@TaNELF zbE1~M@_tyze8zQsyxs1J$#eQ+L){wZy6)%*(~XOCe0ZQ-zc;%=r~2~0C4A}cw#)0S z%sD3Hk-V_-_C#H`|NSjbCQ8UR>U4h)eOuV^ewKN1rr_zL>rZ!Yt3MRNDY;Jn$xWNv z3BIcS%;)&_^Vg*P-9KC8iuE?{YbkT4J>m4%IAkrHE&Vp{QQ$Va zlkUosUS~bv7Z%-QPaGeS$qe)A&zv$P%8{YZCdz#L_dq-%@crKtMj@H3#mr^$ElpPhnmvLf zCf&UqU&43cSL|~|(YKaII3`9&-z{hPVRvQXlpsfz`JBrVp8pAac}ya>{DAgh+d@H= zy`NTvwiJ9m_&)ynN%kGgV*9o+XvlK2Y>_DAo2q5wq`lD+LIyI}6OW;d*wdB3@E&)oIxP$TPxWlJ^fsyI)WDm^$?ujd`RlTZh08bL&ORTq^hO1du%!AF>le+-xaok zQ`G!LkL`Z^*DFNd%VdqI3hTQFMJB0kuhNta7N7svYE5o3vshxhpluS zeq8)`qu6xA==Q6BuRM5gGI6G2@*d_)*XvqdOW1v;7u;M?d;B1u$O8q{ zyg6^Sm+zaHbNJ{V-XBh{x}Qvp2~G&KoFRKR{Mdo%j=RI-BE8QVZHlfldDEQ|z4+`R zJ%3{@@9=WlEsrzbDwX!$nR97PaO-ELe~Y=5nT;G01KylJy`*?q)5d*rhus~Lf8XLO z`8z)?bMg1>+4*Yy)}A6$t4cNhzT2#vx$5%QJ7HG}oJF0cPsj;Rh|vyI%~PMRwepkg z#F|&LQn(!$TV5ai6Q*9pAbgrXk-zy5gE52J+Z!A6ni84s?96Z#p5bm_)1r7RbhQExDtIESr>rU6Y@ad9B*{O$jrV1s z$3D?^t$&ukzaLsU)9A&i^ruEg*8fUtNNy5&$!TfAVvxYe`q@6k-y)(iY|flpk;)$w z-8&Zjovzh;VDi!B+`rxCyUHCBS(meFeb$MMM;|c0QdD?mP=0;S(Uwrd%H)Ne{f@nh ze*2qE;Iwl1a>4aS^)wfmTlR|SUPtEiugHzxwe0QltW#pgS`LTp4cPSMWnE<~pIYMX z6Yh+QbhP$mh;8cm63$TEe_|bzk<9hk2KNn~RD7+S!@#p@TMc8g=gY8@TFPhVPTkXR z^SBFxL1{%Yafw0MTK zdGs&7(vs$#g`2H;E3iBAn{PA%)gEeTdZHFUJOZ>-XeZFPC~!?n3)-0Xx%wg z#>07OxmwbZyHd7SM2X*hR&%@1WR+f+!R0HLzHC1cTcKAc!uz=DPO{_alRa00OxwLb z{Z&4AVWQUCx6hTOQ?F~(9gg0=zw_3c4fjrD7A4Eho)U6)<>km$wT}5^Ia8i}vtDkn zD0-nuR5su3r{8@;N*5k1&N}tKHH`c57g;`UiRg|y$1S4T8fuxCJ(Tr@Z}VjRRgJFW zXW39w?6%-yYM`Xvv&xl=rRUG)u`qCY7rJ`NwwL@2s|CXf*$n?Hes&DhadC7>44gP6 zAzwrEru()I-On-?zA=6aOFQ+Vlg;&WpuEoYZA$kZKmM^MKE{2TdfdTEkE2dPr+MB; zefSx=dCITlwZW2!e^&S(UK9Cd>XT2c7aVOCPd~|8_AlYNtO(1>IcygXME%%!%JHn~ z)%OZjN0d)DBxVRrOZvUf88$Hydy8m&xxaoPf1>Vn+9W4hMnk4^OtfG zM!7Xkzdy{$mH**Y$D?@kbS6jovq>k_en0%r{5Ll;aO%|~8D29sS)D%AJVVrA!j6=k zFIC0+ZRc&v7BPLG&fFj|B2_a zG5cdBCdGOvU(Nf^y{t0s;=AgZbH6zz@2)xb_PY5=i_~d<`{sS_-j&+cTd8xYb7|I^ zJ>G{O95PJX*FW2dmot6i;$Fa3P~ zpS|5Zm+ zT`{IpLJV(zD(Bc>yEp3YRUr}Mh;FMt$0x=y^W0(!$q2}LyMtGA=i8Nc_CC{}b5Y`> z|Gl_99s9zj1xHQ1tzLKj?effv9!q_bKW41Dx40%IDJ77r#*A(6*`_IV*@CrQcQj7( z#d6L#wDIqG{;fB!J5`tbjkFDozUs<#n0)@4nb?roZrpErM-^vf+#S*zZi z5nO$8{hRC~K2!YN-07u2lX@$O5(bXQFe{+r8EXFL|{ zJUPwRiS^IVjKxBmEBx=TeuiGcenMm7QVfDI8^N2tlLwBzs-2`#n5ZV^^=c&RC8^Lo_#=0|JvyV zI+8aHQ}1qY3@?}}=Wns-R>qm*-=i-GI7EALN*xjH%HGs>^`)cP9ft{D!sjH-zqn&5 ze~QA=s6@LC{lZnhZm?}Rc(hf*U-Gc+t6Za64b$JB-KN#1GT--6pTxORW{*RL;K4-c zF7{~$i&i~yl9R7EaB}Jz+ra* zi+@)NztYQObE~j$5I?@5o74G&uBGB0ru$Y+N9M{orB$7I{%mJ@h1db<8M$S`6U`TP zg{)apa9(TDOMi)&hd%{YZND&aw{51j*oqyc>R%K-zgju}t2Fbc3O?tTTbCCFsqfKR zblLjx>Z$+r9o218(xfa8PTp!zvh~hc##cY$JKR+R-2!vJ^)dW=t9f|Qk091bPq#@d z%uJd0x8Z6jQ(X4UoXhFoUbhQ{98cjqbv#w**7;`VAASGAzI~B&TE}E_cJraVta}U2 qUHYIj@7o8X$F0Rbg4X|fv^2sq_3#`v)98!U`zD^BD=c(T{4oIKs|Wr7 diff --git a/users/aspen/secrets/buildkite-token.age b/users/aspen/secrets/buildkite-token.age index f55b31fb08ed9531a85c926063abe95597f0b651..5bd4923de34fab9d84154f9efed0788a76c15ad3 100644 GIT binary patch delta 429 zcmaFN{Fr%yPJN1YxRIr|Q?hoLsk?W6MR1;@nTc7tueooOXNj?kQD#|?cZrW*M7oP> zI+vGcT49xGXis3?TS1|5rddRaN3fG~va^qNK9{bYLUD11 zZfc5=si~o*f=`;0TT-Axq-A8FkGXrLpQoE+m1SX8k(*g+qCs}4epW@Mr*l+Vj-iu( zWmQQSe zj;V=J6~=+NW{DxbWw|bq6=nveCWRh;{$)uq0hwVDQAJVZT<>iaJ@0Hh{>0dt@ej+X z!`0oQ*g94 zBbQTHq)};>W14nGvZsl5sky5~MPa&`d3K4Xhhb@uzd@P4nPEvqX?}2OAeXM4LUD11 zZfc5=si~o*f=`;0TT-Bce?Y!lRbgI=Q&d2NM?`2~kXx2vW{8Woc9wT|K$4f2TXJN1 zMP_7*b6%z|mywBap;xhKfSZ?dT9l`0UP)$Nx|553MpacxW_e_$V?=S4eyMwudz5eT z#E;_jCLtAxl|?4{Miq$x>BeSGVd;+UA*m%E$)<@u8C6A9E|vy{xt^8Deg?i=M!^B@ z2D!x%`o2kKIq5DTSw5bw70y*r$;qY89!ANu+mU%uI zLEb@*B^K^cS?;9)Syd$|m2UZ#&V}K*C1sW|$$n;886K|XTy7csjOyv@Gq^2!>lz9) zZf7?+dojf=e$FyW`%MXVy;NDlODCE8N)4Fx|{CIHw{T z-8%PFSM%aP1=oxuv+VK;i_9X|q_kAGN|W>|Z~gr6T+gVWloEYk*Q9{@%3?2%u%Kcm zN3QfDr_jtOM<+|qQe$%;*9^Z@IVIJ$xYX3LyfoW_ zOIKG{q0~8|(m2f7veMBrv#Ka3H{ZnB*CMx~*vBQHywoE(GoZlTI3+LC*~Hy|>tLoa z!{Y;bLix+@m#JzjnX=?i&s3*5YWs5iQx2C+=apPk6KN7zb7K9Poss(8rT%`2EGG|y z&Jm3=^K|udmJIc_nOXkqXq4$W(~72bn_3TTlD&0sP4K1r5z7{)9-VSf@bLR$!!I%k X89GG*25H4X&Bd3w|F3_bQp*nj63>a^ diff --git a/users/aspen/secrets/ddclient-password.age b/users/aspen/secrets/ddclient-password.age index bc82063c3a286b8719878091062b47b9471f706f..3bbc2e51ffd39874fa4d647bb9c61946b1db1317 100644 GIT binary patch delta 325 zcmaFC^nz)EPQ5{ji?4oFqDfVGaZ0FTfVo+CxPeD@o<(L=d4;yCqfbamNLF%&Usa-! zD_3rrv%7PsOK_%BxVv#mNu_scU{Jc7TT)V9WtMiNiBDmqqnl}QxkrJ8E0?aFLUD11 zZfc5=si~o*f_ti~d2ygZN=|TOMt)_8kD;-?r(aq@RehewdGGfQfc^YD#KIxQAo9c6p*jh-+Scc~qKlwxLBJ*YjoB z70(*ZFR}Y_a=Ve~Dv9E2tD9V3SU!AtUh?FZm%`uIm=x@Kzh_?8Ci&SmvcAtJ7ws0f Zc<1E|z1~}&zAX3IQ8A_Q`KJQLqX38_d}{yz delta 325 zcmaFC^nz)EPQABzpqWLuewu4oO1iI4M7U3In4_6_d6jW`Zg6&mTV|00Z?I2RRd{k* zcB!MQzqxTSSD343UZJOFj)_;PQ;2VYg@>z`uZ4+6RcTqUVOUgnYK4z)aBfI>PIh52 zm#(g^f_6ljNpY%co@G*msaJ8Dg-=10wr^xqa7j@?UZk-}V2FP~aGAM#NqR{+*Mpwf zlWeXtXxPagIR9|Yb=i-rSJ~>kU$;lFmwUy9TE1AdlQ+c=&Xe|&md=?{zN&n~p>4uN YYYu ssh-ed25519 CpJBgQ qVlQpHyewtBSfFIdU8GihXC7JhGbcvQ61ZsJC20wSH4 -mZXwiTICzrG+3aCL67cO6cTWMgHkxhDyBi7tZ8l+QMA --> ssh-ed25519 LfBFbQ 78NQxflRkRMW5vSP1BEvASSQU2pZAfMwd7T2+6W7NQs -u0x986pFtnD9ZqfL3KnRrdYS5z9LRUPJhcmc8FQOuGo --> ssh-ed25519 GeE7sQ aqFQGCywSimHNbN5si0PzmESUXwROjrpTe/5UdTyYw4 -X2thEJIyOnNUsA746VwqZhH+44XBfCTvh7VOEg/zew0 ---- ndSgjJv5Tel6ovKl+SBdDHZHlszgsEhOY1HHpNDvf1s -Iʵu*1t(/X˕3ȒVGT|@K<})se9`*z \ No newline at end of file +-> ssh-ed25519 CpJBgQ PiY6IidA+GRbpjL91BVe9UdejWvi02SRcijiMOjXcm4 +XegOhgjdEdzXtz31PsGVyOZ10gH6P82Q1/txZcSxjIY +-> ssh-ed25519 LfBFbQ uqRF0nKMk1GrK+6pEBdmyHKu2ewDFlWwlKC+myey4gc +dgnX4eprSolXxCDNoVmGzGK9xLEmtmeg/cJihD4/8sU +-> ssh-ed25519 GeE7sQ ikAIyFR/qH1a+aa5mumiiDwa5o5aLsQeJKwQwMzgs1M +8htzhM5t2VnjRBrC+VrL23f9chlQjVGzjxMaFB7Arrs +--- Qm16HTo5wGUBKS0ly3OZDWp2etLyDS/zlxOHxPjS8PI +7NY6k|p2'&=mq`5T N9N)RVU-•) M(%p \ No newline at end of file diff --git a/users/aspen/system/system/machines/lusca.nix b/users/aspen/system/system/machines/lusca.nix index 16dabbd2e..4a9202187 100644 --- a/users/aspen/system/system/machines/lusca.nix +++ b/users/aspen/system/system/machines/lusca.nix @@ -10,6 +10,7 @@ ../modules/sound.nix ../modules/tvl.nix ../modules/development.nix + ../modules/prometheus-exporter.nix ]; networking.hostName = "lusca"; diff --git a/users/aspen/system/system/machines/mugwump.nix b/users/aspen/system/system/machines/mugwump.nix index 1daa92f25..4b72a2476 100644 --- a/users/aspen/system/system/machines/mugwump.nix +++ b/users/aspen/system/system/machines/mugwump.nix @@ -117,149 +117,9 @@ with lib; }; }; - services.grafana = { - enable = true; - dataDir = "/var/lib/grafana"; - - settings = { - server = { - http_port = 3000; - root_url = "https://metrics.gws.fyi"; - domain = "metrics.gws.fyi"; - }; - analytics.reporting_enabled = false; - }; - - provision = { - enable = true; - datasources.settings.datasources = [{ - name = "Prometheus"; - type = "prometheus"; - url = "http://localhost:9090"; - }]; - }; - }; - security.acme.defaults.email = "root@gws.fyi"; security.acme.acceptTerms = true; - services.nginx = { - enable = true; - statusPage = true; - recommendedGzipSettings = true; - recommendedOptimisation = true; - recommendedTlsSettings = true; - recommendedProxySettings = true; - - virtualHosts = { - "metrics.gws.fyi" = { - enableACME = true; - forceSSL = true; - locations."/" = { - proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}"; - }; - }; - }; - }; - - security.acme.certs."metrics.gws.fyi" = { - dnsProvider = "cloudflare"; - credentialsFile = config.age.secretsDir + "/cloudflare"; - webroot = mkForce null; - }; - - services.prometheus = { - enable = true; - exporters = { - node = { - enable = true; - openFirewall = false; - - enabledCollectors = [ - "processes" - "systemd" - "tcpstat" - "wifi" - ]; - }; - - nginx = { - enable = true; - openFirewall = true; - sslVerify = false; - constLabels = [ "host=mugwump" ]; - }; - - blackbox = { - enable = true; - openFirewall = true; - configFile = pkgs.writeText "blackbox-exporter.yaml" (builtins.toJSON { - modules = { - https_2xx = { - prober = "http"; - http = { - method = "GET"; - fail_if_ssl = false; - fail_if_not_ssl = true; - preferred_ip_protocol = "ip4"; - }; - }; - }; - }); - }; - }; - - scrapeConfigs = [ - { - job_name = "node"; - scrape_interval = "5s"; - static_configs = [{ - targets = [ "localhost:${toString config.services.prometheus.exporters.node.port}" ]; - }]; - } - { - job_name = "nginx"; - scrape_interval = "5s"; - static_configs = [{ - targets = [ "localhost:${toString config.services.prometheus.exporters.nginx.port}" ]; - }]; - } - { - job_name = "xanthous_server"; - scrape_interval = "1s"; - static_configs = [{ - targets = [ "localhost:${toString config.services.xanthous-server.metricsPort}" ]; - }]; - } - { - job_name = "blackbox"; - metrics_path = "/probe"; - params.module = [ "https_2xx" ]; - scrape_interval = "5s"; - static_configs = [{ - targets = [ - "https://gws.fyi" - "https://windtunnel.ci" - "https://app.windtunnel.ci" - "https://metrics.gws.fyi" - ]; - }]; - relabel_configs = [{ - source_labels = [ "__address__" ]; - target_label = "__param_target"; - } - { - source_labels = [ "__param_target" ]; - target_label = "instance"; - } - { - target_label = "__address__"; - replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}"; - }]; - } - ]; - }; - services.xanthous-server.enable = true; virtualisation.docker = { diff --git a/users/aspen/system/system/machines/ogopogo.nix b/users/aspen/system/system/machines/ogopogo.nix index 4dbb3d14e..3d41a839e 100644 --- a/users/aspen/system/system/machines/ogopogo.nix +++ b/users/aspen/system/system/machines/ogopogo.nix @@ -11,6 +11,8 @@ ../modules/tvl.nix ../modules/development.nix ../modules/wireshark.nix + ../modules/metrics.nix + ../modules/prometheus-exporter.nix ]; networking.hostName = "ogopogo"; @@ -92,7 +94,6 @@ dataDir = "/data/postgresql"; package = pkgs.postgresql_15; settings = { - port = 5431; wal_level = "logical"; }; }; diff --git a/users/aspen/system/system/modules/metrics.nix b/users/aspen/system/system/modules/metrics.nix new file mode 100644 index 000000000..0abfb27ee --- /dev/null +++ b/users/aspen/system/system/modules/metrics.nix @@ -0,0 +1,197 @@ +{ depot, config, lib, pkgs, ... }: + +with lib; + +let + nodesToScrape = [ + "ogopogo" + # "dobharchu" + "mugwump" + # "yeren" + "lusca" + ]; + + nodesRunningNginx = [ + "ogopogo" + "mugwump" + ]; + + nodesRunningPostgres = [ + "ogopogo" + ]; + + blackboxTargets = [ + "https://gws.fyi" + "https://windtunnel.ci" + "https://app.windtunnel.ci" + "https://metrics.gws.fyi" + ]; +in +{ + imports = [ + (depot.third_party.agenix.src + "/modules/age.nix") + ]; + + config = { + services.postgresql = { + ensureUsers = [{ + name = config.services.grafana.settings.database.user; + ensureDBOwnership = true; + }]; + + ensureDatabases = [ + config.services.grafana.settings.database.name + ]; + }; + + services.grafana = { + enable = true; + dataDir = "/var/lib/grafana"; + + settings = { + server = { + http_port = 3000; + root_url = "https://metrics.gws.fyi"; + domain = "metrics.gws.fyi"; + }; + analytics.reporting_enabled = false; + + database = { + type = "postgres"; + user = "grafana"; + name = "grafana"; + host = "/run/postgresql"; + }; + }; + + provision = { + enable = true; + datasources.settings.datasources = [{ + name = "Prometheus"; + type = "prometheus"; + url = "http://localhost:9090"; + }]; + }; + }; + + security.acme.defaults.email = "root@gws.fyi"; + security.acme.acceptTerms = true; + + services.nginx = { + enable = true; + statusPage = true; + recommendedGzipSettings = true; + recommendedOptimisation = true; + recommendedTlsSettings = true; + recommendedProxySettings = true; + + virtualHosts = { + "metrics.gws.fyi" = { + enableACME = true; + forceSSL = true; + locations."/" = { + proxyPass = "http://localhost:${toString config.services.grafana.settings.server.http_port}"; + }; + }; + }; + }; + + age.secrets = { + cloudflare.file = depot.users.aspen.secrets."cloudflare.age"; + }; + + security.acme.certs."metrics.gws.fyi" = { + dnsProvider = "cloudflare"; + credentialsFile = config.age.secretsDir + "/cloudflare"; + webroot = mkForce null; + }; + + services.prometheus = { + enable = true; + retentionTime = "30d"; + exporters = { + blackbox = { + enable = true; + openFirewall = true; + configFile = pkgs.writeText "blackbox-exporter.yaml" (builtins.toJSON { + modules = { + https_2xx = { + prober = "http"; + http = { + method = "GET"; + fail_if_ssl = false; + fail_if_not_ssl = true; + preferred_ip_protocol = "ip4"; + }; + }; + }; + }); + }; + }; + + scrapeConfigs = [ + { + job_name = "node"; + scrape_interval = "5s"; + static_configs = + map + (node: { + targets = [ "${node}:${toString config.services.prometheus.exporters.node.port}" ]; + labels.node = node; + }) + nodesToScrape; + } + { + job_name = "nginx"; + scrape_interval = "5s"; + static_configs = + map + (node: { + targets = [ "${node}:${toString config.services.prometheus.exporters.nginx.port}" ]; + labels.node = node; + }) + nodesRunningNginx; + } + { + job_name = "postgres"; + scrape_interval = "5s"; + static_configs = + map + (node: { + targets = [ "${node}:${toString config.services.prometheus.exporters.postgres.port}" ]; + labels.node = node; + }) + nodesRunningPostgres; + } + { + job_name = "blackbox"; + metrics_path = "/probe"; + params.module = [ "https_2xx" ]; + scrape_interval = "5s"; + static_configs = [{ + targets = [ + "https://gws.fyi" + "https://windtunnel.ci" + "https://app.windtunnel.ci" + "https://metrics.gws.fyi" + ]; + }]; + relabel_configs = [ + { + source_labels = [ "__address__" ]; + target_label = "__param_target"; + } + { + source_labels = [ "__param_target" ]; + target_label = "instance"; + } + { + target_label = "__address__"; + replacement = "localhost:${toString config.services.prometheus.exporters.blackbox.port}"; + } + ]; + } + ]; + }; + }; +} diff --git a/users/aspen/system/system/modules/prometheus-exporter.nix b/users/aspen/system/system/modules/prometheus-exporter.nix new file mode 100644 index 000000000..2916fc70e --- /dev/null +++ b/users/aspen/system/system/modules/prometheus-exporter.nix @@ -0,0 +1,31 @@ +{ config, lib, pkgs, ... }: + +with lib; + +{ + services.prometheus.exporters = { + node = { + enable = true; + openFirewall = false; + + enabledCollectors = [ + "processes" + "systemd" + "tcpstat" + "wifi" + ]; + }; + + nginx = mkIf config.services.nginx.enable { + enable = true; + openFirewall = true; + sslVerify = false; + constLabels = [ "host=${config.networking.hostName}" ]; + }; + + postgres = mkIf config.services.postgresql.enable { + enable = true; + runAsLocalSuperUser = true; + }; + }; +}