From b422ece932ec6602dc828158f825e221484564ed Mon Sep 17 00:00:00 2001 From: Thomas Koch Date: Fri, 10 Jan 2025 11:39:54 +0200 Subject: [PATCH 01/17] . --- web/planet-mars/.gitignore | 2 + web/planet-mars/Cargo.lock | 2056 ++++++++++++++++++++++++++ web/planet-mars/Cargo.toml | 18 + web/planet-mars/README.org | 18 + web/planet-mars/mars.toml.example | 8 + web/planet-mars/src/feed_store.rs | 141 ++ web/planet-mars/src/fetcher.rs | 70 + web/planet-mars/src/main.rs | 98 ++ web/planet-mars/templates/index.html | 8 + 9 files changed, 2419 insertions(+) create mode 100644 web/planet-mars/.gitignore create mode 100644 web/planet-mars/Cargo.lock create mode 100644 web/planet-mars/Cargo.toml create mode 100644 web/planet-mars/README.org create mode 100644 web/planet-mars/mars.toml.example create mode 100644 web/planet-mars/src/feed_store.rs create mode 100644 web/planet-mars/src/fetcher.rs create mode 100644 web/planet-mars/src/main.rs create mode 100644 web/planet-mars/templates/index.html diff --git a/web/planet-mars/.gitignore b/web/planet-mars/.gitignore new file mode 100644 index 000000000..251429b41 --- /dev/null +++ b/web/planet-mars/.gitignore @@ -0,0 +1,2 @@ +/target +/mars.toml \ No newline at end of file diff --git a/web/planet-mars/Cargo.lock b/web/planet-mars/Cargo.lock new file mode 100644 index 000000000..540750fbd --- /dev/null +++ b/web/planet-mars/Cargo.lock @@ -0,0 +1,2056 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + +[[package]] +name = "autocfg" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" + +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + +[[package]] +name = "base64ct" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + +[[package]] +name = "brotli-decompressor" +version = "4.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a45bd2e4095a8b518033b128020dd4a55aab1c0a381ba4404a472630f4bc362" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + +[[package]] +name = "bstr" +version = "1.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531a9155a481e2ee699d4f98f43c0ca4ff8ee1bfd55c31e9e98fb29d2b176fe0" +dependencies = [ + "memchr", + "serde", +] + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "byteorder" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" + +[[package]] +name = "bytes" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" + +[[package]] +name = "cc" +version = "1.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "chrono" +version = "0.4.39" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e36cc9d416881d2e24f9a963be5fb1cd90966419ac844274161d10488b3e825" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "serde", + "wasm-bindgen", + "windows-targets", +] + +[[package]] +name = "chrono-tz" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93698b29de5e97ad0ae26447b344c482a7284c737d9ddc5f9e52b74a336671bb" +dependencies = [ + "chrono", + "chrono-tz-build", + "phf", +] + +[[package]] +name = "chrono-tz-build" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c088aee841df9c3041febbb73934cfc39708749bf96dc827e3359cd39ef11b1" +dependencies = [ + "parse-zoneinfo", + "phf", + "phf_codegen", +] + +[[package]] +name = "clap" +version = "4.5.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b95dca1b68188a08ca6af9d96a6576150f598824bdb528c1190460c2940a0b48" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ab52925392148efd3f7562f2136a81ffb778076bcc85727c6e020d6dd57cf15" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54b755194d6389280185988721fffba69495eed5ee9feeee9a599b53db80318c" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + +[[package]] +name = "cookie" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ddef33a339a91ea89fb53151bd0a4689cfce27055c291dfa69945475d22c747" +dependencies = [ + "percent-encoding", + "time", + "version_check", +] + +[[package]] +name = "cookie_store" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2eac901828f88a5241ee0600950ab981148a18f2f756900ffba1b125ca6a3ef9" +dependencies = [ + "cookie", + "document-features", + "idna", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "time", + "url", +] + +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" + +[[package]] +name = "cpufeatures" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16b80225097f2e5ae4e7179dd2266824648f3e2f49d9134d584b76389d31c4c3" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + +[[package]] +name = "der" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" +dependencies = [ + "pem-rfc7468", + "zeroize", +] + +[[package]] +name = "deranged" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b42b6fa04a440b495c8b04d0e71b707c585f83cb9cb28cf8cd0d976c315e31b4" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "deunicode" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339544cc9e2c4dc3fc7149fd630c5f22263a4fdf18a98afd0075784968b5cf00" + +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", +] + +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "document-features" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb6969eaabd2421f8a2775cfd2471a2b634372b4a25d41e3bd647b79912850a0" +dependencies = [ + "litrs", +] + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "errno" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33d852cb9b869c2a9b3df2f71a3074817f01e1844f839a144f5fcef059a4eb5d" +dependencies = [ + "libc", + "windows-sys 0.59.0", +] + +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + +[[package]] +name = "feed-rs" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4c0591d23efd0d595099af69a31863ac1823046b1b021e3b06ba3aae7e00991" +dependencies = [ + "chrono", + "mediatype", + "quick-xml", + "regex", + "serde", + "serde_json", + "siphasher", + "url", + "uuid", +] + +[[package]] +name = "flate2" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c936bfdafb507ebbf50b8074c54fa31c5be9a1e7e5f467dd659697041407d07c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + +[[package]] +name = "getrandom" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "globset" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15f1ce686646e7f1e19bf7d5533fe443a45dbfb990e00629110797578b42fb19" +dependencies = [ + "aho-corasick", + "bstr", + "log", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "globwalk" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf760ebf69878d9fd8f110c89703d90ce35095324d1f1edcb595c63945ee757" +dependencies = [ + "bitflags", + "ignore", + "walkdir", +] + +[[package]] +name = "hashbrown" +version = "0.15.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f16ca2af56261c99fba8bac40a10251ce8188205a4c448fbb745a2e4daa76fea" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "httparse" +version = "1.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d71d3574edd2771538b901e6549113b4006ece66150fb69c0fb6d9a2adae946" + +[[package]] +name = "humansize" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6cb51c9a029ddc91b07a787f1d86b53ccfa49b0e86688c946ebe8d3555685dd7" +dependencies = [ + "libm", +] + +[[package]] +name = "humantime" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" + +[[package]] +name = "iana-time-zone" +version = "0.1.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "235e081f3925a06703c2d0117ea8b91f042756fd6e7a6e5d901e8ca1a996b220" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "icu_collections" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" +dependencies = [ + "displaydoc", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_locid" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" +dependencies = [ + "displaydoc", + "litemap", + "tinystr", + "writeable", + "zerovec", +] + +[[package]] +name = "icu_locid_transform" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_locid_transform_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_locid_transform_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" + +[[package]] +name = "icu_normalizer" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_normalizer_data", + "icu_properties", + "icu_provider", + "smallvec", + "utf16_iter", + "utf8_iter", + "write16", + "zerovec", +] + +[[package]] +name = "icu_normalizer_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" + +[[package]] +name = "icu_properties" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" +dependencies = [ + "displaydoc", + "icu_collections", + "icu_locid_transform", + "icu_properties_data", + "icu_provider", + "tinystr", + "zerovec", +] + +[[package]] +name = "icu_properties_data" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" + +[[package]] +name = "icu_provider" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" +dependencies = [ + "displaydoc", + "icu_locid", + "icu_provider_macros", + "stable_deref_trait", + "tinystr", + "writeable", + "yoke", + "zerofrom", + "zerovec", +] + +[[package]] +name = "icu_provider_macros" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "idna" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +dependencies = [ + "idna_adapter", + "smallvec", + "utf8_iter", +] + +[[package]] +name = "idna_adapter" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" +dependencies = [ + "icu_normalizer", + "icu_properties", +] + +[[package]] +name = "ignore" +version = "0.4.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d89fd380afde86567dfba715db065673989d6253f42b88179abd3eae47bda4b" +dependencies = [ + "crossbeam-deque", + "globset", + "log", + "memchr", + "regex-automata", + "same-file", + "walkdir", + "winapi-util", +] + +[[package]] +name = "indexmap" +version = "2.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62f822373a4fe84d4bb149bf54e584a7f4abec90e072ed49cda0edea5b95471f" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + +[[package]] +name = "itoa" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d75a2a4b1b190afb6f5425f10f6a8f959d2ea0b9c2b1d79553551850539e4674" + +[[package]] +name = "js-sys" +version = "0.3.76" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6717b6b5b077764fb5966237269cb3c64edddde4b14ce42647430a78ced9e7b7" +dependencies = [ + "once_cell", + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.169" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" + +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + +[[package]] +name = "litemap" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee93343901ab17bd981295f2cf0026d4ad018c7c31ba84549a4ddbb47a45104" + +[[package]] +name = "litrs" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ce301924b7887e9d637144fdade93f9dfff9b60981d4ac161db09720d39aa5" + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "mediatype" +version = "0.19.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8878cd8d1b3c8c8ae4b2ba0a36652b7cf192f618a599a7fbdfa25cffd4ea72dd" +dependencies = [ + "serde", +] + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "miniz_oxide" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" +dependencies = [ + "adler2", +] + +[[package]] +name = "native-tls" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1261fe7e33c73b354eab43b1273a57c8f967d0391e80353e51f764ac02cf6775" + +[[package]] +name = "openssl" +version = "0.10.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6174bc48f102d208783c2c84bf931bb75927a617866870de8a4ea85597f871f5" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45abf306cbf99debc8195b66b7346498d7b10c210de50418b5ccd7ceba08c741" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "parse-zoneinfo" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f2a05b18d44e2957b88f96ba460715e295bc1d7510468a2f3d3b44535d26c24" +dependencies = [ + "regex", +] + +[[package]] +name = "pem-rfc7468" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" +dependencies = [ + "base64ct", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pest" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "816518421cfc6887a0d62bf441b6ffb4536fcc926395a69e1a85852d4363f57e" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d1396fd3a870fc7838768d171b4616d5c91f6cc25e377b673d714567d99377b" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1e58089ea25d717bfd31fb534e4f3afcc2cc569c70de3e239778991ea3b7dea" +dependencies = [ + "once_cell", + "pest", + "sha2", +] + +[[package]] +name = "phf" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared", +] + +[[package]] +name = "phf_codegen" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_shared" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67eabc2ef2a60eb7faa00097bd1ffdb5bd28e62bf39990626a582201b7a754e5" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pkg-config" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" + +[[package]] +name = "planet-mars" +version = "0.1.0" +dependencies = [ + "clap", + "env_logger", + "feed-rs", + "log", + "serde", + "slug", + "tera", + "toml", + "ureq", + "url", +] + +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + +[[package]] +name = "ppv-lite86" +version = "0.2.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" +dependencies = [ + "zerocopy", +] + +[[package]] +name = "proc-macro2" +version = "1.0.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quick-xml" +version = "0.37.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165859e9e55f79d67b96c5d96f4e88b6f2695a1972849c15a6a3f5c59fc2c003" +dependencies = [ + "encoding_rs", + "memchr", +] + +[[package]] +name = "quote" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "regex" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" + +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustix" +version = "0.38.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a78891ee6bf2340288408954ac787aa063d8e8817e9f53abb37c695c6d834ef6" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.59.0", +] + +[[package]] +name = "rustls" +version = "0.23.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" + +[[package]] +name = "rustls-webpki" +version = "0.102.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + +[[package]] +name = "schannel" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "security-framework" +version = "2.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "serde" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02fc4265df13d6fa1d00ecff087228cc0a2b5f3c0e87e258d8b94a156e984c70" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.217" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a9bf7cf98d04a2b28aead066b7496853d4779c9cc183c440dbac457641e19a0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.135" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b0d7ba2887406110130a978386c4e1befb98c674b4fba677954e4db976630d9" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "serde_spanned" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +dependencies = [ + "serde", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "siphasher" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" + +[[package]] +name = "slug" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882a80f72ee45de3cc9a5afeb2da0331d58df69e4e7d8eeb5d3c7784ae67e724" +dependencies = [ + "deunicode", + "wasm-bindgen", +] + +[[package]] +name = "smallvec" +version = "1.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" + +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tempfile" +version = "3.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a8a559c81686f576e8cd0290cd2a24a2a9ad80c98b3478856500fcbd7acd704" +dependencies = [ + "cfg-if", + "fastrand", + "getrandom", + "once_cell", + "rustix", + "windows-sys 0.59.0", +] + +[[package]] +name = "tera" +version = "1.20.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab9d851b45e865f178319da0abdbfe6acbc4328759ff18dafc3a41c16b4cd2ee" +dependencies = [ + "chrono", + "chrono-tz", + "globwalk", + "humansize", + "lazy_static", + "percent-encoding", + "pest", + "pest_derive", + "rand", + "regex", + "serde", + "serde_json", + "slug", + "unic-segment", +] + +[[package]] +name = "thiserror" +version = "2.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3ac7f54ca534db81081ef1c1e7f6ea8a3ef428d2fc069097c079443d24124d3" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e9465d30713b56a37ede7185763c3492a91be2f5fa68d958c44e41ab9248beb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "time" +version = "0.3.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35e7868883861bd0e56d9ac6efcaaca0d6d5d82a2a7ec8209ff492c07cf37b21" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" + +[[package]] +name = "time-macros" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2834e6017e3e5e4b9834939793b282bc03b37a3336245fa820e35e233e2a85de" +dependencies = [ + "num-conv", + "time-core", +] + +[[package]] +name = "tinystr" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" +dependencies = [ + "displaydoc", + "zerovec", +] + +[[package]] +name = "toml" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit", +] + +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" +dependencies = [ + "serde", +] + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + +[[package]] +name = "ucd-trie" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" + +[[package]] +name = "unic-char-property" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8c57a407d9b6fa02b4795eb81c5b6652060a15a7903ea981f3d723e6c0be221" +dependencies = [ + "unic-char-range", +] + +[[package]] +name = "unic-char-range" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0398022d5f700414f6b899e10b8348231abf9173fa93144cbc1a43b9793c1fbc" + +[[package]] +name = "unic-common" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80d7ff825a6a654ee85a63e80f92f054f904f21e7d12da4e22f9834a4aaa35bc" + +[[package]] +name = "unic-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ed5d26be57f84f176157270c112ef57b86debac9cd21daaabbe56db0f88f23" +dependencies = [ + "unic-ucd-segment", +] + +[[package]] +name = "unic-ucd-segment" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2079c122a62205b421f499da10f3ee0f7697f012f55b675e002483c73ea34700" +dependencies = [ + "unic-char-property", + "unic-char-range", + "unic-ucd-version", +] + +[[package]] +name = "unic-ucd-version" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96bd2f2237fe450fcd0a1d2f5f4e91711124f7857ba2e964247776ebeeb7b0c4" +dependencies = [ + "unic-common", +] + +[[package]] +name = "unicode-ident" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb9e6ca4f869e1180728b7950e35922a7fc6397f7b641499e8f3ef06e50dc83" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + +[[package]] +name = "ureq" +version = "3.0.0-rc5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77f9bd1d03fe3bf87c90b115a0e09e22535cbcff3f8d0a6487524f006325f173" +dependencies = [ + "base64", + "brotli-decompressor", + "cc", + "cookie_store", + "der", + "encoding_rs", + "flate2", + "log", + "native-tls", + "once_cell", + "percent-encoding", + "rustls", + "rustls-pemfile", + "rustls-pki-types", + "serde", + "serde_json", + "ureq-proto", + "utf-8", + "webpki-root-certs", + "webpki-roots", +] + +[[package]] +name = "ureq-proto" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a27729fd2c15426f48992a911ce31bd1f29c1cd0898c2c86b410d24f51c99eb3" +dependencies = [ + "base64", + "http", + "httparse", + "log", + "url", +] + +[[package]] +name = "url" +version = "2.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", + "serde", +] + +[[package]] +name = "utf-8" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" + +[[package]] +name = "utf16_iter" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" + +[[package]] +name = "utf8_iter" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "uuid" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b913a3b5fe84142e269d63cc62b64319ccaf89b748fc31fe025177f767a756c4" +dependencies = [ + "getrandom", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "version_check" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" + +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" + +[[package]] +name = "wasm-bindgen" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a474f6281d1d70c17ae7aa6a613c87fce69a127e2624002df63dcb39d6cf6396" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f89bb38646b4f81674e8f5c3fb81b562be1fd936d84320f3264486418519c79" +dependencies = [ + "bumpalo", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cc6181fd9a7492eef6fef1f33961e3695e4579b9872a6f7c83aee556666d4fe" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943aab3fdaaa029a6e0271b35ea10b72b943135afe9bffca82384098ad0e06a6" + +[[package]] +name = "webpki-root-certs" +version = "0.26.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cd5da49bdf1f30054cfe0b8ce2958b8fbeb67c4d82c8967a598af481bef255c" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "0.26.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d642ff16b7e79272ae451b7322067cdc17cadf68c23264be9d94a32319efe7e" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "winapi-util" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + +[[package]] +name = "winnow" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39281189af81c07ec09db316b302a3e67bf9bd7cbf6c820b50e35fee9c2fa980" +dependencies = [ + "memchr", +] + +[[package]] +name = "write16" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" + +[[package]] +name = "writeable" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" + +[[package]] +name = "yoke" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120e6aef9aa629e3d4f52dc8cc43a015c7724194c97dfaf45180d2daf2b77f40" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zerocopy" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" +dependencies = [ + "byteorder", + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zerofrom" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff3ee08c995dee1859d998dea82f7374f2826091dd9cd47def953cae446cd2e" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" + +[[package]] +name = "zerovec" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" +dependencies = [ + "yoke", + "zerofrom", + "zerovec-derive", +] + +[[package]] +name = "zerovec-derive" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/web/planet-mars/Cargo.toml b/web/planet-mars/Cargo.toml new file mode 100644 index 000000000..92a58ebee --- /dev/null +++ b/web/planet-mars/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "planet-mars" +version = "0.1.0" +edition = "2021" + +[dependencies] +# ammonia = "*" done by feed-rs +clap = { version = "*", features = ["derive"] } +env_logger = "*" +feed-rs = "*" +log = "*" +serde = { version = "*", features = ["derive"] } +slug = "*" +tera = "*" +toml = "*" +ureq = { version = "3.0.0-rc5", features = ["brotli", "charset", "gzip", "native-tls"]} +url = "*" + diff --git a/web/planet-mars/README.org b/web/planet-mars/README.org new file mode 100644 index 000000000..5d30e42e3 --- /dev/null +++ b/web/planet-mars/README.org @@ -0,0 +1,18 @@ +Simple planet like planet venus but in rust and maintained. + +** todo +Also see todos in the source files + +*** do clippy and rustfmt +*** TODO error handling everywhere +*** write templates for html and atom/rss feeds +*** use a nice lib to process the config file +- should check whether dirs exists and are writeable +- should check whether feed urls can be parsed +** Credits + +While writing this, I read and also copied code from: + +- [[https://docs.rs/crate/agro/0.1.1][agro]] +- [[https://github.com/kitallis/hades][haded]] by Akshay Gupta +- [[https://github.com/djc/planetrs][planetrs]] by Vagdish/Adau, Dirkjan Ochtman, Josh Matthews diff --git a/web/planet-mars/mars.toml.example b/web/planet-mars/mars.toml.example new file mode 100644 index 000000000..dfaba8734 --- /dev/null +++ b/web/planet-mars/mars.toml.example @@ -0,0 +1,8 @@ +bot_name = "planet-mars" +feed_dir = "/var/lib/planet-mars/feeds" +from = "thomas@koch.ro" +out_dir = "/var/lib/planet-mars/out" +templates_dir = "/var/lib/planet-mars/templates" + +[[feeds]] +url = "https://blog.fefe.de/rss.xml" diff --git a/web/planet-mars/src/feed_store.rs b/web/planet-mars/src/feed_store.rs new file mode 100644 index 000000000..6b162de8c --- /dev/null +++ b/web/planet-mars/src/feed_store.rs @@ -0,0 +1,141 @@ +use feed_rs::model::Entry; +use feed_rs::model::Feed; +use serde::{Deserialize, Serialize}; +use std::fs; +use std::io::BufReader; +use std::path::PathBuf; +use ureq::http::HeaderMap; +use ureq::http::Response; +use ureq::Body; +use url::Url; + +#[derive(Deserialize, Serialize, Default)] +pub struct FetchData { + pub etag: String, + pub date: String, +} + +pub struct FeedStore { + pub dir: PathBuf, +} + +impl FeedStore { + pub fn new(dir: String) -> Self { + Self { + dir: super::to_checked_pathbuf(dir), + } + } + + fn slugify_url(url: &Url) -> String { + let domain = url.domain().unwrap(); + let query = url.query().unwrap_or(""); + slug::slugify(format!("{domain}{}{query}", url.path())) + } + + fn feed_path(&self, url: &Url) -> String { + format!("{}/{}", self.dir.display(), Self::slugify_url(url)) + } + + fn fetchdata_path(&self, url: &Url) -> String { + format!("{}.toml", self.feed_path(url)) + } + + pub fn get_fetchdata(&self, url: &Url) -> FetchData { + let path = self.fetchdata_path(url); + if !fs::exists(path.clone()).unwrap() { + return FetchData::default(); + } + toml::from_str(&fs::read_to_string(path).unwrap()).unwrap() + } + + fn has_changed(&self, url: &Url, new_feed: &Feed) -> bool { + let Some(old_feed) = self.load_feed(url, false) else { + return true; + }; + + let mut old_iter = old_feed.entries.iter(); + for new in &new_feed.entries { + let Some(old) = old_iter.next() else { + return true; + }; + if old != new { + return true; + } + } + // ignoring any entries left in old_iter + false + } + + pub fn store(&self, url: &Url, mut response: Response) -> bool { + let headers = response.headers(); + let fetchdata = FetchData { + etag: hv(headers, "etag"), + date: hv(headers, "date"), + }; + + let body = response + .body_mut() + .with_config() + // .limit(MAX_BODY_SIZE) + .read_to_vec() + .unwrap(); + let feed = match feed_rs::parser::parse(body.as_slice()) { + Ok(f) => f, + Err(e) => { + warn!("Error when parsing feed for {url}: {e:?}"); + return false; + } + }; + if !self.has_changed(url, &feed) { + return false; + } + let _ = fs::write(self.feed_path(url), body); + let _ = fs::write( + self.fetchdata_path(url), + toml::to_string(&fetchdata).unwrap(), + ); + true + } + + fn load_feed(&self, url: &Url, sanitize: bool) -> Option { + let parser = feed_rs::parser::Builder::new() + .sanitize_content(sanitize) + .build(); + + let path = self.feed_path(url); + if !fs::exists(path.clone()).unwrap() { + return None; + } + let file = fs::File::open(path).unwrap(); + Some(parser.parse(BufReader::new(file)).unwrap()) + } + + pub fn collect(&self, feed_configs: &Vec) -> Vec { + let mut entries = vec![]; + + for feed_config in feed_configs { + let url = Url::parse(&feed_config.url).unwrap(); + let Some(mut feed) = self.load_feed(&url, true) else { + // todo error handling! + warn!("Problem parsing feed file for feed {}", feed_config.url); + continue; + }; + entries.append(&mut feed.entries); + // todo also trim mid-way when length > something, trading cpu for memory + } + trim_entries(entries) + } +} + +fn trim_entries(mut entries: Vec) -> Vec { + entries.sort_by_key(|e| std::cmp::Reverse(e.updated.or(e.published).unwrap_or_default())); + entries.truncate(10); + entries +} + +fn hv(headers: &HeaderMap, key: &str) -> String { + match headers.get(key) { + Some(hv) => hv.to_str().unwrap_or_default().to_string(), + _ => "".to_string(), + } +} diff --git a/web/planet-mars/src/fetcher.rs b/web/planet-mars/src/fetcher.rs new file mode 100644 index 000000000..43d08bb1a --- /dev/null +++ b/web/planet-mars/src/fetcher.rs @@ -0,0 +1,70 @@ +use std::time::Instant; +use ureq::tls::{TlsConfig, TlsProvider}; +use ureq::Agent; +use url::Url; + +use crate::FeedStore; + +pub struct Fetcher { + agent: Agent, + /// FROM header for requests + from: String, +} + +impl Fetcher { + pub fn new(bot_name: &str, from: &str) -> Fetcher { + // TODO Get URL from a better place, e.g. Cargo.toml? + let ua_name = format!("{bot_name}/{} https://TODO", env!("CARGO_PKG_VERSION")); + let agent = Agent::config_builder() + .http_status_as_error(false) + .user_agent(ua_name) + .tls_config( + TlsConfig::builder() + .provider(TlsProvider::NativeTls) + .build(), + ) + .build() + .into(); + Fetcher { + agent, + from: from.to_string(), + } + } + + pub fn fetch(&self, url: Url, feed_store: &FeedStore) -> bool { + let fetchdata = feed_store.get_fetchdata(&url); + let mut builder = self + .agent + .get(url.to_string()) + .header("FROM", self.from.clone()); + if fetchdata.etag != "" { + builder = builder.header("If-None-Match", fetchdata.etag); + } + if fetchdata.date != "" { + builder = builder.header("If-Modified-Since", fetchdata.date); + } + + let start_instant = Instant::now(); + let result = builder.call(); + let duration = start_instant.elapsed(); + + let response = result.unwrap(); // todo log and return false + debug!( + "fetched with status {} in {} ms: {url}", + response.status(), + duration.as_millis() + ); + let status = response.status(); + match status.as_u16() { + 304 => false, // Not Modified -> nothing to do + 200 => feed_store.store(&url, response), + _ => { + warn!( + "HTTP Status {} not implemented for {url}", + response.status() + ); + false + } + } + } +} diff --git a/web/planet-mars/src/main.rs b/web/planet-mars/src/main.rs new file mode 100644 index 000000000..5729cd730 --- /dev/null +++ b/web/planet-mars/src/main.rs @@ -0,0 +1,98 @@ +#[macro_use] +extern crate log; + +use crate::feed_store::FeedStore; +use crate::fetcher::Fetcher; +use clap::Parser; +use serde::Deserialize; +use std::fs; +use std::path::PathBuf; +use url::Url; + +mod feed_store; +mod fetcher; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Args { + #[arg( + short, + long, + default_value_t = String::from("mars.toml") + )] + config: String, +} + +#[derive(Deserialize)] +struct Config { + /// to be used as part of the fetchers username header + bot_name: String, + /// where to store downloaded feeds and their metadata + feed_dir: String, + /// feeds to be agregated + feeds: Vec, + /// Email adress to use for the from header when fetching feeds + from: String, + /// where to build the output files + out_dir: String, + /// templates folder + templates_dir: String, +} + +pub fn to_checked_pathbuf(dir: String) -> PathBuf { + let dir: PathBuf = PathBuf::from(dir); + + let m = dir + .metadata() + .unwrap_or_else(|_| panic!("Could not get metadata of dir: {}", dir.display())); + assert!(m.is_dir(), "Not a dir: {}", dir.display()); + dir +} + +#[derive(Deserialize)] +struct FeedConfig { + url: String, +} + +fn main() -> Result<(), Box> { + env_logger::init(); + info!("starting up"); + + let args = Args::parse(); + let config_path = &args.config; + if !fs::exists(config_path)? { + panic!("Configuration file {config_path} does not exist!"); + } + let config: Config = toml::from_str(&fs::read_to_string(config_path)?)?; + let templates_dir = to_checked_pathbuf(config.templates_dir); + let out_dir = to_checked_pathbuf(config.out_dir); + + let feed_store = FeedStore::new(config.feed_dir); + let fetcher = Fetcher::new(&config.bot_name, &config.from); + + let mut rebuild = false; + for feed in &config.feeds { + let url = Url::parse(&feed.url)?; + rebuild |= fetcher.fetch(url, &feed_store); + } + info!("Done fetching. Rebuild needed: {rebuild}"); + if rebuild { + let entries = feed_store.collect(&config.feeds); + let mut tera = match tera::Tera::new(&format!("{}/*", &templates_dir.display())) { + Ok(t) => t, + Err(e) => { + println!("Parsing error(s): {}", e); + ::std::process::exit(1); + } + }; + tera.autoescape_on(vec![]); + let mut context = tera::Context::new(); + context.insert("entries", &entries); + for name in tera.get_template_names() { + debug!("Processing template {name}"); + let file = fs::File::create(&format!("{}/{name}", out_dir.display()))?; + let _ = tera.render_to(name, &context, file)?; + } + } + Ok(()) +} diff --git a/web/planet-mars/templates/index.html b/web/planet-mars/templates/index.html new file mode 100644 index 000000000..5ecbc15f1 --- /dev/null +++ b/web/planet-mars/templates/index.html @@ -0,0 +1,8 @@ +hello world + +{% for entry in entries %} + {% for link in entry.links %} + {{link.href}} + {% endfor %} +{% endfor %} + From 3bee93ecb3ec4bff49758a276dba22123644aea9 Mon Sep 17 00:00:00 2001 From: Thomas Koch Date: Fri, 10 Jan 2025 20:17:06 +0200 Subject: [PATCH 02/17] generating html from 12 feeds works --- web/planet-mars/Cargo.lock | 7 ++ web/planet-mars/Cargo.toml | 1 + web/planet-mars/src/feed_store.rs | 18 +++++- web/planet-mars/src/fetcher.rs | 4 +- web/planet-mars/src/main.rs | 95 +++++++++++++++++++--------- web/planet-mars/src/simple_entry.rs | 43 +++++++++++++ web/planet-mars/templates/index.html | 38 +++++++++-- 7 files changed, 165 insertions(+), 41 deletions(-) create mode 100644 web/planet-mars/src/simple_entry.rs diff --git a/web/planet-mars/Cargo.lock b/web/planet-mars/Cargo.lock index 540750fbd..322b0bf09 100644 --- a/web/planet-mars/Cargo.lock +++ b/web/planet-mars/Cargo.lock @@ -96,6 +96,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "anyhow" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" + [[package]] name = "autocfg" version = "1.4.0" @@ -1093,6 +1099,7 @@ checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" name = "planet-mars" version = "0.1.0" dependencies = [ + "anyhow", "clap", "env_logger", "feed-rs", diff --git a/web/planet-mars/Cargo.toml b/web/planet-mars/Cargo.toml index 92a58ebee..ab6f16493 100644 --- a/web/planet-mars/Cargo.toml +++ b/web/planet-mars/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] # ammonia = "*" done by feed-rs +anyhow = "*" clap = { version = "*", features = ["derive"] } env_logger = "*" feed-rs = "*" diff --git a/web/planet-mars/src/feed_store.rs b/web/planet-mars/src/feed_store.rs index 6b162de8c..7b75e5164 100644 --- a/web/planet-mars/src/feed_store.rs +++ b/web/planet-mars/src/feed_store.rs @@ -1,6 +1,7 @@ use feed_rs::model::Entry; use feed_rs::model::Feed; use serde::{Deserialize, Serialize}; +use std::convert::AsRef; use std::fs; use std::io::BufReader; use std::path::PathBuf; @@ -20,7 +21,7 @@ pub struct FeedStore { } impl FeedStore { - pub fn new(dir: String) -> Self { + pub fn new(dir: &str) -> Self { Self { dir: super::to_checked_pathbuf(dir), } @@ -66,6 +67,16 @@ impl FeedStore { false } + fn write + std::fmt::Display, C: AsRef<[u8]>>( + path: P, + contents: C, + ) -> std::io::Result<()> { + if fs::exists(&path)? { + fs::rename(&path, format!("{path}.backup"))?; + } + fs::write(path, contents) + } + pub fn store(&self, url: &Url, mut response: Response) -> bool { let headers = response.headers(); let fetchdata = FetchData { @@ -89,8 +100,9 @@ impl FeedStore { if !self.has_changed(url, &feed) { return false; } - let _ = fs::write(self.feed_path(url), body); - let _ = fs::write( + debug!("Storing feed for {url}."); + let _ = Self::write(self.feed_path(url), body); + let _ = Self::write( self.fetchdata_path(url), toml::to_string(&fetchdata).unwrap(), ); diff --git a/web/planet-mars/src/fetcher.rs b/web/planet-mars/src/fetcher.rs index 43d08bb1a..13c326f8a 100644 --- a/web/planet-mars/src/fetcher.rs +++ b/web/planet-mars/src/fetcher.rs @@ -37,10 +37,10 @@ impl Fetcher { .agent .get(url.to_string()) .header("FROM", self.from.clone()); - if fetchdata.etag != "" { + if !fetchdata.etag.is_empty() { builder = builder.header("If-None-Match", fetchdata.etag); } - if fetchdata.date != "" { + if !fetchdata.date.is_empty() { builder = builder.header("If-Modified-Since", fetchdata.date); } diff --git a/web/planet-mars/src/main.rs b/web/planet-mars/src/main.rs index 5729cd730..f479cae01 100644 --- a/web/planet-mars/src/main.rs +++ b/web/planet-mars/src/main.rs @@ -3,14 +3,17 @@ extern crate log; use crate::feed_store::FeedStore; use crate::fetcher::Fetcher; +use anyhow::Result; use clap::Parser; use serde::Deserialize; +use simple_entry::SimpleEntry; use std::fs; use std::path::PathBuf; use url::Url; mod feed_store; mod fetcher; +mod simple_entry; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -21,6 +24,8 @@ struct Args { default_value_t = String::from("mars.toml") )] config: String, + #[arg(long, default_value_t = false)] + no_fetch: bool, } #[derive(Deserialize)] @@ -39,7 +44,7 @@ struct Config { templates_dir: String, } -pub fn to_checked_pathbuf(dir: String) -> PathBuf { +pub fn to_checked_pathbuf(dir: &str) -> PathBuf { let dir: PathBuf = PathBuf::from(dir); let m = dir @@ -54,7 +59,54 @@ struct FeedConfig { url: String, } -fn main() -> Result<(), Box> { +fn fetch(config: &Config, feed_store: &FeedStore) -> Result { + let fetcher = Fetcher::new(&config.bot_name, &config.from); + let mut rebuild = false; + for feed in &config.feeds { + let url = match Url::parse(&feed.url) { + Ok(x) => x, + Err(e) => { + error!("Error parsing url '{}': {e:?}", feed.url); + continue; + } + }; + rebuild |= fetcher.fetch(url, feed_store); + } + info!("Done fetching. Rebuild needed: {rebuild}"); + Ok(rebuild) +} + +fn build(config: &Config, feed_store: &FeedStore) -> Result<()> { + let templates_dir = to_checked_pathbuf(&config.templates_dir); + let out_dir = to_checked_pathbuf(&config.out_dir); + + let mut tera = match tera::Tera::new(&format!("{}/*", &templates_dir.display())) { + Ok(t) => t, + Err(e) => { + println!("Parsing error(s): {}", e); + ::std::process::exit(1); + } + }; + // disable autoescape as this would corrupt urls or the entriy contents. todo check this! + tera.autoescape_on(vec![]); + + let mut context = tera::Context::new(); + let entries: Vec = feed_store + .collect(&config.feeds) + .into_iter() + .map(SimpleEntry::from_feed_entry) + .collect(); + context.insert("entries", &entries); + + for name in tera.get_template_names() { + debug!("Processing template {name}"); + let file = fs::File::create(format!("{}/{name}", out_dir.display()))?; + tera.render_to(name, &context, file)?; + } + Ok(()) +} + +fn main() -> Result<()> { env_logger::init(); info!("starting up"); @@ -64,35 +116,20 @@ fn main() -> Result<(), Box> { panic!("Configuration file {config_path} does not exist!"); } let config: Config = toml::from_str(&fs::read_to_string(config_path)?)?; - let templates_dir = to_checked_pathbuf(config.templates_dir); - let out_dir = to_checked_pathbuf(config.out_dir); + // only check here to avoid fetching with broken config + // todo: get a config lib that provides validation! + let _ = to_checked_pathbuf(&config.templates_dir); + let _ = to_checked_pathbuf(&config.out_dir); - let feed_store = FeedStore::new(config.feed_dir); - let fetcher = Fetcher::new(&config.bot_name, &config.from); + let feed_store = FeedStore::new(&config.feed_dir); + let should_build = if args.no_fetch { + true + } else { + fetch(&config, &feed_store)? + }; - let mut rebuild = false; - for feed in &config.feeds { - let url = Url::parse(&feed.url)?; - rebuild |= fetcher.fetch(url, &feed_store); - } - info!("Done fetching. Rebuild needed: {rebuild}"); - if rebuild { - let entries = feed_store.collect(&config.feeds); - let mut tera = match tera::Tera::new(&format!("{}/*", &templates_dir.display())) { - Ok(t) => t, - Err(e) => { - println!("Parsing error(s): {}", e); - ::std::process::exit(1); - } - }; - tera.autoescape_on(vec![]); - let mut context = tera::Context::new(); - context.insert("entries", &entries); - for name in tera.get_template_names() { - debug!("Processing template {name}"); - let file = fs::File::create(&format!("{}/{name}", out_dir.display()))?; - let _ = tera.render_to(name, &context, file)?; - } + if should_build { + build(&config, &feed_store)?; } Ok(()) } diff --git a/web/planet-mars/src/simple_entry.rs b/web/planet-mars/src/simple_entry.rs new file mode 100644 index 000000000..4f531bfce --- /dev/null +++ b/web/planet-mars/src/simple_entry.rs @@ -0,0 +1,43 @@ +use feed_rs::model::Entry; + +/// Simplified Feed entry for easier value access in template +#[derive(serde::Serialize)] +pub struct SimpleEntry { + pub date: String, + pub content: String, + pub author: String, + pub link: String, + pub title: String, +} + +/// format for the entries timestamp +/// +const FMT: &str = "%c"; + +impl SimpleEntry { + pub fn from_feed_entry(entry: Entry) -> Self { + Self { + date: entry + .updated + .or(entry.published) + .unwrap_or_default() + .format(FMT) + .to_string(), + content: entry + .content + .map(|x| x.body.unwrap_or_default()) + .unwrap_or_default(), + author: if !entry.authors.is_empty() { + entry.authors[0].name.clone() + } else { + "".to_string() + }, + link: if !entry.links.is_empty() { + entry.links[0].href.clone() + } else { + "".to_string() + }, + title: entry.title.map(|x| x.content).unwrap_or_default(), + } + } +} diff --git a/web/planet-mars/templates/index.html b/web/planet-mars/templates/index.html index 5ecbc15f1..0c781cffd 100644 --- a/web/planet-mars/templates/index.html +++ b/web/planet-mars/templates/index.html @@ -1,8 +1,32 @@ -hello world - -{% for entry in entries %} - {% for link in entry.links %} - {{link.href}} - {% endfor %} -{% endfor %} + + + Planet TVL + + + + + {# todo #} + + + +
+

Planet TVL

+ {% for entry in entries %} + + {% endfor %} +
+ + + From 85b5c2be89e942d75191db58c51003c4a0e23210 Mon Sep 17 00:00:00 2001 From: Thomas Koch Date: Sat, 11 Jan 2025 13:38:20 +0200 Subject: [PATCH 03/17] Fix caching Nginx's std cfg only returns 304 with If-Unmodified-Since header, if the date is exactly the one it expects, not the date the client did its last request. This makes sense as it is much easier to compare two strings for equality than to parse the date received from the client and check the ordering with the server known last_modified value. --- web/planet-mars/src/feed_store.rs | 4 ++-- web/planet-mars/src/fetcher.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/web/planet-mars/src/feed_store.rs b/web/planet-mars/src/feed_store.rs index 7b75e5164..8b9cba3e5 100644 --- a/web/planet-mars/src/feed_store.rs +++ b/web/planet-mars/src/feed_store.rs @@ -13,7 +13,7 @@ use url::Url; #[derive(Deserialize, Serialize, Default)] pub struct FetchData { pub etag: String, - pub date: String, + pub last_modified: String, } pub struct FeedStore { @@ -81,7 +81,7 @@ impl FeedStore { let headers = response.headers(); let fetchdata = FetchData { etag: hv(headers, "etag"), - date: hv(headers, "date"), + last_modified: hv(headers, "last_modified"), }; let body = response diff --git a/web/planet-mars/src/fetcher.rs b/web/planet-mars/src/fetcher.rs index 13c326f8a..bb982698b 100644 --- a/web/planet-mars/src/fetcher.rs +++ b/web/planet-mars/src/fetcher.rs @@ -40,8 +40,8 @@ impl Fetcher { if !fetchdata.etag.is_empty() { builder = builder.header("If-None-Match", fetchdata.etag); } - if !fetchdata.date.is_empty() { - builder = builder.header("If-Modified-Since", fetchdata.date); + if !fetchdata.last_modified.is_empty() { + builder = builder.header("If-Modified-Since", fetchdata.last_modified); } let start_instant = Instant::now(); From 8b17e93d74014b2dc7c6865f37913ddb391c72a6 Mon Sep 17 00:00:00 2001 From: Thomas Koch Date: Sat, 11 Jan 2025 13:45:03 +0200 Subject: [PATCH 04/17] stuff --- web/planet-mars/Cargo.toml | 1 - web/planet-mars/src/feed_store.rs | 2 +- web/planet-mars/src/fetcher.rs | 2 +- web/planet-mars/templates/index.html | 40 +++++++++++++++------------- 4 files changed, 24 insertions(+), 21 deletions(-) diff --git a/web/planet-mars/Cargo.toml b/web/planet-mars/Cargo.toml index ab6f16493..5e7894a50 100644 --- a/web/planet-mars/Cargo.toml +++ b/web/planet-mars/Cargo.toml @@ -4,7 +4,6 @@ version = "0.1.0" edition = "2021" [dependencies] -# ammonia = "*" done by feed-rs anyhow = "*" clap = { version = "*", features = ["derive"] } env_logger = "*" diff --git a/web/planet-mars/src/feed_store.rs b/web/planet-mars/src/feed_store.rs index 8b9cba3e5..9e72e6cff 100644 --- a/web/planet-mars/src/feed_store.rs +++ b/web/planet-mars/src/feed_store.rs @@ -41,7 +41,7 @@ impl FeedStore { format!("{}.toml", self.feed_path(url)) } - pub fn get_fetchdata(&self, url: &Url) -> FetchData { + pub fn load_fetchdata(&self, url: &Url) -> FetchData { let path = self.fetchdata_path(url); if !fs::exists(path.clone()).unwrap() { return FetchData::default(); diff --git a/web/planet-mars/src/fetcher.rs b/web/planet-mars/src/fetcher.rs index bb982698b..8cc469a1c 100644 --- a/web/planet-mars/src/fetcher.rs +++ b/web/planet-mars/src/fetcher.rs @@ -32,7 +32,7 @@ impl Fetcher { } pub fn fetch(&self, url: Url, feed_store: &FeedStore) -> bool { - let fetchdata = feed_store.get_fetchdata(&url); + let fetchdata = feed_store.load_fetchdata(&url); let mut builder = self .agent .get(url.to_string()) diff --git a/web/planet-mars/templates/index.html b/web/planet-mars/templates/index.html index 0c781cffd..c82073e0e 100644 --- a/web/planet-mars/templates/index.html +++ b/web/planet-mars/templates/index.html @@ -9,24 +9,28 @@ -
-

Planet TVL

- {% for entry in entries %} - - {% endfor %} -
+
+

Planet TVL

+
+ From cfab9ef5c0dd384a5c663fe8b1a07e254bf25605 Mon Sep 17 00:00:00 2001 From: Thomas Koch Date: Sat, 11 Jan 2025 21:17:55 +0200 Subject: [PATCH 05/17] implemented also atom feed via template --- web/planet-mars/Cargo.lock | 28 +++++++++++- web/planet-mars/Cargo.toml | 3 +- web/planet-mars/planet.css | 59 ++++++++++++++++++++++++++ web/planet-mars/src/feed_store.rs | 52 ++++++++++++++--------- web/planet-mars/src/fetcher.rs | 11 ++--- web/planet-mars/src/main.rs | 38 ++--------------- web/planet-mars/src/simple_entry.rs | 43 ------------------- web/planet-mars/src/template_engine.rs | 35 +++++++++++++++ web/planet-mars/templates/atom.xml | 52 +++++++++++++++++++++++ web/planet-mars/templates/index.html | 52 +++++++++++++++++++---- 10 files changed, 259 insertions(+), 114 deletions(-) create mode 100644 web/planet-mars/planet.css delete mode 100644 web/planet-mars/src/simple_entry.rs create mode 100644 web/planet-mars/src/template_engine.rs create mode 100644 web/planet-mars/templates/atom.xml diff --git a/web/planet-mars/Cargo.lock b/web/planet-mars/Cargo.lock index 322b0bf09..744b4571b 100644 --- a/web/planet-mars/Cargo.lock +++ b/web/planet-mars/Cargo.lock @@ -108,6 +108,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -125,6 +131,9 @@ name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +dependencies = [ + "serde", +] [[package]] name = "block-buffer" @@ -1104,6 +1113,8 @@ dependencies = [ "env_logger", "feed-rs", "log", + "quick-xml", + "ron", "serde", "slug", "tera", @@ -1144,6 +1155,7 @@ checksum = "165859e9e55f79d67b96c5d96f4e88b6f2695a1972849c15a6a3f5c59fc2c003" dependencies = [ "encoding_rs", "memchr", + "serde", ] [[package]] @@ -1229,6 +1241,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ron" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" +dependencies = [ + "base64 0.21.7", + "bitflags", + "serde", + "serde_derive", +] + [[package]] name = "rustix" version = "0.38.43" @@ -1667,7 +1691,7 @@ version = "3.0.0-rc5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77f9bd1d03fe3bf87c90b115a0e09e22535cbcff3f8d0a6487524f006325f173" dependencies = [ - "base64", + "base64 0.22.1", "brotli-decompressor", "cc", "cookie_store", @@ -1695,7 +1719,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a27729fd2c15426f48992a911ce31bd1f29c1cd0898c2c86b410d24f51c99eb3" dependencies = [ - "base64", + "base64 0.22.1", "http", "httparse", "log", diff --git a/web/planet-mars/Cargo.toml b/web/planet-mars/Cargo.toml index 5e7894a50..848af9c6e 100644 --- a/web/planet-mars/Cargo.toml +++ b/web/planet-mars/Cargo.toml @@ -9,10 +9,11 @@ clap = { version = "*", features = ["derive"] } env_logger = "*" feed-rs = "*" log = "*" +ron = "*" # todo for development, to check atom-rs internal representation of feeds serde = { version = "*", features = ["derive"] } slug = "*" tera = "*" toml = "*" ureq = { version = "3.0.0-rc5", features = ["brotli", "charset", "gzip", "native-tls"]} url = "*" - +quick-xml = { version = "*", features = ["serialize"] } diff --git a/web/planet-mars/planet.css b/web/planet-mars/planet.css new file mode 100644 index 000000000..aca682fd3 --- /dev/null +++ b/web/planet-mars/planet.css @@ -0,0 +1,59 @@ +p, h1, h2, h3, h4, h5, h6, small { + max-width: 48em; +} + +h1 a, h2 a, h3 a, h4 a, h5 a, h6 a { + color: inherit !important; + text-decoration: none; +} + +ul, ol { + /* account for the 1em -webkit-margin-start for the list icon */ + max-width: 45em; +} + +ul,ol,dl, p { + line-height: 1.4; +} + +body { + margin-top: 1em; + margin-bottom: 1em; +} + +#maincontainer { + display: flex; + max-width: 80em; +} + +#maincontainer main { + max-width: 50em; +} + +#maincontainer main * { + max-width: 50em; +} + +#maincontainer aside { + margin-left: 5em; + max-width: 25em; +} + +article > h2.entry_header { + margin-bottom: 3px; +} + +.entry_meta { + border: 1px thin; + padding: 3px 0; + background-color: LightBlue; +} + +hr.entry_sep { + border: none; +} +hr.entry_sep::before { + content: '* * *'; + display: block; + text-align: center; +} diff --git a/web/planet-mars/src/feed_store.rs b/web/planet-mars/src/feed_store.rs index 9e72e6cff..7b8642604 100644 --- a/web/planet-mars/src/feed_store.rs +++ b/web/planet-mars/src/feed_store.rs @@ -1,5 +1,7 @@ +use anyhow::Result; use feed_rs::model::Entry; use feed_rs::model::Feed; +use ron::ser::{to_string_pretty, PrettyConfig}; use serde::{Deserialize, Serialize}; use std::convert::AsRef; use std::fs; @@ -28,43 +30,47 @@ impl FeedStore { } fn slugify_url(url: &Url) -> String { - let domain = url.domain().unwrap(); + let domain = url.domain().unwrap(); // todo don't hide error let query = url.query().unwrap_or(""); slug::slugify(format!("{domain}{}{query}", url.path())) } + fn generic_path(&self, url: &Url, ext: &str) -> String { + format!("{}/{}{ext}", self.dir.display(), Self::slugify_url(url)) + } + fn feed_path(&self, url: &Url) -> String { - format!("{}/{}", self.dir.display(), Self::slugify_url(url)) + self.generic_path(url, "") } fn fetchdata_path(&self, url: &Url) -> String { - format!("{}.toml", self.feed_path(url)) + self.generic_path(url, ".toml") } - pub fn load_fetchdata(&self, url: &Url) -> FetchData { + pub fn load_fetchdata(&self, url: &Url) -> Result { let path = self.fetchdata_path(url); - if !fs::exists(path.clone()).unwrap() { - return FetchData::default(); + if !fs::exists(path.clone())? { + return Ok(FetchData::default()); } - toml::from_str(&fs::read_to_string(path).unwrap()).unwrap() + Ok(toml::from_str(&fs::read_to_string(path)?)?) } - fn has_changed(&self, url: &Url, new_feed: &Feed) -> bool { + fn has_changed(&self, url: &Url, new_feed: &Feed) -> Result { let Some(old_feed) = self.load_feed(url, false) else { - return true; + return Ok(true); }; let mut old_iter = old_feed.entries.iter(); for new in &new_feed.entries { let Some(old) = old_iter.next() else { - return true; + return Ok(true); }; if old != new { - return true; + return Ok(true); } } // ignoring any entries left in old_iter - false + Ok(false) } fn write + std::fmt::Display, C: AsRef<[u8]>>( @@ -77,7 +83,7 @@ impl FeedStore { fs::write(path, contents) } - pub fn store(&self, url: &Url, mut response: Response) -> bool { + pub fn store(&self, url: &Url, mut response: Response) -> Result { let headers = response.headers(); let fetchdata = FetchData { etag: hv(headers, "etag"), @@ -94,19 +100,24 @@ impl FeedStore { Ok(f) => f, Err(e) => { warn!("Error when parsing feed for {url}: {e:?}"); - return false; + return Ok(false); } }; - if !self.has_changed(url, &feed) { - return false; + if !self.has_changed(url, &feed)? { + return Ok(false); } debug!("Storing feed for {url}."); - let _ = Self::write(self.feed_path(url), body); - let _ = Self::write( + // todo don't serialize to string but to writer + Self::write( + self.generic_path(url, ".ron"), + to_string_pretty(&feed, PrettyConfig::default())?, + )?; + Self::write(self.feed_path(url), body)?; + Self::write( self.fetchdata_path(url), toml::to_string(&fetchdata).unwrap(), - ); - true + )?; + Ok(true) } fn load_feed(&self, url: &Url, sanitize: bool) -> Option { @@ -132,6 +143,7 @@ impl FeedStore { warn!("Problem parsing feed file for feed {}", feed_config.url); continue; }; + entries.append(&mut feed.entries); // todo also trim mid-way when length > something, trading cpu for memory } diff --git a/web/planet-mars/src/fetcher.rs b/web/planet-mars/src/fetcher.rs index 8cc469a1c..289bbe13a 100644 --- a/web/planet-mars/src/fetcher.rs +++ b/web/planet-mars/src/fetcher.rs @@ -1,3 +1,4 @@ +use anyhow::Result; use std::time::Instant; use ureq::tls::{TlsConfig, TlsProvider}; use ureq::Agent; @@ -31,8 +32,8 @@ impl Fetcher { } } - pub fn fetch(&self, url: Url, feed_store: &FeedStore) -> bool { - let fetchdata = feed_store.load_fetchdata(&url); + pub fn fetch(&self, url: Url, feed_store: &FeedStore) -> Result { + let fetchdata = feed_store.load_fetchdata(&url)?; let mut builder = self .agent .get(url.to_string()) @@ -48,7 +49,7 @@ impl Fetcher { let result = builder.call(); let duration = start_instant.elapsed(); - let response = result.unwrap(); // todo log and return false + let response = result?; // todo log and return false debug!( "fetched with status {} in {} ms: {url}", response.status(), @@ -56,14 +57,14 @@ impl Fetcher { ); let status = response.status(); match status.as_u16() { - 304 => false, // Not Modified -> nothing to do + 304 => Ok(false), // Not Modified -> nothing to do 200 => feed_store.store(&url, response), _ => { warn!( "HTTP Status {} not implemented for {url}", response.status() ); - false + Ok(false) } } } diff --git a/web/planet-mars/src/main.rs b/web/planet-mars/src/main.rs index f479cae01..296cf9e6d 100644 --- a/web/planet-mars/src/main.rs +++ b/web/planet-mars/src/main.rs @@ -6,14 +6,14 @@ use crate::fetcher::Fetcher; use anyhow::Result; use clap::Parser; use serde::Deserialize; -use simple_entry::SimpleEntry; use std::fs; use std::path::PathBuf; use url::Url; +//mod atom_serializer; mod feed_store; mod fetcher; -mod simple_entry; +mod template_engine; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -70,42 +70,12 @@ fn fetch(config: &Config, feed_store: &FeedStore) -> Result { continue; } }; - rebuild |= fetcher.fetch(url, feed_store); + rebuild |= fetcher.fetch(url, feed_store)?; } info!("Done fetching. Rebuild needed: {rebuild}"); Ok(rebuild) } -fn build(config: &Config, feed_store: &FeedStore) -> Result<()> { - let templates_dir = to_checked_pathbuf(&config.templates_dir); - let out_dir = to_checked_pathbuf(&config.out_dir); - - let mut tera = match tera::Tera::new(&format!("{}/*", &templates_dir.display())) { - Ok(t) => t, - Err(e) => { - println!("Parsing error(s): {}", e); - ::std::process::exit(1); - } - }; - // disable autoescape as this would corrupt urls or the entriy contents. todo check this! - tera.autoescape_on(vec![]); - - let mut context = tera::Context::new(); - let entries: Vec = feed_store - .collect(&config.feeds) - .into_iter() - .map(SimpleEntry::from_feed_entry) - .collect(); - context.insert("entries", &entries); - - for name in tera.get_template_names() { - debug!("Processing template {name}"); - let file = fs::File::create(format!("{}/{name}", out_dir.display()))?; - tera.render_to(name, &context, file)?; - } - Ok(()) -} - fn main() -> Result<()> { env_logger::init(); info!("starting up"); @@ -129,7 +99,7 @@ fn main() -> Result<()> { }; if should_build { - build(&config, &feed_store)?; + template_engine::build(&config, &feed_store)?; } Ok(()) } diff --git a/web/planet-mars/src/simple_entry.rs b/web/planet-mars/src/simple_entry.rs deleted file mode 100644 index 4f531bfce..000000000 --- a/web/planet-mars/src/simple_entry.rs +++ /dev/null @@ -1,43 +0,0 @@ -use feed_rs::model::Entry; - -/// Simplified Feed entry for easier value access in template -#[derive(serde::Serialize)] -pub struct SimpleEntry { - pub date: String, - pub content: String, - pub author: String, - pub link: String, - pub title: String, -} - -/// format for the entries timestamp -/// -const FMT: &str = "%c"; - -impl SimpleEntry { - pub fn from_feed_entry(entry: Entry) -> Self { - Self { - date: entry - .updated - .or(entry.published) - .unwrap_or_default() - .format(FMT) - .to_string(), - content: entry - .content - .map(|x| x.body.unwrap_or_default()) - .unwrap_or_default(), - author: if !entry.authors.is_empty() { - entry.authors[0].name.clone() - } else { - "".to_string() - }, - link: if !entry.links.is_empty() { - entry.links[0].href.clone() - } else { - "".to_string() - }, - title: entry.title.map(|x| x.content).unwrap_or_default(), - } - } -} diff --git a/web/planet-mars/src/template_engine.rs b/web/planet-mars/src/template_engine.rs new file mode 100644 index 000000000..b1d9bd8d9 --- /dev/null +++ b/web/planet-mars/src/template_engine.rs @@ -0,0 +1,35 @@ +use crate::feed_store::FeedStore; +use crate::to_checked_pathbuf; +use crate::Config; +use anyhow::Result; +use feed_rs::model::Entry; +use std::fs::File; +use tera::Tera; + +pub fn build(config: &Config, feed_store: &FeedStore) -> Result<()> { + let tera = create_tera(&config.templates_dir)?; + let out_dir = to_checked_pathbuf(&config.out_dir); + + let mut context = tera::Context::new(); + let feed_entries: Vec = feed_store.collect(&config.feeds); + context.insert("entries", &feed_entries); + context.insert("PKG_AUTHORS", env!("CARGO_PKG_AUTHORS")); + context.insert("PKG_HOMEPAGE", env!("CARGO_PKG_HOMEPAGE")); + context.insert("PKG_NAME", env!("CARGO_PKG_NAME")); + context.insert("PKG_VERSION", env!("CARGO_PKG_VERSION")); + + for name in tera.get_template_names() { + debug!("Processing template {name}"); + let file = File::create(format!("{}/{name}", out_dir.display()))?; + tera.render_to(name, &context, file)?; + } + Ok(()) +} + +fn create_tera(templates_dir: &str) -> Result { + let dir = to_checked_pathbuf(templates_dir); + let mut tera = tera::Tera::new(&format!("{}/*", &dir.display()))?; + // disable autoescape as this would corrupt urls or the entriy contents. todo check this! + tera.autoescape_on(vec![]); + Ok(tera) +} diff --git a/web/planet-mars/templates/atom.xml b/web/planet-mars/templates/atom.xml new file mode 100644 index 000000000..bc11aa917 --- /dev/null +++ b/web/planet-mars/templates/atom.xml @@ -0,0 +1,52 @@ + + + Planet TVL + + {{now()|date(format="%Y-%m-%dT%H:%M:%SZ")}} + https::/planet.tvl.fyi + + {{ PKG_NAME }} by {{ PKG_AUTHORS }} + + https://planet.tvl.fyi/logo.svg + + {% for entry in entries %} + + {{ entry.id }}/planet.tvl.fyi + {% if entry.title -%} + {{ entry.title.content }} + {% endif -%} + {% for link in entry.links %} + + {% endfor %} + {% if entry.updated %} + {{ entry.updated }} + {% endif %} + {% if entry.published %} + {{ entry.published }} + {% endif %} + {% if entry.summary -%} + + {{ entry.summary.content|escape }} + + {% endif -%} + {% for author in entry.authors %} + + {% if author.name -%} + {{ author.name }} + {% endif -%} + {% if author.email -%} + {{ author.email }} + {% endif -%} + + {% if author.email -%} + {{ author.uri }} + {% endif -%} + {% endfor %} + {% if entry.content -%} + + {{ entry.content.body|escape }} + + {% endif -%} + + {% endfor %} + diff --git a/web/planet-mars/templates/index.html b/web/planet-mars/templates/index.html index c82073e0e..0b5c1f123 100644 --- a/web/planet-mars/templates/index.html +++ b/web/planet-mars/templates/index.html @@ -1,3 +1,5 @@ +{% set dateformat = "%d.%m.%Y %H:%M" -%} + Planet TVL @@ -5,7 +7,6 @@ - {# todo #} @@ -14,17 +15,50 @@
- {% for entry in entries %} + {% for entry in entries -%} + {% if loop.index > 1 -%} +
+ {% endif -%} + {% if entry.links.0 -%} + {% set link = entry.links.0.href -%} + {% else -%} + {% set link = "" -%} + {% endif -%}
-

{{ entry.title|striptags }}

- {% if entry.published %}{{ entry.published | date(format="%Y-%m-%d %H:%M", timezone="Europe/Moscow") }}{% endif %}{# todo: maybe group posts by day? #} -

{{ entry.author|striptags }}

-
- {{ entry.content }} +

+ + {% if entry.title -%} + {{ entry.title.content|striptags }} + {% else -%} + NO TITLE + {% endif -%} + +

+
+ + {% if entry.updated -%} + {{ entry.updated | date(format=dateformat) }} + {% else -%} + {{ entry.published | date(format=dateformat) }} + {% endif -%} + + {% if entry.authors -%} + — {{ entry.authors.0.name | striptags }} + {% endif -%}
-

full post

+ + {% if entry.summary -%} +
+ {{ entry.summary.content }} +
+ {% endif -%} + {% if entry.content -%} +
+ {{ entry.content.body }} +
+ {% endif -%}
- {% endfor %} + {% endfor -%}
From 4a9f5201d1360d5a2369224317b26c288b438236 Mon Sep 17 00:00:00 2001 From: Thomas Koch Date: Sun, 12 Jan 2025 13:02:45 +0200 Subject: [PATCH 10/17] add some docs --- web/planet-mars/README.md | 17 +++++++++++++++++ web/planet-mars/README.org | 15 --------------- web/planet-mars/src/feed_store.rs | 1 + web/planet-mars/src/main.rs | 29 +++++++++++++++++++++++++++++ 4 files changed, 47 insertions(+), 15 deletions(-) create mode 100644 web/planet-mars/README.md delete mode 100644 web/planet-mars/README.org diff --git a/web/planet-mars/README.md b/web/planet-mars/README.md new file mode 100644 index 000000000..664fb790d --- /dev/null +++ b/web/planet-mars/README.md @@ -0,0 +1,17 @@ +Simple planet like planet venus but in rust and maintained. + +Please see the rustdoc of main.rs for further information. + +## todo + +* use a nice lib to process the config file + * should check whether dirs exists and are writeable + * should check whether feed urls can be parsed + +## Credits + +While writing this, I read and also copied code from: + +* [agro](https://docs.rs/crate/agro/0.1.1) +* [hades](https://github.com/kitallis/hades) +* [planetrs](https://github.com/djc/planetrs) diff --git a/web/planet-mars/README.org b/web/planet-mars/README.org deleted file mode 100644 index 6daa5729b..000000000 --- a/web/planet-mars/README.org +++ /dev/null @@ -1,15 +0,0 @@ -Simple planet like planet venus but in rust and maintained. - -** todo -Also see todos in the source files - -*** use a nice lib to process the config file -- should check whether dirs exists and are writeable -- should check whether feed urls can be parsed -** Credits - -While writing this, I read and also copied code from: - -- [[https://docs.rs/crate/agro/0.1.1][agro]] -- [[https://github.com/kitallis/hades][haded]] by Akshay Gupta -- [[https://github.com/djc/planetrs][planetrs]] by Vagdish/Adau, Dirkjan Ochtman, Josh Matthews diff --git a/web/planet-mars/src/feed_store.rs b/web/planet-mars/src/feed_store.rs index f64a353a2..621b06d86 100644 --- a/web/planet-mars/src/feed_store.rs +++ b/web/planet-mars/src/feed_store.rs @@ -13,6 +13,7 @@ use ureq::http::Response; use ureq::Body; use url::Url; +/// How many feed entries should be included in the planet const ENTRIES_LEN: usize = 10; #[derive(Deserialize, Serialize, Default)] diff --git a/web/planet-mars/src/main.rs b/web/planet-mars/src/main.rs index 296cf9e6d..ccbb0647f 100644 --- a/web/planet-mars/src/main.rs +++ b/web/planet-mars/src/main.rs @@ -1,3 +1,25 @@ +//! Planet software to aggregate many feeds into one +//! +//! Input feeds are defined in a toml config file given as cmdline +//! argument. See the [`Config`] struct and the mars.toml.example file. +//! +//! The program iterates over all [feed urls], fetches them, stores them in +//! [feed_dir] and only rebuilds when at least one feed has updates. The +//! fetcher implements HTTP ETag and LastModified caching. +//! +//! During rebuild, all files in [templates_dir] are processed and written to +//! [out_dir]. +//! +//! The software is supposed to be run like every 15 minutes. +//! +//! Use a reserved (sub)domain to publish the planet! Although this software +//! tries to sanitize input feeds, there could still be bugs that open the +//! planets domain to cross-site attacks. +//! +//! [templates_dir]: Config#structfield.templates_dir +//! [feed_dir]: Config#structfield.feed_dir +//! [out_dir]: Config#structfield.out_dir +//! [feed urls]: Config#structfield.feeds #[macro_use] extern crate log; @@ -18,6 +40,7 @@ mod template_engine; #[derive(Parser)] #[command(author, version, about, long_about = None)] struct Args { + /// config file in toml format #[arg( short, long, @@ -28,6 +51,7 @@ struct Args { no_fetch: bool, } +/// Config to be parsed from toml file given as cmdline option #[derive(Deserialize)] struct Config { /// to be used as part of the fetchers username header @@ -54,8 +78,13 @@ pub fn to_checked_pathbuf(dir: &str) -> PathBuf { dir } +/// Config for one individual input feed +/// +/// This is a separate struct in case one wants to configure additional +/// information in the future. #[derive(Deserialize)] struct FeedConfig { + /// url of an ATOM, RSS or Json feed url: String, } From 150314936c46fc438aebd74106f6f2a6a0f30b30 Mon Sep 17 00:00:00 2001 From: Thomas Koch Date: Sun, 12 Jan 2025 20:12:27 +0200 Subject: [PATCH 11/17] get authors also from feed data + some css --- web/planet-mars/planet.css | 69 ++++++++++++++++++++------ web/planet-mars/src/feed_store.rs | 27 ++++++---- web/planet-mars/src/main.rs | 2 + web/planet-mars/src/template_engine.rs | 61 +++++++++++++++++++++-- web/planet-mars/templates/index.html | 20 ++++---- 5 files changed, 140 insertions(+), 39 deletions(-) diff --git a/web/planet-mars/planet.css b/web/planet-mars/planet.css index aca682fd3..31c5ca5fa 100644 --- a/web/planet-mars/planet.css +++ b/web/planet-mars/planet.css @@ -1,5 +1,5 @@ p, h1, h2, h3, h4, h5, h6, small { - max-width: 48em; + max-width: 48em; } h1 a, h2 a, h3 a, h4 a, h5 a, h6 a { @@ -9,34 +9,71 @@ h1 a, h2 a, h3 a, h4 a, h5 a, h6 a { ul, ol { /* account for the 1em -webkit-margin-start for the list icon */ - max-width: 45em; + max-width: 45em; } ul,ol,dl, p { - line-height: 1.4; + margin-top: 0.3em; + margin-bottom: 0.3em; + line-height: 1.2; } -body { - margin-top: 1em; - margin-bottom: 1em; +ul, ol { + padding-inline-start: 1.5em; } +#bodydiv { + margin: auto; + max-width: 80em; +} + +#maincontainer aside img { + max-width: 10em; +} + +#maincontainer main blockquote { + margin-left: 0; + margin-right: 10px; + box-shadow: 10px 0px 0px 0px #C4C4C4; +} + +blockquote, pre code { + padding: 0.5ex 0; + display: block; + background-color: #EEE; +} + +#maincontainer main * { + max-width: 100%; +} + +#maincontainer main pre { + overflow-x: auto; +} + +.entry_meta { + margin-bottom: 1em; +} + +@media only screen and (min-width: 1024px) { #maincontainer { display: flex; - max-width: 80em; } #maincontainer main { max-width: 50em; -} - -#maincontainer main * { - max-width: 50em; + flex: 5; } #maincontainer aside { margin-left: 5em; - max-width: 25em; + max-width: 15em; + flex: 1; +} + +#maincontainer aside img { + margin: auto; + display: block; } article > h2.entry_header { @@ -44,7 +81,6 @@ article > h2.entry_header { } .entry_meta { - border: 1px thin; padding: 3px 0; background-color: LightBlue; } @@ -53,7 +89,8 @@ hr.entry_sep { border: none; } hr.entry_sep::before { - content: '* * *'; - display: block; - text-align: center; + content: '* * *'; + display: block; + text-align: center; +} } diff --git a/web/planet-mars/src/feed_store.rs b/web/planet-mars/src/feed_store.rs index 621b06d86..ac0572307 100644 --- a/web/planet-mars/src/feed_store.rs +++ b/web/planet-mars/src/feed_store.rs @@ -4,6 +4,7 @@ use feed_rs::model::Entry; use feed_rs::model::Feed; use ron::ser::{to_string_pretty, PrettyConfig}; use serde::{Deserialize, Serialize}; +use std::collections::HashMap; use std::convert::AsRef; use std::fs; use std::io::BufReader; @@ -13,9 +14,6 @@ use ureq::http::Response; use ureq::Body; use url::Url; -/// How many feed entries should be included in the planet -const ENTRIES_LEN: usize = 10; - #[derive(Deserialize, Serialize, Default)] pub struct FetchData { pub etag: String, @@ -135,8 +133,12 @@ impl FeedStore { Ok(Some(parser.parse(BufReader::new(file))?)) } - pub fn collect(&self, feed_configs: &Vec) -> (Vec, Vec) { - let mut feeds = Vec::new(); + pub fn collect( + &self, + feed_configs: &Vec, + max_entries: usize, + ) -> (HashMap, Vec) { + let mut feeds = HashMap::new(); let mut entries = Vec::new(); for feed_config in feed_configs { @@ -154,21 +156,24 @@ impl FeedStore { Ok(None) => continue, Ok(Some(f)) => f, }; + for entry in &mut feed.entries { + entry.source = Some(feed_config.url.clone()); + } entries.append(&mut std::mem::take(&mut feed.entries)); - feeds.push(feed); + feeds.insert(feed_config.url.clone(), feed); // optimization to reduce memory usage - if entries.len() > 4 * ENTRIES_LEN { - entries = trim_entries(entries); + if entries.len() > 4 * max_entries { + entries = trim_entries(entries, max_entries); } } - (feeds, trim_entries(entries)) + (feeds, trim_entries(entries, max_entries)) } } -fn trim_entries(mut entries: Vec) -> Vec { +fn trim_entries(mut entries: Vec, max_entries: usize) -> Vec { entries.sort_by_key(|e| std::cmp::Reverse(e.updated.or(e.published).unwrap_or_default())); - entries.truncate(ENTRIES_LEN); + entries.truncate(max_entries); entries } diff --git a/web/planet-mars/src/main.rs b/web/planet-mars/src/main.rs index ccbb0647f..42ce8c35f 100644 --- a/web/planet-mars/src/main.rs +++ b/web/planet-mars/src/main.rs @@ -66,6 +66,8 @@ struct Config { out_dir: String, /// templates folder templates_dir: String, + /// How many feed entries should be included in the planet + max_entries: usize, } pub fn to_checked_pathbuf(dir: &str) -> PathBuf { diff --git a/web/planet-mars/src/template_engine.rs b/web/planet-mars/src/template_engine.rs index 827ac23ff..aecead991 100644 --- a/web/planet-mars/src/template_engine.rs +++ b/web/planet-mars/src/template_engine.rs @@ -2,21 +2,25 @@ use crate::feed_store::FeedStore; use crate::to_checked_pathbuf; use crate::Config; use anyhow::Result; +use feed_rs::model::Feed; +use std::collections::HashMap; use std::fs::File; -use tera::Tera; +use tera::{from_value, Tera}; pub fn build(config: &Config, feed_store: &FeedStore) -> Result<()> { - let tera = create_tera(&config.templates_dir)?; + let mut tera = create_tera(&config.templates_dir)?; let out_dir = to_checked_pathbuf(&config.out_dir); let mut context = tera::Context::new(); - let (feeds, entries) = feed_store.collect(&config.feeds); + let (feeds, entries): (HashMap, _) = + feed_store.collect(&config.feeds, config.max_entries); context.insert("feeds", &feeds); context.insert("entries", &entries); context.insert("PKG_AUTHORS", env!("CARGO_PKG_AUTHORS")); context.insert("PKG_HOMEPAGE", env!("CARGO_PKG_HOMEPAGE")); context.insert("PKG_NAME", env!("CARGO_PKG_NAME")); context.insert("PKG_VERSION", env!("CARGO_PKG_VERSION")); + tera.register_function("get_author", GetAuthorFunction { feeds }); for name in tera.get_template_names() { debug!("Processing template {name}"); @@ -33,3 +37,54 @@ fn create_tera(templates_dir: &str) -> Result { tera.autoescape_on(vec![]); Ok(tera) } + +struct GetAuthorFunction { + feeds: HashMap, +} + +impl tera::Function for GetAuthorFunction { + fn call(&self, args: &HashMap) -> Result { + let entry_val: tera::Map<_, _> = match args.get("entry") { + None => { + return Err(tera::Error::msg( + "No argument of name 'entry' given to function.", + )) + } + Some(val) => from_value(val.clone())?, + }; + + let feed_url: String = from_value(entry_val.get("source").unwrap().clone())?; + let authors_val: Vec> = + from_value(entry_val.get("authors").unwrap().clone())?; + + let mut authors: Vec = Vec::new(); + for author_val in authors_val { + let name: String = from_value(author_val.get("name").unwrap().clone())?; + if is_valid_name(&name) { + authors.push(name.clone()); + } + } + + if authors.is_empty() { + authors.append(&mut self.find_authors_from_feed(&feed_url)); + } + Ok(tera::Value::String(authors.join(", "))) + } +} + +impl GetAuthorFunction { + fn find_authors_from_feed(&self, feed_url: &str) -> Vec { + let feed = self.feeds.get(feed_url).unwrap(); + + feed.authors + .clone() + .into_iter() + .map(|x| x.name) + .filter(is_valid_name) + .collect() + } +} + +fn is_valid_name(n: &String) -> bool { + !n.is_empty() && n != "unknown" && n != "author" +} diff --git a/web/planet-mars/templates/index.html b/web/planet-mars/templates/index.html index 767bfa370..d72f10f53 100644 --- a/web/planet-mars/templates/index.html +++ b/web/planet-mars/templates/index.html @@ -1,5 +1,4 @@ {% set dateformat = "%d.%m.%Y %H:%M" -%} - Planet TVL @@ -10,6 +9,7 @@ +

Planet TVL

@@ -42,20 +42,20 @@ {{ entry.published | date(format=dateformat) }} {% endif -%} - {% if entry.authors -%} - — {{ entry.authors.0.name | striptags }} + {% set author = get_author(entry=entry) -%} + {% if author -%} + — {{ author | striptags }} {% endif -%}
- {% if entry.summary -%} -
- {{ entry.summary.content }} -
- {% endif -%} {% if entry.content -%}
{{ entry.content.body }}
+ {% elif entry.summary -%} +
+ {{ entry.summary.content }} +
{% endif -%} {% endfor -%} @@ -65,7 +65,7 @@

Last updated: {{now()|date(format="%Y-%m-%d %H:%M")}}

+ From c8c0f691ab88ca8a8a48c925e63fc0bbdcc07bd0 Mon Sep 17 00:00:00 2001 From: Thomas Koch Date: Sun, 12 Jan 2025 20:49:38 +0200 Subject: [PATCH 12/17] fix typo in atom template, link element --- web/planet-mars/templates/atom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/planet-mars/templates/atom.xml b/web/planet-mars/templates/atom.xml index 3a9c4572e..02dce53f1 100644 --- a/web/planet-mars/templates/atom.xml +++ b/web/planet-mars/templates/atom.xml @@ -1,7 +1,7 @@ Planet TVL - + {{now()|date(format="%Y-%m-%dT%H:%M:%SZ")}} https::/planet.tvl.fyi From ef8c39ed859ad0ea8ddaf00ffbd108844f077880 Mon Sep 17 00:00:00 2001 From: Thomas Koch Date: Sun, 12 Jan 2025 21:14:20 +0200 Subject: [PATCH 13/17] fix double https://https:// in useragent --- web/planet-mars/src/fetcher.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/planet-mars/src/fetcher.rs b/web/planet-mars/src/fetcher.rs index 8c7e0bcaf..0cf7f260f 100644 --- a/web/planet-mars/src/fetcher.rs +++ b/web/planet-mars/src/fetcher.rs @@ -15,7 +15,7 @@ pub struct Fetcher { impl Fetcher { pub fn new(bot_name: &str, from: &str) -> Fetcher { let ua_name = format!( - "{bot_name}/{} https://{} software: {}", + "{bot_name}/{} {} software: {}", env!("CARGO_PKG_VERSION"), env!("CARGO_PKG_HOMEPAGE"), env!("CARGO_PKG_NAME") From ac805654bd76543d3a3cd3c7aeb9519901f4ba21 Mon Sep 17 00:00:00 2001 From: Thomas Koch Date: Sun, 12 Jan 2025 21:14:37 +0200 Subject: [PATCH 14/17] set explicit versions for deps and update --- web/planet-mars/Cargo.lock | 42 ++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/web/planet-mars/Cargo.lock b/web/planet-mars/Cargo.lock index 744b4571b..f36f7ec4d 100644 --- a/web/planet-mars/Cargo.lock +++ b/web/planet-mars/Cargo.lock @@ -128,9 +128,9 @@ checksum = "8c3c1a368f70d6cf7302d78f8f7093da241fb8e8807c05cc9e51a125895a6d5b" [[package]] name = "bitflags" -version = "2.6.0" +version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +checksum = "1be3f42a67d6d345ecd59f675f3f012d6974981560836e938c22b424b85ce1be" dependencies = [ "serde", ] @@ -184,9 +184,9 @@ checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cc" -version = "1.2.7" +version = "1.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a012a0df96dd6d06ba9a1b29d6402d1a5d77c6befd2566afdc26e10603dc93d7" +checksum = "c8293772165d9345bdaaa39b45b2109591e63fe5e6fbc23c6ff930a048aa310b" dependencies = [ "shlex", ] @@ -236,9 +236,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.25" +version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b95dca1b68188a08ca6af9d96a6576150f598824bdb528c1190460c2940a0b48" +checksum = "a8eb5e908ef3a6efbe1ed62520fb7287959888c88485abe072543190ecc66783" dependencies = [ "clap_builder", "clap_derive", @@ -246,9 +246,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.25" +version = "4.5.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab52925392148efd3f7562f2136a81ffb778076bcc85727c6e020d6dd57cf15" +checksum = "96b01801b5fc6a0a232407abc821660c9c6d25a1cafc0d4f85f29fb8d9afc121" dependencies = [ "anstream", "anstyle", @@ -1113,7 +1113,6 @@ dependencies = [ "env_logger", "feed-rs", "log", - "quick-xml", "ron", "serde", "slug", @@ -1140,9 +1139,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.92" +version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37d3544b3f2748c54e147655edb5025752e2303145b5aefb3c3ea2c78b973bb0" +checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] @@ -1155,7 +1154,6 @@ checksum = "165859e9e55f79d67b96c5d96f4e88b6f2695a1972849c15a6a3f5c59fc2c003" dependencies = [ "encoding_rs", "memchr", - "serde", ] [[package]] @@ -1268,9 +1266,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.20" +version = "0.23.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" +checksum = "8f287924602bf649d949c63dc8ac8b235fa5387d394020705b80c4eb597ce5b8" dependencies = [ "log", "once_cell", @@ -1460,9 +1458,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.95" +version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46f71c0377baf4ef1cc3e3402ded576dccc315800fbc62dfc7fe04b009773b4a" +checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", @@ -1518,18 +1516,18 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.10" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3ac7f54ca534db81081ef1c1e7f6ea8a3ef428d2fc069097c079443d24124d3" +checksum = "d452f284b73e6d76dd36758a0c8684b1d5be31f92b89d07fd5822175732206fc" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.10" +version = "2.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e9465d30713b56a37ede7185763c3492a91be2f5fa68d958c44e41ab9248beb" +checksum = "26afc1baea8a989337eeb52b6e72a039780ce45c3edfcc9c5b9d112feeb173c2" dependencies = [ "proc-macro2", "quote", @@ -1973,9 +1971,9 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "0.6.22" +version = "0.6.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39281189af81c07ec09db316b302a3e67bf9bd7cbf6c820b50e35fee9c2fa980" +checksum = "c8d71a593cc5c42ad7876e2c1fda56f314f3754c084128833e64f1345ff8a03a" dependencies = [ "memchr", ] From 08160c315bac08c0e75fb87676e693d889e0a736 Mon Sep 17 00:00:00 2001 From: Thomas Koch Date: Sun, 12 Jan 2025 21:15:26 +0200 Subject: [PATCH 15/17] fix previous commit :-( --- web/planet-mars/Cargo.toml | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/web/planet-mars/Cargo.toml b/web/planet-mars/Cargo.toml index 830bce60f..b5e00a006 100644 --- a/web/planet-mars/Cargo.toml +++ b/web/planet-mars/Cargo.toml @@ -10,16 +10,16 @@ keywords = ["atom", "rss", "planet", "feed", "blogging"] categories = ["web-programming"] [dependencies] -anyhow = "*" -clap = { version = "*", features = ["derive"] } -env_logger = "*" -feed-rs = "*" -log = "*" -ron = "*" # todo for development, to check atom-rs internal representation of feeds -serde = { version = "*", features = ["derive"] } -slug = "*" -tera = "*" -toml = "*" +anyhow = "1" +clap = { version = "4", features = ["derive"] } +env_logger = "0" +feed-rs = "2" +log = "0" +ron = "0" +serde = { version = "1", features = ["derive"] } +slug = "0" +tera = "1" +toml = "0" ureq = { version = "3.0.0-rc5", features = ["brotli", "charset", "gzip", "native-tls"]} -url = "*" -quick-xml = { version = "*", features = ["serialize"] } +url = "2" + From 0bd9eb5051f7db583f6bfd62f734624c587a6219 Mon Sep 17 00:00:00 2001 From: Thomas Koch Date: Mon, 13 Jan 2025 09:26:36 +0200 Subject: [PATCH 16/17] update README --- web/planet-mars/Cargo.toml | 2 +- web/planet-mars/README.md | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/web/planet-mars/Cargo.toml b/web/planet-mars/Cargo.toml index b5e00a006..400ac8333 100644 --- a/web/planet-mars/Cargo.toml +++ b/web/planet-mars/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "planet-mars" -version = "0.1.0" +version = "0.1.1" edition = "2021" authors = ["Thomas Koch "] description = "Feed aggregation planet like Planet Venus, produces static HTML and ATOM feed from fetched feeds." diff --git a/web/planet-mars/README.md b/web/planet-mars/README.md index 664fb790d..467fdeb1e 100644 --- a/web/planet-mars/README.md +++ b/web/planet-mars/README.md @@ -1,13 +1,24 @@ -Simple planet like planet venus but in rust and maintained. +Simple successor to Planet Venus but in Rust and maintained. Please see the rustdoc of main.rs for further information. -## todo +## Todo -* use a nice lib to process the config file +* find and use a nice lib to process the config file * should check whether dirs exists and are writeable * should check whether feed urls can be parsed +## Planet Venus + +Planet Venus is used by many planets on the internet. However its code has not +been maintained since ~2011 and it uses Python 2. + +Planet Mars should be a lightweight successor to Planet Venus. + +Still the Planet Venus documentation contains some useful information on +[Etiquette](https://intertwingly.net/code/venus/docs/etiquette.html) for +Planet hosters. + ## Credits While writing this, I read and also copied code from: From 1d9812d3cb305c1db0d892d7e0513d0a4eabd852 Mon Sep 17 00:00:00 2001 From: Thomas Koch Date: Mon, 13 Jan 2025 09:27:39 +0200 Subject: [PATCH 17/17] also commit Cargo.lock --- web/planet-mars/Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/planet-mars/Cargo.lock b/web/planet-mars/Cargo.lock index f36f7ec4d..3cfb214ca 100644 --- a/web/planet-mars/Cargo.lock +++ b/web/planet-mars/Cargo.lock @@ -1106,7 +1106,7 @@ checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" [[package]] name = "planet-mars" -version = "0.1.0" +version = "0.1.1" dependencies = [ "anyhow", "clap",