feat(users/Profpatsch/whatcd-resolver): parallelize search pages

This bunches up 5 search page requests to run at the same time.
We use a conduit now, so we could get smart about returning partial
results and such (if for example upstream puts us into the rate limit,
which they do after 10 requests.

Change-Id: Idbb174334fa499c16b3426a8d129deaf3a1d3b0b
Reviewed-on: https://cl.tvl.fyi/c/depot/+/13245
Tested-by: BuildkiteCI
Reviewed-by: Profpatsch <mail@profpatsch.de>
This commit is contained in:
Profpatsch 2025-03-11 13:10:28 +01:00
parent ca6c5ac59e
commit ae0e75aaf2
4 changed files with 156 additions and 90 deletions

View file

@ -213,11 +213,11 @@ asUtcTimeLenient = Field.toJsonParser (Field.jsonString >>> Field.utcTimeLenient
-- We dont provide a version that infers the json object key,
-- since that conflates internal naming with the external API, which is dangerous.
--
-- @@
-- @
-- do
-- txt <- keyLabel @"myLabel" "jsonKeyName" Json.asText
-- pure (txt :: Label "myLabel" Text)
-- @@
-- @
keyLabel ::
forall label err m a.
(Monad m) =>
@ -230,11 +230,11 @@ keyLabel = do
-- | Parse a key from the object, à la 'Json.key', return a labelled value.
-- Version of 'keyLabel' that requires a proxy.
--
-- @@
-- @
-- do
-- txt <- keyLabel' (Proxy @"myLabel") "jsonKeyName" Json.asText
-- pure (txt :: Label "myLabel" Text)
-- @@
-- @
keyLabel' ::
forall label err m a.
(Monad m) =>
@ -249,11 +249,11 @@ keyLabel' Proxy key parser = label @label <$> Json.key key parser
-- We dont provide a version that infers the json object key,
-- since that conflates internal naming with the external API, which is dangerous.
--
-- @@
-- @
-- do
-- txt <- keyLabelMay @"myLabel" "jsonKeyName" Json.asText
-- pure (txt :: Label "myLabel" (Maybe Text))
-- @@
-- @
keyLabelMay ::
forall label err m a.
(Monad m) =>
@ -263,14 +263,33 @@ keyLabelMay ::
keyLabelMay = do
keyLabelMay' (Proxy @label)
-- | Parse an optional key from the object. The inner parsers return value has to be a Monoid,
-- and we collapse the missing key into its 'mempty'.
--
-- For example, if the inner parser returns a list, the missing key will be parsed as an empty list.
--
-- @
-- do
-- txt <- keyMay' "jsonKeyName" (Json.eachInArray Json.asText)
-- pure (txt :: [Text])
-- @
--
-- will return @[]@ if the key is missing or if the value is the empty array.
keyMayMempty ::
(Monad m, Monoid a) =>
Text ->
Json.ParseT err m a ->
Json.ParseT err m a
keyMayMempty key parser = Json.keyMay key parser <&> fromMaybe mempty
-- | Parse an optional key from the object, à la 'Json.keyMay', return a labelled value.
-- Version of 'keyLabelMay' that requires a proxy.
--
-- @@
-- @
-- do
-- txt <- keyLabelMay' (Proxy @"myLabel") "jsonKeyName" Json.asText
-- pure (txt :: Label "myLabel" (Maybe Text))
-- @@
-- @
keyLabelMay' ::
forall label err m a.
(Monad m) =>