feat(users/Profpatsch/whatcd-resolver): parallelize search pages
This bunches up 5 search page requests to run at the same time. We use a conduit now, so we could get smart about returning partial results and such (if for example upstream puts us into the rate limit, which they do after 10 requests. Change-Id: Idbb174334fa499c16b3426a8d129deaf3a1d3b0b Reviewed-on: https://cl.tvl.fyi/c/depot/+/13245 Tested-by: BuildkiteCI Reviewed-by: Profpatsch <mail@profpatsch.de>
This commit is contained in:
parent
ca6c5ac59e
commit
ae0e75aaf2
4 changed files with 156 additions and 90 deletions
|
|
@ -213,11 +213,11 @@ asUtcTimeLenient = Field.toJsonParser (Field.jsonString >>> Field.utcTimeLenient
|
|||
-- We don’t provide a version that infers the json object key,
|
||||
-- since that conflates internal naming with the external API, which is dangerous.
|
||||
--
|
||||
-- @@
|
||||
-- @
|
||||
-- do
|
||||
-- txt <- keyLabel @"myLabel" "jsonKeyName" Json.asText
|
||||
-- pure (txt :: Label "myLabel" Text)
|
||||
-- @@
|
||||
-- @
|
||||
keyLabel ::
|
||||
forall label err m a.
|
||||
(Monad m) =>
|
||||
|
|
@ -230,11 +230,11 @@ keyLabel = do
|
|||
-- | Parse a key from the object, à la 'Json.key', return a labelled value.
|
||||
-- Version of 'keyLabel' that requires a proxy.
|
||||
--
|
||||
-- @@
|
||||
-- @
|
||||
-- do
|
||||
-- txt <- keyLabel' (Proxy @"myLabel") "jsonKeyName" Json.asText
|
||||
-- pure (txt :: Label "myLabel" Text)
|
||||
-- @@
|
||||
-- @
|
||||
keyLabel' ::
|
||||
forall label err m a.
|
||||
(Monad m) =>
|
||||
|
|
@ -249,11 +249,11 @@ keyLabel' Proxy key parser = label @label <$> Json.key key parser
|
|||
-- We don’t provide a version that infers the json object key,
|
||||
-- since that conflates internal naming with the external API, which is dangerous.
|
||||
--
|
||||
-- @@
|
||||
-- @
|
||||
-- do
|
||||
-- txt <- keyLabelMay @"myLabel" "jsonKeyName" Json.asText
|
||||
-- pure (txt :: Label "myLabel" (Maybe Text))
|
||||
-- @@
|
||||
-- @
|
||||
keyLabelMay ::
|
||||
forall label err m a.
|
||||
(Monad m) =>
|
||||
|
|
@ -263,14 +263,33 @@ keyLabelMay ::
|
|||
keyLabelMay = do
|
||||
keyLabelMay' (Proxy @label)
|
||||
|
||||
-- | Parse an optional key from the object. The inner parser’s return value has to be a Monoid,
|
||||
-- and we collapse the missing key into its 'mempty'.
|
||||
--
|
||||
-- For example, if the inner parser returns a list, the missing key will be parsed as an empty list.
|
||||
--
|
||||
-- @
|
||||
-- do
|
||||
-- txt <- keyMay' "jsonKeyName" (Json.eachInArray Json.asText)
|
||||
-- pure (txt :: [Text])
|
||||
-- @
|
||||
--
|
||||
-- will return @[]@ if the key is missing or if the value is the empty array.
|
||||
keyMayMempty ::
|
||||
(Monad m, Monoid a) =>
|
||||
Text ->
|
||||
Json.ParseT err m a ->
|
||||
Json.ParseT err m a
|
||||
keyMayMempty key parser = Json.keyMay key parser <&> fromMaybe mempty
|
||||
|
||||
-- | Parse an optional key from the object, à la 'Json.keyMay', return a labelled value.
|
||||
-- Version of 'keyLabelMay' that requires a proxy.
|
||||
--
|
||||
-- @@
|
||||
-- @
|
||||
-- do
|
||||
-- txt <- keyLabelMay' (Proxy @"myLabel") "jsonKeyName" Json.asText
|
||||
-- pure (txt :: Label "myLabel" (Maybe Text))
|
||||
-- @@
|
||||
-- @
|
||||
keyLabelMay' ::
|
||||
forall label err m a.
|
||||
(Monad m) =>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue