feat: add optional nulls=stripped parameter for mediatypes applicatio…

CHANGELOG.md

@@ -24,6 +24,7 @@ This project adheres to [Semantic Versioning](http://semver.org/).
  - #2856, Add the `--version` CLI option that prints the version information - @laurenceisla
  - #1655, Improve `details` field of the singular error response - @taimoorzaeem
  - #740, Add `Preference-Applied` in response for `Prefer: return=representation/headers-only/minimal` - @taimoorzaeem
+ - #1601, Add optional `nulls=stripped` parameter for mediatypes `application/vnd.pgrst.array+json` and `application/vnd.pgrst.object+json` - @taimoorzaeem
 
 ### Fixed
 

postgrest.cabal

@@ -218,6 +218,7 @@ test-suite spec
                       Feature.Query.RelatedQueriesSpec
                       Feature.Query.RpcSpec
                       Feature.Query.SingularSpec
+                      Feature.Query.NullsStrip
                       Feature.Query.SpreadQueriesSpec
                       Feature.Query.UnicodeSpec
                       Feature.Query.UpdateSpec

src/PostgREST/ApiRequest.hs

@@ -366,13 +366,14 @@ producedMediaTypes conf action path =
     ActionInvoke _  -> invokeMediaTypes
     ActionInfo      -> defaultMediaTypes
     ActionMutate _  -> defaultMediaTypes
-    ActionInspect _ -> [MTOpenAPI, MTApplicationJSON, MTAny]
+    ActionInspect _ -> inspectMediaTypes
   where
+    inspectMediaTypes = [MTOpenAPI, MTApplicationJSON, MTArrayJSONStrip, MTAny]
     invokeMediaTypes =
       defaultMediaTypes
         ++ rawMediaTypes
         ++ [MTOpenAPI | pathIsRootSpec path]
     defaultMediaTypes =
-      [MTApplicationJSON, MTSingularJSON, MTGeoJSON, MTTextCSV] ++
+      [MTApplicationJSON, MTArrayJSONStrip, MTSingularJSON True, MTSingularJSON False, MTGeoJSON, MTTextCSV] ++
       [MTPlan MTApplicationJSON PlanText mempty | configDbPlanEnabled conf] ++ [MTAny]
     rawMediaTypes = configRawMediaTypes conf `union` [MTOctetStream, MTTextPlain, MTTextXML]

src/PostgREST/Error.hs

@@ -478,7 +478,7 @@ instance PgrstError Error where
   headers (JwtTokenInvalid m)    = [MediaType.toContentType MTApplicationJSON, invalidTokenHeader m]
   headers JwtTokenRequired       = [MediaType.toContentType MTApplicationJSON, requiredTokenHeader]
   headers (PgErr err)            = headers err
-  headers SingularityError{}     = [MediaType.toContentType MTSingularJSON]
+  headers SingularityError{}     = [MediaType.toContentType (MTSingularJSON False)]
   headers _                      = [MediaType.toContentType MTApplicationJSON]
 
 instance JSON.ToJSON Error where

src/PostgREST/MediaType.hs

@@ -19,7 +19,8 @@ import Protolude
 -- | Enumeration of currently supported media types
 data MediaType
   = MTApplicationJSON
-  | MTSingularJSON
+  | MTArrayJSONStrip
+  | MTSingularJSON Bool
   | MTGeoJSON
   | MTTextCSV
   | MTTextPlain
@@ -33,19 +34,20 @@ data MediaType
   | MTPlan MediaType MTPlanFormat [MTPlanOption]
   deriving Show
 instance Eq MediaType where
-  MTApplicationJSON == MTApplicationJSON = True
-  MTSingularJSON    == MTSingularJSON    = True
-  MTGeoJSON         == MTGeoJSON         = True
-  MTTextCSV         == MTTextCSV         = True
-  MTTextPlain       == MTTextPlain       = True
-  MTTextXML         == MTTextXML         = True
-  MTOpenAPI         == MTOpenAPI         = True
-  MTUrlEncoded      == MTUrlEncoded      = True
-  MTOctetStream     == MTOctetStream     = True
-  MTAny             == MTAny             = True
-  MTOther x         == MTOther y         = x == y
-  MTPlan{}          == MTPlan{}          = True
-  _                 == _                 = False
+  MTApplicationJSON    == MTApplicationJSON = True
+  MTArrayJSONStrip     == MTArrayJSONStrip  = True
+  MTSingularJSON x     == MTSingularJSON y  = x == y
+  MTGeoJSON            == MTGeoJSON         = True
+  MTTextCSV            == MTTextCSV         = True
+  MTTextPlain          == MTTextPlain       = True
+  MTTextXML            == MTTextXML         = True
+  MTOpenAPI            == MTOpenAPI         = True
+  MTUrlEncoded         == MTUrlEncoded      = True
+  MTOctetStream        == MTOctetStream     = True
+  MTAny                == MTAny             = True
+  MTOther x            == MTOther y         = x == y
+  MTPlan{}             == MTPlan{}          = True
+  _                    == _                 = False
 
 data MTPlanOption
   = PlanAnalyze | PlanVerbose | PlanSettings | PlanBuffers | PlanWAL
@@ -66,18 +68,20 @@ toContentType ct = (hContentType, toMime ct <> charset)
 
 -- | Convert from MediaType to a ByteString representing the mime type
 toMime :: MediaType -> ByteString
-toMime MTApplicationJSON = "application/json"
-toMime MTGeoJSON         = "application/geo+json"
-toMime MTTextCSV         = "text/csv"
-toMime MTTextPlain       = "text/plain"
-toMime MTTextXML         = "text/xml"
-toMime MTOpenAPI         = "application/openapi+json"
-toMime MTSingularJSON    = "application/vnd.pgrst.object+json"
-toMime MTUrlEncoded      = "application/x-www-form-urlencoded"
-toMime MTOctetStream     = "application/octet-stream"
-toMime MTAny             = "*/*"
-toMime (MTOther ct)      = ct
-toMime (MTPlan mt fmt opts) =
+toMime MTApplicationJSON      = "application/json"
+toMime MTArrayJSONStrip       = "application/vnd.pgrst.array+json;nulls=stripped"
+toMime MTGeoJSON              = "application/geo+json"
+toMime MTTextCSV              = "text/csv"
+toMime MTTextPlain            = "text/plain"
+toMime MTTextXML              = "text/xml"
+toMime MTOpenAPI              = "application/openapi+json"
+toMime (MTSingularJSON True)  = "application/vnd.pgrst.object+json;nulls=stripped"
+toMime (MTSingularJSON False) = "application/vnd.pgrst.object+json"
+toMime MTUrlEncoded           = "application/x-www-form-urlencoded"
+toMime MTOctetStream          = "application/octet-stream"
+toMime MTAny                  = "*/*"
+toMime (MTOther ct)           = ct
+toMime (MTPlan mt fmt opts)   =
   "application/vnd.pgrst.plan+" <> toMimePlanFormat fmt <>
   ("; for=\"" <> toMime mt <> "\"") <>
   (if null opts then mempty else "; options=" <> BS.intercalate "|" (toMimePlanOption <$> opts))
@@ -106,26 +110,46 @@ toMimePlanFormat PlanText = "text"
 --
 -- >>> decodeMediaType "application/vnd.pgrst.plan+json;for=\"text/csv\""
 -- MTPlan MTTextCSV PlanJSON []
+--
+-- >>> decodeMediaType "application/vnd.pgrst.array+json;nulls=stripped"
+-- MTArrayJSONStrip
+--
+-- >>> decodeMediaType "application/vnd.pgrst.array+json"
+-- MTApplicationJSON
+--
+-- >>> decodeMediaType "application/vnd.pgrst.object+json;nulls=stripped"
+-- MTSingularJSON True
+--
+-- >>> decodeMediaType "application/vnd.pgrst.object+json"
+-- MTSingularJSON False
+
 decodeMediaType :: BS.ByteString -> MediaType
 decodeMediaType mt =
   case BS.split (BS.c2w ';') mt of
-    "application/json":_                   -> MTApplicationJSON
-    "application/geo+json":_               -> MTGeoJSON
-    "text/csv":_                           -> MTTextCSV
-    "text/plain":_                         -> MTTextPlain
-    "text/xml":_                           -> MTTextXML
-    "application/openapi+json":_           -> MTOpenAPI
-    "application/vnd.pgrst.object+json":_  -> MTSingularJSON
-    "application/vnd.pgrst.object":_       -> MTSingularJSON
-    "application/x-www-form-urlencoded":_  -> MTUrlEncoded
-    "application/octet-stream":_           -> MTOctetStream
-    "application/vnd.pgrst.plan":rest      -> getPlan PlanText rest
-    "application/vnd.pgrst.plan+text":rest -> getPlan PlanText rest
-    "application/vnd.pgrst.plan+json":rest -> getPlan PlanJSON rest
-    "*/*":_                                -> MTAny
-    other:_                                -> MTOther other
-    _                                      -> MTAny
+    "application/json":_                     -> MTApplicationJSON
+    "application/geo+json":_                 -> MTGeoJSON
+    "text/csv":_                             -> MTTextCSV
+    "text/plain":_                           -> MTTextPlain
+    "text/xml":_                             -> MTTextXML
+    "application/openapi+json":_             -> MTOpenAPI
+    "application/x-www-form-urlencoded":_    -> MTUrlEncoded
+    "application/octet-stream":_             -> MTOctetStream
+    "application/vnd.pgrst.plan":rest        -> getPlan PlanText rest
+    "application/vnd.pgrst.plan+text":rest   -> getPlan PlanText rest
+    "application/vnd.pgrst.plan+json":rest   -> getPlan PlanJSON rest
+    "application/vnd.pgrst.object+json":rest -> checkSingularNullStrip rest
+    "application/vnd.pgrst.object":rest      -> checkSingularNullStrip rest
+    "application/vnd.pgrst.array+json":rest  -> checkArrayNullStrip rest
+    "*/*":_                                  -> MTAny
+    other:_                                  -> MTOther other
+    _                                        -> MTAny
   where
+    checkArrayNullStrip ["nulls=stripped"] = MTArrayJSONStrip
+    checkArrayNullStrip  _                 = MTApplicationJSON
+
+    checkSingularNullStrip ["nulls=stripped"] = MTSingularJSON True
+    checkSingularNullStrip  _                 = MTSingularJSON False
+
     getPlan fmt rest =
       let
         opts         = BS.split (BS.c2w '|') $ fromMaybe mempty (BS.stripPrefix "options=" =<< find (BS.isPrefixOf "options=") rest)

src/PostgREST/Plan.hs

@@ -847,7 +847,8 @@ mediaToAggregate mt binField apiReq@ApiRequest{iAction=act, iPreferences=Prefere
   if noAgg then NoAgg
   else case mt of
     MTApplicationJSON     -> BuiltinAggJson
-    MTSingularJSON        -> BuiltinAggSingleJson
+    MTSingularJSON strip  -> BuiltinAggSingleJson strip
+    MTArrayJSONStrip      -> BuiltinAggArrayJsonStrip
     MTGeoJSON             -> BuiltinAggGeoJson
     MTTextCSV             -> BuiltinAggCsv
     MTAny                 -> BuiltinAggJson

src/PostgREST/Query.hs

@@ -206,7 +206,7 @@ writeQuery MutateReadPlan{mrReadPlan, mrMutatePlan, mrResAgg} apiReq@ApiRequest{
 failNotSingular :: MediaType -> ResultSet -> DbHandler ()
 failNotSingular _ RSPlan{} = pure ()
 failNotSingular mediaType RSStandard{rsQueryTotal=queryTotal} =
-  when (mediaType == MTSingularJSON && queryTotal /= 1) $ do
+  when (elem mediaType [MTSingularJSON True,MTSingularJSON False]  && queryTotal /= 1) $ do
     lift SQL.condemn
     throwError $ Error.singularityError queryTotal
 

src/PostgREST/Query/SqlFragment.hs

@@ -186,24 +186,29 @@ asCsvF = asCsvHeaderF <> " || '\n' || " <> asCsvBodyF
       ")"
     asCsvBodyF = "coalesce(string_agg(substring(_postgrest_t::text, 2, length(_postgrest_t::text) - 2), '\n'), '')"
 
-asJsonSingleF :: Maybe Routine -> SQL.Snippet
-asJsonSingleF rout
-  | returnsScalar = "coalesce(json_agg(_postgrest_t.pgrst_scalar)->0, 'null')"
-  | otherwise     = "coalesce(json_agg(_postgrest_t)->0, 'null')"
+addNullsToSnip :: Bool -> SQL.Snippet -> SQL.Snippet
+addNullsToSnip strip snip =
+  if strip then "json_strip_nulls(" <> snip <> ")" else  snip
+
+asJsonSingleF :: Maybe Routine -> Bool -> SQL.Snippet
+asJsonSingleF rout strip
+  | returnsScalar = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)->0"  <> ", 'null')"
+  | otherwise     = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)->0"  <> ", 'null')"
   where
     returnsScalar = maybe False funcReturnsScalar rout
 
-asJsonF :: Maybe Routine -> SQL.Snippet
-asJsonF rout
-  | returnsSingleComposite              = "coalesce(json_agg(_postgrest_t)->0, 'null')"
-  | returnsScalar                       = "coalesce(json_agg(_postgrest_t.pgrst_scalar)->0, 'null')"
-  | returnsSetOfScalar                  = "coalesce(json_agg(_postgrest_t.pgrst_scalar), '[]')"
-  | otherwise                           = "coalesce(json_agg(_postgrest_t), '[]')"
+asJsonF :: Maybe Routine -> Bool -> SQL.Snippet
+asJsonF rout strip
+  | returnsSingleComposite    = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)->0" <> ", 'null')"
+  | returnsScalar             = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)->0" <> ", 'null')"
+  | returnsSetOfScalar        = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t.pgrst_scalar)" <> ", '[]')"
+  | otherwise                 = "coalesce(" <> addNullsToSnip strip "json_agg(_postgrest_t)" <> ", '[]')"
   where
     (returnsSingleComposite, returnsScalar, returnsSetOfScalar) = case rout of
       Just r  -> (funcReturnsSingleComposite r, funcReturnsScalar r, funcReturnsSetOfScalar r)
       Nothing -> (False, False, False)
 
+
 asXmlF :: Maybe FieldName -> SQL.Snippet
 asXmlF (Just fieldName) = "coalesce(xmlagg(_postgrest_t." <> pgFmtIdent fieldName <> "), '')"
 -- TODO unreachable because a previous step(binaryField) will validate that there's a field. This will be cleared once custom media types are implemented.
@@ -493,10 +498,11 @@ setConfigLocalJson prefix keyVals = [setConfigLocal mempty (prefix, gucJsonVal k
 
 aggF :: Maybe Routine -> ResultAggregate -> SQL.Snippet
 aggF rout = \case
-  BuiltinAggJson          -> asJsonF rout
-  BuiltinAggSingleJson    -> asJsonSingleF rout
-  BuiltinAggGeoJson       -> asGeoJsonF
-  BuiltinAggCsv           -> asCsvF
-  BuiltinAggXml    bField -> asXmlF    bField
-  BuiltinAggBinary bField -> asBinaryF bField
-  NoAgg                   -> "''::text"
+  BuiltinAggJson             -> asJsonF rout False
+  BuiltinAggArrayJsonStrip   -> asJsonF rout True
+  BuiltinAggSingleJson strip -> asJsonSingleF rout strip
+  BuiltinAggGeoJson          -> asGeoJsonF
+  BuiltinAggCsv              -> asCsvF
+  BuiltinAggXml    bField    -> asXmlF    bField
+  BuiltinAggBinary bField    -> asBinaryF bField
+  NoAgg                      -> "''::text"

src/PostgREST/Response/OpenAPI.hs

@@ -350,7 +350,7 @@ makeProcPathItem pd = ("/rpc/" ++ toS (pdName pd), pe)
       & summary .~ pSum
       & description .~ mfilter (/="") pDesc
       & tags .~ Set.fromList ["(rpc) " <> pdName pd]
-      & produces ?~ makeMimeList [MTApplicationJSON, MTSingularJSON]
+      & produces ?~ makeMimeList [MTApplicationJSON, MTSingularJSON True, MTSingularJSON False]
       & at 200 ?~ "OK"
     getOp = procOp
       & parameters .~ makeProcGetParams (pdParams pd)
@@ -406,8 +406,8 @@ postgrestSpec (prettyVersion, docsVersion) rels pds ti (s, h, p, b) sd allowSecu
   & definitions .~ fromList (makeTableDef rels <$> ti)
   & parameters .~ fromList (makeParamDefs ti)
   & paths .~ makePathItems pds ti
-  & produces .~ makeMimeList [MTApplicationJSON, MTSingularJSON, MTTextCSV]
-  & consumes .~ makeMimeList [MTApplicationJSON, MTSingularJSON, MTTextCSV]
+  & produces .~ makeMimeList [MTApplicationJSON, MTSingularJSON True, MTSingularJSON False, MTTextCSV]
+  & consumes .~ makeMimeList [MTApplicationJSON, MTSingularJSON True, MTSingularJSON False, MTTextCSV]
   & securityDefinitions .~ makeSecurityDefinitions securityDefName allowSecurityDef
   & security .~ [SecurityRequirement (fromList [(securityDefName, [])]) | allowSecurityDef]
     where

src/PostgREST/SchemaCache/Routine.hs

@@ -90,7 +90,8 @@ type RoutineMap = HM.HashMap QualifiedIdentifier [Routine]
 
 data ResultAggregate
    = BuiltinAggJson
-   | BuiltinAggSingleJson
+   | BuiltinAggSingleJson Bool
+   | BuiltinAggArrayJsonStrip
    | BuiltinAggGeoJson
    | BuiltinAggCsv
    | BuiltinAggXml    (Maybe FieldName)