Add default instance limits for distributors and ingesters

Signed-off-by: Friedrich Gonzalez <[email protected]>

Author: friedrichg

CHANGELOG.md

@@ -1,6 +1,8 @@
 # Changelog
 
 ## master / unreleased
+* [CHANGE] Add default max series for ingesters
+* [CHANGE] Add default max inflight pushes for distributors
 * [CHANGE] Remove mem-ballast from distributor and querier.
 * [CHANGE] Increase cpu requests for querier to 2.
 * [CHANGE] Configure default GOMAXPROCS and GOMEMLIMIT for all cortex modules

cortex/config.libsonnet

@@ -395,13 +395,12 @@
     ingester_stream_chunks_when_using_blocks: true,
 
     // Ingester limits are put directly into runtime config, if not null. Available limits:
-    //    ingester_instance_limits: {
-    //      max_inflight_push_requests: 0,  // Max inflight push requests per ingester. 0 = no limit.
-    //      max_ingestion_rate: 0,  // Max ingestion rate (samples/second) per ingester. 0 = no limit.
-    //      max_series: 0,  // Max number of series per ingester. 0 = no limit.
-    //      max_tenants: 0,  // Max number of tenants per ingester. 0 = no limit.
-    //    },
-    ingester_instance_limits: null,
+    ingester_instance_limits: {
+      // max_inflight_push_requests: ,  // Max inflight push requests per ingester. 0 = no limit.
+      // max_ingestion_rate: 0,  // Max ingestion rate (samples/second) per ingester. 0 = no limit.
+      max_series: 4.8e+6,  // Max number of series per ingester. 0 = no limit. 4.8 million is closely tied to 15Gb in requests per ingester
+      // max_tenants: 0,  // Max number of tenants per ingester. 0 = no limit.
+    },
   },
 
   local configMap = $.core.v1.configMap,

cortex/distributor.libsonnet

@@ -31,6 +31,7 @@
       // Do not extend the replication set on unhealthy (or LEAVING) ingester when "unregister on shutdown"
       // is set to false.
       'distributor.extend-writes': $._config.unregister_ingesters_on_shutdown,
+      'distributor.instance-limits.max-inflight-push-requests': 60,  //60 is very conservative to protect the distributor from OOMs
     },
 
   distributor_env_map:: {