feat(pdk) support HTTP/2 on Admin API server by adding new PDK function nginx.get_statistics() for fetching Nginx statistics

Retired the `/nginx_status` location block and use LuaJIT FFI to fetch the counters directly instead.

Co-authored-by: Datong Sun <[email protected]>

Author: outsinre

kong/api/routes/health.lua

@@ -1,17 +1,12 @@
 local utils = require "kong.tools.utils"
 local declarative = require "kong.db.declarative"
 
-local find = string.find
-local select = select
 local tonumber = tonumber
 local kong = kong
 local knode  = (kong and kong.node) and kong.node or
                require "kong.pdk.node".new()
 
 
-local select = select
-local tonumber = tonumber
-local kong = kong
 local dbless = kong.configuration.database == "off"
 local data_plane_role = kong.configuration.role == "data_plane"
 
@@ -41,27 +36,9 @@ return {
       end
 
       -- nginx stats
-
-      local r = ngx.location.capture "/nginx_status"
-      if r.status ~= 200 then
-        kong.log.err(r.body)
-        return kong.response.exit(500, { message = "An unexpected error happened" })
-      end
-
-      local var = ngx.var
-      local accepted, handled, total = select(3, find(r.body, "accepts handled requests\n (%d*) (%d*) (%d*)"))
-
       local status_response = {
         memory = knode.get_memory_stats(unit, scale),
-        server = {
-          connections_active = tonumber(var.connections_active),
-          connections_reading = tonumber(var.connections_reading),
-          connections_writing = tonumber(var.connections_writing),
-          connections_waiting = tonumber(var.connections_waiting),
-          connections_accepted = tonumber(accepted),
-          connections_handled = tonumber(handled),
-          total_requests = tonumber(total)
-        },
+        server = kong.nginx.get_statistics(),
         database = {
           reachable = true,
         },

kong/pdk/nginx.lua

@@ -4,8 +4,39 @@
 -- details and meta information.
 -- @module kong.nginx
 
+local ffi = require "ffi"
 
-local ngx = ngx
+
+local C    = ffi.C
+local arch = ffi.arch
+local ngx  = ngx
+local tonumber = tonumber
+
+if arch == "x64" or arch == "arm64" then
+  ffi.cdef[[
+    uint64_t *ngx_stat_active;
+    uint64_t *ngx_stat_reading;
+    uint64_t *ngx_stat_writing;
+    uint64_t *ngx_stat_waiting;
+    uint64_t *ngx_stat_requests;
+    uint64_t *ngx_stat_accepted;
+    uint64_t *ngx_stat_handled;
+  ]]
+
+elseif arch == "x86" or arch == "arm" then
+  ffi.cdef[[
+    uint32_t *ngx_stat_active;
+    uint32_t *ngx_stat_reading;
+    uint32_t *ngx_stat_writing;
+    uint32_t *ngx_stat_waiting;
+    uint32_t *ngx_stat_requests;
+    uint32_t *ngx_stat_accepted;
+    uint32_t *ngx_stat_handled;
+  ]]
+
+else
+  kong.log.err("Unsupported arch: " .. arch)
+end
 
 
 local function new(self)
@@ -26,6 +57,38 @@ local function new(self)
   end
 
 
+  ---
+  -- Returns various connection and request metrics exposed by
+  -- Nginx, similar to those reported by the
+  -- [ngx_http_stub_status_module](https://nginx.org/en/docs/http/ngx_http_stub_status_module.html#data).
+  --
+  -- The following fields are included in the returned table:
+  -- * `connections_active` - the current number of active client connections including `connections_waiting`.
+  -- * `connections_reading` - the current number of connections where nginx is reading the request header.
+  -- * `connections_writing` - the current number of connections where nginx is writing the response back to the client.
+  -- * `connections_waiting` - the current number of idle client connections waiting for a request.
+  -- * `connections_accepted` - the total number of accepted client connections.
+  -- * `connections_handled` - the total number of handled connections. Same as `connections_accepted` unless some resource limits have been reached
+  --   (for example, the [`worker_connections`](https://nginx.org/en/docs/ngx_core_module.html#worker_connections) limit).
+  -- * `total_requests` - the total number of client requests.
+  --
+  -- @function kong.nginx.get_statistics
+  -- @treturn table Nginx connections and requests statistics
+  -- @usage
+  -- local nginx_statistics = kong.nginx.get_statistics()
+  function _NGINX.get_statistics()
+    return {
+      connections_active = tonumber(C.ngx_stat_active[0]),
+      connections_reading = tonumber(C.ngx_stat_reading[0]),
+      connections_writing = tonumber(C.ngx_stat_writing[0]),
+      connections_waiting = tonumber(C.ngx_stat_waiting[0]),
+      connections_accepted = tonumber(C.ngx_stat_accepted[0]),
+      connections_handled = tonumber(C.ngx_stat_handled[0]),
+      total_requests = tonumber(C.ngx_stat_requests[0])
+    }
+  end
+
+
   return _NGINX
 end
 

kong/plugins/prometheus/exporter.lua

@@ -1,9 +1,7 @@
 local kong = kong
 local ngx = ngx
-local find = string.find
 local lower = string.lower
 local concat = table.concat
-local select = select
 local ngx_timer_pending_count = ngx.timer.pending_count
 local ngx_timer_running_count = ngx.timer.running_count
 local balancer = require("kong.runloop.balancer")
@@ -297,26 +295,14 @@ local function metric_data()
     return kong.response.exit(500, { message = "An unexpected error occurred" })
   end
 
-  if ngx.location then
-    local r = ngx.location.capture "/nginx_status"
-
-    if r.status ~= 200 then
-      kong.log.warn("prometheus: failed to retrieve /nginx_status ",
-        "while processing /metrics endpoint")
-
-    else
-      local accepted, handled, total = select(3, find(r.body,
-        "accepts handled requests\n (%d*) (%d*) (%d*)"))
-      metrics.connections:set(accepted, { "accepted" })
-      metrics.connections:set(handled, { "handled" })
-      metrics.connections:set(total, { "total" })
-    end
-  end
-
-  metrics.connections:set(ngx.var.connections_active or 0, { "active" })
-  metrics.connections:set(ngx.var.connections_reading or 0, { "reading" })
-  metrics.connections:set(ngx.var.connections_writing or 0, { "writing" })
-  metrics.connections:set(ngx.var.connections_waiting or 0, { "waiting" })
+  local nginx_statistics = kong.nginx.get_statistics()
+  metrics.connections:set(nginx_statistics['connections_accepted'], { "accepted" })
+  metrics.connections:set(nginx_statistics['connections_handled'], { "handled" })
+  metrics.connections:set(nginx_statistics['total_requests'], { "total" })
+  metrics.connections:set(nginx_statistics['connections_active'], { "active" })
+  metrics.connections:set(nginx_statistics['connections_reading'], { "reading" })
+  metrics.connections:set(nginx_statistics['connections_writing'], { "writing" })
+  metrics.connections:set(nginx_statistics['connections_waiting'], { "waiting" })
 
   metrics.timers:set(ngx_timer_running_count(), {"running"})
   metrics.timers:set(ngx_timer_pending_count(), {"pending"})

kong/templates/nginx_kong.lua

@@ -363,12 +363,6 @@ server {
         }
     }
 
-    location /nginx_status {
-        internal;
-        access_log off;
-        stub_status;
-    }
-
     location /robots.txt {
         return 200 'User-agent: *\nDisallow: /';
     }
@@ -408,12 +402,6 @@ server {
         }
     }
 
-    location /nginx_status {
-        internal;
-        access_log off;
-        stub_status;
-    }
-
     location /robots.txt {
         return 200 'User-agent: *\nDisallow: /';
     }

spec/03-plugins/26-prometheus/05-metrics_spec.lua

@@ -0,0 +1,155 @@
+local helpers = require "spec.helpers" -- hard dependency
+
+
+local ngx = ngx
+
+local fixtures = {
+  dns_mock = helpers.dns_mock.new({
+    mocks_only = true
+  }),
+  http_mock = {},
+  stream_mock = {}
+}
+
+fixtures.dns_mock:A{
+  name = "mock.example.com",
+  address = "127.0.0.1"
+}
+
+fixtures.dns_mock:A{
+  name = "status.example.com",
+  address = "127.0.0.1"
+}
+
+local status_api_port = helpers.get_available_port()
+
+
+for _, strategy in helpers.each_strategy() do
+  describe("Plugin: prometheus (metrics)", function()
+    local bp
+    local admin_ssl_client -- admin_ssl_client (lua-resty-http) does not support h2
+    local proxy_ssl_client -- proxy_ssl_client (lua-resty-http) does not support h2
+
+    setup(function()
+      bp = helpers.get_db_utils(strategy, {"services", "routes", "plugins"})
+
+      local mock_ssl_service = bp.services:insert{
+        name = "mock-ssl-service",
+        host = helpers.mock_upstream_ssl_host,
+        port = helpers.mock_upstream_ssl_port,
+        protocol = helpers.mock_upstream_ssl_protocol
+      }
+      bp.routes:insert{
+        name = "mock-ssl-route",
+        protocols = {"https"},
+        hosts = {"mock.example.com"},
+        paths = {"/"},
+        service = {
+            id = mock_ssl_service.id
+        }
+      }
+
+      local status_api_ssl_service = bp.services:insert{
+        name = "status-api-ssl-service",
+        url = "https://127.0.0.1:" .. status_api_port .. "/metrics"
+      }
+      bp.routes:insert{
+        name = "status-api-ssl-route",
+        protocols = {"https"},
+        hosts = {"status.example.com"},
+        paths = {"/metrics"},
+        service = {
+          id = status_api_ssl_service.id
+        }
+      }
+
+      bp.plugins:insert{
+        name = "prometheus" -- globally enabled
+      }
+
+      assert(helpers.start_kong({
+        nginx_conf = "spec/fixtures/custom_nginx.template",
+        plugins = "bundled,prometheus",
+        status_listen = '127.0.0.1:' .. status_api_port .. ' ssl', -- status api does not support h2
+        status_access_log = "logs/status_access.log",
+        status_error_log = "logs/status_error.log"
+      }, nil, nil, fixtures))
+
+    end)
+
+    teardown(function()
+      if admin_ssl_client then
+        admin_ssl_client:close()
+      end
+      if proxy_ssl_client then
+        proxy_ssl_client:close()
+      end
+
+      helpers.stop_kong()
+    end)
+
+    before_each(function()
+      admin_ssl_client = helpers.admin_client()
+      proxy_ssl_client = helpers.proxy_ssl_client()
+    end)
+
+    after_each(function()
+      if admin_ssl_client then
+        admin_ssl_client:close()
+      end
+      if proxy_ssl_client then
+        proxy_ssl_client:close()
+      end
+    end)
+
+    it("expose Nginx connection metrics by admin API #a1.1", function()
+      local res = assert(admin_ssl_client:send{
+        method = "GET",
+        path = "/metrics"
+      })
+      local body = assert.res_status(200, res)
+
+      assert.matches('kong_nginx_metric_errors_total 0', body, nil, true)
+      assert.matches('kong_nginx_' .. ngx.config.subsystem .. '_current_connections{state="%w+"} %d+', body)
+    end)
+
+    it("increments the count of proxied requests #p1.1", function()
+      local res = assert(proxy_ssl_client:send{
+        method = "GET",
+        path = "/status/400",
+        headers = {
+          ["Host"] = "mock.example.com"
+        }
+      })
+      assert.res_status(400, res)
+
+      helpers.wait_until(function()
+        local res = assert(admin_ssl_client:send{
+          method = "GET",
+          path = "/metrics"
+        })
+        local body = assert.res_status(200, res)
+
+        assert.matches('kong_nginx_metric_errors_total 0', body, nil, true)
+
+        return body:find('kong_http_status{service="mock-ssl-service",route="mock-ssl-route",code="400"} 1',
+          nil, true)
+      end)
+    end)
+
+    it("expose Nginx connection metrics by status API #s1.1", function()
+      local res = assert(proxy_ssl_client:send{
+        method = "GET",
+        path = "/metrics",
+        headers = {
+          ["Host"] = "status.example.com"
+        }
+      })
+      local body = assert.res_status(200, res)
+
+      assert.matches('kong_nginx_metric_errors_total 0', body, nil, true)
+      assert.matches('kong_nginx_' .. ngx.config.subsystem .. '_current_connections{state="%w+"} %d+', body)
+    end)
+
+  end)
+end

spec/fixtures/custom_nginx.template

@@ -385,12 +385,6 @@ http {
             }
         }
 
-        location /nginx_status {
-            internal;
-            access_log off;
-            stub_status;
-        }
-
         location /robots.txt {
             return 200 'User-agent: *\nDisallow: /';
         }
@@ -430,12 +424,6 @@ http {
             }
         }
 
-        location /nginx_status {
-            internal;
-            access_log off;
-            stub_status;
-        }
-
         location /robots.txt {
             return 200 'User-agent: *\nDisallow: /';
         }

spec/fixtures/prometheus/metrics.conf

@@ -10,9 +10,4 @@ server {
        }
    }
 
-   location /nginx_status {
-       internal;
-       access_log off;
-       stub_status;
-   }
 }