fix: remove goss from healthcheck (#2600)

ikheifets-splunk · mstopa-splunk · commit b6ac66b05a82 · 2024-10-16T16:40:00.000+02:00
Signed-off-by: Ilya Kheifets &lt;ikheifets@splunk.com&gt;
diff --git a/ansible/resources/podman_sc4s.service b/ansible/resources/podman_sc4s.service
@@ -38,8 +38,8 @@ ExecStart=/usr/bin/podman run \
         -v "$SC4S_ARCHIVE_MOUNT" \
         -v "$SC4S_TLS_MOUNT" \
         --env-file=/opt/sc4s/env_file \
-        --health-cmd="/healthcheck.sh" \
-        --health-interval=10s --health-retries=6 --health-timeout=6s \
+        --health-cmd="/usr/sbin/syslog-ng-ctl healthcheck --timeout 5" \
+        --health-interval=2m --health-retries=6 --health-timeout=5s \
         --network host \
         --name SC4S \
         --rm $SC4S_IMAGE
diff --git a/charts/splunk-connect-for-syslog/templates/statefulset.yaml b/charts/splunk-connect-for-syslog/templates/statefulset.yaml
@@ -134,9 +134,6 @@ spec:
             - name: ietf-dflt-tls
               containerPort: 5425
               protocol: TCP
-            - name: health
-              containerPort: 8080
-              protocol: TCP
           {{- if .Values.sc4s }}
           {{- if .Values.sc4s.vendor_product }}
             {{- range $vp := .Values.sc4s.vendor_product }}
@@ -216,16 +213,14 @@ spec:
             readOnly: true
           {{- end }}
           livenessProbe:
-            httpGet:
-              path: /healthz
-              port: 8080
+            exec:
+              command: ["syslog-ng-ctl", "healthcheck", "--timeout", "5"]
             initialDelaySeconds: 10
             periodSeconds: 3              
             timeoutSeconds: 5            
           readinessProbe:
-            httpGet:
-              path: /healthz
-              port: 8080
+            exec:
+              command: ["syslog-ng-ctl", "healthcheck", "--timeout", "5"]
             initialDelaySeconds: 10
             failureThreshold: 60
             periodSeconds: 10
diff --git a/docs/configuration.md b/docs/configuration.md
@@ -334,8 +334,8 @@ ExecStart=/usr/bin/podman run \
         -v "$SC4S_TLS_MOUNT" \
         --privileged \
         --env-file=/opt/sc4s/env_file \
-        --health-cmd="/healthcheck.sh" \
-        --health-interval=10s --health-retries=6 --health-timeout=6s \
+        --health-cmd="/usr/sbin/syslog-ng-ctl healthcheck --timeout 5" \
+        --health-interval=2m --health-retries=6 --health-timeout=5s \
         --network host \
         --name SC4S \
         --rm $SC4S_IMAGE
diff --git a/docs/gettingstarted/ansible-docker-podman.md b/docs/gettingstarted/ansible-docker-podman.md
@@ -79,7 +79,6 @@ SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=main for sour
 SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=main for sourcetype=sc4s:events...
 syslog-ng checking config
 sc4s version=v1.36.0
-starting goss
 starting syslog-ng
 ```
 
diff --git a/docs/gettingstarted/ansible-docker-swarm.md b/docs/gettingstarted/ansible-docker-swarm.md
@@ -103,7 +103,6 @@ SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=main for sour
 SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=main for sourcetype=sc4s:events...
 syslog-ng checking config
 sc4s version=v1.36.0
-starting goss
 starting syslog-ng
 ```
 
diff --git a/docs/gettingstarted/ansible-mk8s.md b/docs/gettingstarted/ansible-mk8s.md
@@ -75,6 +75,5 @@ SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=main for sour
 SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=main for sourcetype=sc4s:events...
 syslog-ng checking config
 sc4s version=v1.36.0
-starting goss
 starting syslog-ng
 ```
diff --git a/docs/gettingstarted/byoe-rhel8.md b/docs/gettingstarted/byoe-rhel8.md
@@ -64,28 +64,13 @@ sudo wget -c https://github.com/splunk/splunk-connect-for-syslog/releases/downlo
 sudo pip3 install -r /etc/syslog-ng/requirements.txt
 ```
 
-6. Optionally, to use monitoring, install `goss` and confirm that the version is v0.3.16 or later. `goss` installs in 
-`/usr/local/bin` by default, so do one of the following:
-
-* Make sure that `entrypoint.sh` is modified to include `/usr/local/bin` in the full path.
-* Move the `goss` binary to `/bin` or `/usr/bin`.
-
-```
-curl -L https://github.com/aelsabbahy/goss/releases/latest/download/goss-linux-amd64 -o /usr/local/bin/goss
-chmod +rx /usr/local/bin/goss
-curl -L https://github.com/aelsabbahy/goss/releases/latest/download/dgoss -o /usr/local/bin/dgoss
-# Alternatively, using the latest
-# curl -L https://raw.githubusercontent.com/aelsabbahy/goss/latest/extras/dgoss/dgoss -o /usr/local/bin/dgoss
-chmod +rx /usr/local/bin/dgoss
-```
-
-7. You can run SC4S using systemd in one of two ways, depending on administrator preference and
+6. You can run SC4S using systemd in one of two ways, depending on administrator preference and
 orchestration methodology. These are not the only ways to run in a custom environment:
 
 * Run the `entrypoint.sh` script (identical to that used in the container) directly using systemd.
 * Alter the script to preconfigure SC4S, after which only the syslog-ng are run using systemd. 
 
-8. To run the `entrypoint.sh` script directly in systemd, create the SC4S unit file ``/lib/systemd/system/sc4s.service`` and add the following:
+7. To run the `entrypoint.sh` script directly in systemd, create the SC4S unit file ``/lib/systemd/system/sc4s.service`` and add the following:
 
 ```ini
 [Unit]
@@ -107,7 +92,7 @@ Restart=on-abnormal
 WantedBy=multi-user.target
 ```
 
-9. To run `entrypoint.sh` as a preconfigured script, modify the script by commenting out or removing the stanzas following the
+8. To run `entrypoint.sh` as a preconfigured script, modify the script by commenting out or removing the stanzas following the
 `OPTIONAL for BYOE` comments in the script. This prevents syslog-ng from being launched by the script. Then create the SC4S unit file ``/lib/systemd/system/syslog-ng.service`` and add the following content:
 
 ```ini
@@ -130,7 +115,7 @@ Restart=on-failure
 WantedBy=multi-user.target
 ```
 
-10. Create the file ``/etc/syslog-ng/env_file`` and add the following environment variables. Adjust the URL/TOKEN as needed.
+9. Create the file ``/etc/syslog-ng/env_file`` and add the following environment variables. Adjust the URL/TOKEN as needed.
 
 ```dotenv
 # The following "path" variables can differ from the container defaults specified in the entrypoint.sh script. 
@@ -150,7 +135,7 @@ SC4S_DEST_SPLUNK_HEC_DEFAULT_TOKEN=a778f63a-5dff-4e3c-a72c-a03183659e94
 # SC4S_DEST_SPLUNK_HEC_DEFAULT_TLS_VERIFY=no
 ```
 
-11. Reload systemctl and restart syslog-ng (example here is shown for systemd option (1) above)
+10. Reload systemctl and restart syslog-ng (example here is shown for systemd option (1) above)
 
 ```bash
 sudo systemctl daemon-reload
diff --git a/docs/gettingstarted/docker-compose-MacOS.md b/docs/gettingstarted/docker-compose-MacOS.md
@@ -120,7 +120,6 @@ You should see events similar to those below in the output:
 ```ini
 syslog-ng checking config
 sc4s version=v1.36.0
-starting goss
 starting syslog-ng
 ```
 
diff --git a/docs/gettingstarted/docker-compose.md b/docs/gettingstarted/docker-compose.md
@@ -112,7 +112,6 @@ You should see events similar to those below in the output:
 ```ini
 syslog-ng checking config
 sc4s version=v1.36.0
-starting goss
 starting syslog-ng
 ```
 
diff --git a/docs/gettingstarted/docker-systemd-general.md b/docs/gettingstarted/docker-systemd-general.md
@@ -116,7 +116,6 @@ You should see events similar to those below in the output:
 ```ini
 syslog-ng checking config
 sc4s version=v1.36.0
-starting goss
 starting syslog-ng
 ```
 
diff --git a/docs/gettingstarted/podman-systemd-general.md b/docs/gettingstarted/podman-systemd-general.md
@@ -109,7 +109,6 @@ You should see events similar to those below in the output:
 ```ini
 syslog-ng checking config
 sc4s version=v1.36.0
-starting goss
 starting syslog-ng
 ```
 
@@ -178,8 +177,8 @@ ExecStart=/usr/bin/podman run -p 2514:514 -p 2514:514/udp -p 6514:6514  \
         -v "$SC4S_ARCHIVE_MOUNT" \
         -v "$SC4S_TLS_MOUNT" \
         --env-file=/home/sc4s/env_file \
-        --health-cmd="/healthcheck.sh" \
-        --health-interval=10s --health-retries=6 --health-timeout=6s \
+        --health-cmd="/usr/sbin/syslog-ng-ctl healthcheck --timeout 5" \
+        --health-interval=2m --health-retries=6 --health-timeout=5s \
         --network host \
         --name SC4S \
         --rm $SC4S_IMAGE
diff --git a/docs/resources/podman/sc4s.service b/docs/resources/podman/sc4s.service
@@ -38,8 +38,8 @@ ExecStart=/usr/bin/podman run \
         -v "$SC4S_ARCHIVE_MOUNT" \
         -v "$SC4S_TLS_MOUNT" \
         --env-file=/opt/sc4s/env_file \
-        --health-cmd="/healthcheck.sh" \
-        --health-interval=10s --health-retries=6 --health-timeout=6s \
+        --health-cmd="/usr/sbin/syslog-ng-ctl healthcheck --timeout 5" \
+        --health-interval=2m --health-retries=6 --health-timeout=5s \
         --network host \
         --name SC4S \
         --rm $SC4S_IMAGE
diff --git a/docs/v3_upgrade.md b/docs/v3_upgrade.md
@@ -27,7 +27,6 @@ SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=main for sour
 SC4S_ENV_CHECK_HEC: Splunk HEC connection test successful to index=main for sourcetype=sc4s:events...
 syslog-ng checking config
 sc4s version=3.0.0
-starting goss
 starting syslog-ng 
 ```
 
diff --git a/package/Dockerfile b/package/Dockerfile
@@ -41,7 +41,6 @@ RUN apk add -U --upgrade --no-cache \
       cargo \
       ca-certificates \
       poetry \
-    && curl -fsSL https://goss.rocks/install | GOSS_VER=v0.4.8 sh \
     && groupadd --gid 1024 syslog \
     && useradd -M -g 1024 -u 1024 syslog \
     && usermod -L syslog \
@@ -56,9 +55,7 @@ EXPOSE 6514/tcp
 #/dev/log a low priv user cannot read this and the container will fail in SC4S
 #and other uses the low user may be selected
 
-HEALTHCHECK --interval=10s --retries=6 --timeout=6s CMD /healthcheck.sh
-
-COPY package/etc/goss.yaml /etc/syslog-ng/goss.yaml
+HEALTHCHECK --interval=2m --timeout=5s --start-period=30s CMD /usr/sbin/syslog-ng-ctl healthcheck --timeout 5
 
 COPY pyproject.toml /
 COPY poetry.lock /
@@ -75,7 +72,6 @@ COPY package/etc/test_parsers /etc/syslog-ng/test_parsers
 COPY package/etc/local_config /etc/syslog-ng/local_config
 COPY package/etc/local_config /etc/syslog-ng/local_config
 COPY package/sbin/entrypoint.sh /
-COPY package/sbin/healthcheck.sh /
 COPY package/sbin/source_ports_validator.py /
 
 ENV SC4S_CONTAINER_OPTS=--no-caps
diff --git a/package/Dockerfile.lite b/package/Dockerfile.lite
@@ -41,7 +41,6 @@ RUN apk add -U --upgrade --no-cache \
       cargo \
       ca-certificates \
       poetry \
-    && curl -fsSL https://goss.rocks/install | GOSS_VER=v0.4.8 sh \
     && groupadd --gid 1024 syslog \
     && useradd -M -g 1024 -u 1024 syslog \
     && usermod -L syslog \
@@ -56,9 +55,7 @@ EXPOSE 6514/tcp
 #/dev/log a low priv user cannot read this and the container will fail in SC4S
 #and other uses the low user may be selected
 
-HEALTHCHECK --interval=10s --retries=6 --timeout=6s CMD /healthcheck.sh
-
-COPY package/etc/goss.yaml /etc/syslog-ng/goss.yaml
+HEALTHCHECK --interval=2m --timeout=5s --start-period=30s CMD /usr/sbin/syslog-ng-ctl healthcheck --timeout 5
 
 COPY pyproject.toml /
 COPY poetry.lock /
@@ -97,7 +94,6 @@ COPY package/lite/etc/config.yaml /etc/syslog-ng/config.yaml
 COPY package/lite/etc/addons /etc/syslog-ng/addons
 
 COPY package/sbin/entrypoint.sh /
-COPY package/sbin/healthcheck.sh /
 COPY package/sbin/source_ports_validator.py /
 
 
diff --git a/package/etc/goss.yaml b/package/etc/goss.yaml
diff --git a/package/sbin/entrypoint.sh b/package/sbin/entrypoint.sh
@@ -224,13 +224,6 @@ echo sc4s version=$(cat $SC4S_ETC/VERSION)
 echo sc4s version=$(cat $SC4S_ETC/VERSION) >>$SC4S_VAR/log/syslog-ng.out
 $SC4S_SBIN/syslog-ng --no-caps $SC4S_CONTAINER_OPTS -s >>$SC4S_VAR/log/syslog-ng.out 2>$SC4S_VAR/log/syslog-ng.err
 
-# Use goss to pick up default listening ports for health check
-if command -v goss &> /dev/null
-then
-  echo starting goss
-  goss -g $SC4S_ETC/goss.yaml serve -l 0.0.0.0:$SC4S_LISTEN_STATUS_PORT --format json >/dev/null 2>/dev/null &
-fi
-
 # OPTIONAL for BYOE:  Comment out/remove all remaining lines and launch syslog-ng directly from systemd
 if [ "${SC4S_DEBUG_CONTAINER}" == "yes" ]
 then
diff --git a/package/sbin/healthcheck.sh b/package/sbin/healthcheck.sh
