Skip to content

Commit 83a9960

Browse files
0fatal0fatal
authored
feat: replace current logging solution with log server (#1381)
* feat: add log server * feat(runtimes&server): adapt to log server * chore(build): add helm template for log-server * feat(log-server): health check * refactor(server): fetch function logs from log server * chore(docs): tidy docs for function log * chore: clean code * chore(ci): dockerize log server * fix(server): replace db in FunctionConsole.write * feat: verify appid for log server * chore(runtimes): restart app when file changes * chore: adjust env and region conf for supporting log server * refactor(server): get logs from log server
1 parent 1c4a760 commit 83a9960

40 files changed

+3204
-137
lines changed

‎.github/workflows/dockerize-log-server.yml

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: dockerize-log-server
2+
3+
on:
4+
workflow_dispatch:
5+
push:
6+
branches:
7+
- main
8+
paths:
9+
- "services/log-server/**"
10+
- ".github/workflows/dockerize-log-server.yml"
11+
- "!**/*.md"
12+
- "!services/log-server/package-lock.json"
13+
14+
concurrency:
15+
group: dockerize-log-server-${{ github.ref }}
16+
cancel-in-progress: true
17+
18+
jobs:
19+
dockerize-log-server:
20+
runs-on: ubuntu-latest
21+
22+
steps:
23+
- uses: actions/checkout@v3
24+
with:
25+
fetch-depth: 0
26+
27+
- name: Docker meta
28+
id: meta
29+
uses: docker/metadata-action@v4
30+
with:
31+
images: |
32+
ghcr.io/${{ github.repository_owner }}/log-server
33+
docker.io/${{ secrets.DOCKER_USERNAME }}/log-server
34+
# https://github.com/docker/metadata-action#typesemver
35+
tags: |
36+
type=raw,value=latest,enable=true
37+
type=sha,enable=true,format=short
38+
39+
- name: Set up QEMU
40+
uses: docker/setup-qemu-action@v2
41+
42+
- name: Set up Docker Buildx
43+
uses: docker/setup-buildx-action@v2
44+
45+
- name: Login to DockerHub
46+
uses: docker/login-action@v2
47+
with:
48+
username: ${{ secrets.DOCKER_USERNAME }}
49+
password: ${{ secrets.DOCKER_PASSWORD }}
50+
51+
- name: Login to Github Container Hub
52+
uses: docker/login-action@v2
53+
with:
54+
registry: ghcr.io
55+
username: ${{ github.repository_owner }}
56+
password: ${{ secrets.GITHUB_TOKEN }}
57+
58+
- name: Build and push
59+
uses: docker/build-push-action@v3
60+
with:
61+
context: ./services/log-server
62+
file: ./services/log-server/Dockerfile
63+
push: true
64+
tags: ${{ steps.meta.outputs.tags }}
65+
labels: ${{ steps.meta.outputs.labels }}
66+
platforms: linux/amd64, linux/arm64
67+
68+
trigger-workflow-build-cluster-image:
69+
needs: [dockerize-log-server]
70+
runs-on: ubuntu-latest
71+
steps:
72+
- name: trigger cluster image workflow
73+
uses: peter-evans/repository-dispatch@v2
74+
with:
75+
event-type: docker_build_success
76+
client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}", "version": "latest"}'

‎.prettierignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
charts

‎build/charts/laf-server/templates/_helpers.tpl

+20
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,18 @@ app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
4242
app.kubernetes.io/managed-by: {{ .Release.Service }}
4343
{{- end }}
4444

45+
{{/*
46+
Common labels
47+
*/}}
48+
{{- define "log-server.labels" -}}
49+
helm.sh/chart: {{ include "laf-server.chart" . }}
50+
{{ include "log-server.selectorLabels" . }}
51+
{{- if .Chart.AppVersion }}
52+
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
53+
{{- end }}
54+
app.kubernetes.io/managed-by: {{ .Release.Service }}
55+
{{- end }}
56+
4557
{{/*
4658
Selector labels
4759
*/}}
@@ -50,6 +62,14 @@ app.kubernetes.io/name: {{ include "laf-server.name" . }}
5062
app.kubernetes.io/instance: {{ .Release.Name }}
5163
{{- end }}
5264

65+
{{/*
66+
Selector labels
67+
*/}}
68+
{{- define "log-server.selectorLabels" -}}
69+
app.kubernetes.io/name: log-server
70+
app.kubernetes.io/instance: {{ .Release.Name }}
71+
{{- end }}
72+
5373
{{/*
5474
Create the name of the service account to use
5575
*/}}

‎build/charts/laf-server/templates/deployment.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,12 @@ spec:
8383
value: {{ .Values.default_region.apisix_api_url }}
8484
- name: DEFAULT_REGION_APISIX_API_KEY
8585
value: {{ .Values.default_region.apisix_api_key }}
86+
- name: DEFAULT_REGION_LOG_SERVER_URL
87+
value: {{ .Values.default_region.log_server_url }}
88+
- name: DEFAULT_REGION_LOG_SERVER_SECRET
89+
value: {{ .Values.default_region.log_server_secret }}
90+
- name: DEFAULT_REGION_LOG_SERVER_DATABASE_URL
91+
value: {{ .Values.default_region.log_server_database_url }}
8692
- name: SITE_NAME
8793
value: {{ .Values.siteName | quote}}
8894
{{- with .Values.nodeSelector }}

‎build/charts/laf-server/templates/log-server.yaml

+52
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
apiVersion: apps/v1
2+
kind: Deployment
3+
metadata:
4+
labels:
5+
{{- include "log-server.labels" . | nindent 4 }}
6+
name: log-server
7+
spec:
8+
replicas: 1
9+
selector:
10+
matchLabels:
11+
{{- include "log-server.selectorLabels" . | nindent 6 }}
12+
template:
13+
metadata:
14+
labels:
15+
{{- include "log-server.selectorLabels" . | nindent 8 }}
16+
spec:
17+
containers:
18+
- image: docker.io/lafyun/log-server:latest
19+
imagePullPolicy: Always
20+
name: log-server
21+
ports:
22+
- name: http
23+
containerPort: 5060
24+
protocol: TCP
25+
livenessProbe:
26+
httpGet:
27+
path: /healthz
28+
port: http
29+
readinessProbe:
30+
httpGet:
31+
path: /healthz
32+
port: http
33+
env:
34+
- name: DB_URI
35+
value: {{ .Values.default_region.log_server_database_url | quote }}
36+
- name: JWT_SECRET
37+
value: {{ .Values.default_region.log_server_secret | quote }}
38+
---
39+
apiVersion: v1
40+
kind: Service
41+
metadata:
42+
labels:
43+
{{- include "log-server.labels" . | nindent 4 }}
44+
name: log-server
45+
spec:
46+
ports:
47+
- name: http
48+
port: 5060
49+
protocol: TCP
50+
targetPort: http
51+
selector:
52+
{{- include "log-server.selectorLabels" . | nindent 4 }}

‎build/charts/laf-server/values.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ default_region:
2424
tls: false
2525
runtime_domain: ""
2626
website_domain: ""
27+
# log-server
28+
log_server_url: ""
29+
log_server_secret: ""
30+
log_server_database_url: ""
2731
jwt:
2832
secret: laf_server_abc123
2933
expires_in: 7d

‎build/start.sh

+6
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ helm install minio -n ${NAMESPACE} \
7171

7272
## 4. install laf-server
7373
SERVER_JWT_SECRET=$PASSWD_OR_SECRET
74+
LOG_SERVER_URL="http://log-server.${NAMESPACE}.svc.cluster.local:5060"
75+
LOG_SERVER_DATABASE_URL="mongodb://${DB_USERNAME:-admin}:${PASSWD_OR_SECRET}@mongodb-0.mongo.${NAMESPACE}.svc.cluster.local:27017/function-logs?authSource=admin&replicaSet=rs0&w=majority"
76+
LOG_SERVER_SECRET=$PASSWD_OR_SECRET
7477
helm install server -n ${NAMESPACE} \
7578
--set databaseUrl=${DATABASE_URL} \
7679
--set meteringDatabaseUrl=${METERING_DATABASE_URL} \
@@ -90,6 +93,9 @@ helm install server -n ${NAMESPACE} \
9093
--set default_region.apisix_api_url=${APISIX_API_URL} \
9194
--set default_region.apisix_api_key=${APISIX_API_KEY} \
9295
--set default_region.apisix_public_port=80 \
96+
--set default_region.log_server_url=${LOG_SERVER_URL} \
97+
--set default_region.log_server_secret=${LOG_SERVER_SECRET} \
98+
--set default_region.log_server_database_url=${LOG_SERVER_DATABASE_URL} \
9399
./charts/laf-server
94100

95101
## 6. install laf-web

‎docs/en/guide/function/logs.md

-22
Original file line numberDiff line numberDiff line change
@@ -17,25 +17,3 @@ You can filter logs based on the `requestId` and the name of the cloud function.
1717
1. Click on a log to switch to the log section.
1818
2. Use the `requestId` and the name of the cloud function to search for specific logs.
1919
3. Click on an individual log to view detailed content.
20-
21-
## Manual Log Cleaning
22-
23-
The running logs of Laf cloud functions are stored in a hidden collection called `__function_logs__`. Therefore, we can use the method to manipulate the database in cloud functions to clean the logs.
24-
25-
The following is the code to clean all logs in a cloud function:
26-
27-
::: danger
28-
The following operation will delete all historical logs. Please proceed with caution.
29-
:::
30-
31-
```typescript
32-
import cloud from '@lafjs/cloud'
33-
34-
export async function main(ctx: FunctionContext) {
35-
console.log('Hello World')
36-
// Database, remove all logs
37-
const db = cloud.database();
38-
const res = await db.collection('__function_logs__').remove({multi:true})
39-
console.log(res)
40-
}
41-
```

‎docs/guide/function/logs.md

-24
Original file line numberDiff line numberDiff line change
@@ -19,27 +19,3 @@ title: 云函数历史日志
1919
2、可根据 `requestId` 和云函数名搜索指定日志
2020

2121
3、点击单个日志，可查看详细内容
22-
23-
## 手动清理日志
24-
25-
Laf 云函数的运行日志都在一个隐藏的集合中：`__function_logs__`
26-
27-
所以我们可以通过云函数操作数据库的方法，清理日志
28-
29-
以下是清理全部日志的云函数写法：
30-
31-
::: danger
32-
以下操作会删除全部历史日志，请谨慎操作
33-
:::
34-
35-
```typescript
36-
import cloud from '@lafjs/cloud'
37-
38-
export async function main(ctx: FunctionContext) {
39-
console.log('Hello World')
40-
// 数据库，删除全部日志
41-
const db = cloud.database();
42-
const res = await db.collection('__function_logs__').remove({multi:true})
43-
console.log(res)
44-
}
45-
```

‎lerna.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,8 @@
55
"./server",
66
"./web",
77
"./runtimes/nodejs",
8-
"./cli"
8+
"./cli",
9+
"./services/*"
910
],
1011
"version": "1.0.0-beta.10",
1112
"command": {

‎runtimes/nodejs/package.json

+5-3
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,10 @@
66
"typings": "./dist/index.d.ts",
77
"private": "true",
88
"scripts": {
9-
"start": "node --expose-internals ./dist/index.js",
10-
"dev": "NODE_ENV=development npm run start",
9+
"start": "node ./dist/index.js",
10+
"dev": "npx concurrently npm:dev:*",
11+
"dev:start": "npx nodemon ./dist/index.js",
12+
"dev:watch": "npm run watch",
1113
"build": "tsc -p tsconfig.json",
1214
"watch": "tsc -p tsconfig.json -w",
1315
"prepublishOnly": "npm run build",
@@ -73,4 +75,4 @@
7375
],
7476
"delay": 1000
7577
}
76-
}
78+
}

‎runtimes/nodejs/src/config.ts

+8-7
Original file line numberDiff line numberDiff line change
@@ -57,13 +57,6 @@ export default class Config {
5757
return process.env.NODE_ENV === 'production'
5858
}
5959

60-
/**
61-
* Expired time of function logs, in seconds
62-
*/
63-
static get FUNCTION_LOG_EXPIRED_TIME(): number {
64-
return (process.env.FUNCTION_LOG_EXPIRED_TIME ?? 3600 * 24 * 3) as number
65-
}
66-
6760
static get RUNTIME_IMAGE(): string {
6861
return process.env.RUNTIME_IMAGE
6962
}
@@ -83,4 +76,12 @@ export default class Config {
8376
static get REQUEST_LIMIT_SIZE(): string {
8477
return process.env.REQUEST_LIMIT_SIZE || '10mb'
8578
}
79+
80+
static get LOG_SERVER_URL(): string {
81+
return process.env.LOG_SERVER_URL || ''
82+
}
83+
84+
static get LOG_SERVER_TOKEN(): string {
85+
return process.env.LOG_SERVER_TOKEN || ''
86+
}
8687
}

‎runtimes/nodejs/src/constants.ts

-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
*/
44
export const CLOUD_FUNCTION_COLLECTION = '__functions__'
55
export const POLICY_COLLECTION = '__policies__'
6-
export const FUNCTION_LOG_COLLECTION = '__function_logs__'
76
export const CONFIG_COLLECTION = '__conf__'
87

98
export const WEBSOCKET_FUNCTION_NAME = '__websocket__'

‎runtimes/nodejs/src/index.ts

-2
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,10 @@ import xmlparser from 'express-xml-bodyparser'
2121
import './support/cloud-sdk'
2222
import { FunctionCache } from './support/function-engine/cache'
2323
import { DatabaseChangeStream } from './support/db-change-stream'
24-
import { ensureCollectionIndexes } from './support/function-log'
2524

2625
const app = express()
2726

2827
DatabaseAgent.accessor.ready.then(() => {
29-
ensureCollectionIndexes()
3028
FunctionCache.initialize()
3129
DatabaseChangeStream.initialize()
3230
})

‎runtimes/nodejs/src/support/function-engine/console.ts

+21-1
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,30 @@
11
import * as util from 'util'
22
import { FunctionContext } from './types'
3+
import Config from '../../config'
4+
import axios from 'axios'
35

46
export class FunctionConsole {
57
ctx: FunctionContext
68

7-
static write: (message: string, ctx: FunctionContext) => void = console.log
9+
static write(message: string, ctx: FunctionContext) {
10+
if (!Config.LOG_SERVER_URL || !Config.LOG_SERVER_TOKEN) return
11+
12+
const doc = {
13+
request_id: ctx.requestId,
14+
func: ctx.__function_name,
15+
data: message,
16+
created_at: new Date(),
17+
}
18+
19+
axios.post(`${Config.LOG_SERVER_URL}/function/log`, {
20+
appid: Config.APPID,
21+
log: doc,
22+
}, {
23+
headers: {
24+
'x-token': Config.LOG_SERVER_TOKEN
25+
}
26+
})
27+
}
828

929
constructor(ctx: FunctionContext) {
1030
this.ctx = ctx

0 commit comments

Comments
 (0)