Skip to content

Commit ef39f9a

Browse files
committed
feature: run a clean-up job when uninstalling
Signed-off-by: tao.yang <[email protected]>
1 parent dc8e9af commit ef39f9a

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+2722
-161
lines changed

.github/workflows/e2e-init.yaml

+1-1
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,7 @@ jobs:
210210
path: e2ereport.json
211211
retention-days: 1
212212

213-
- name: helm uninstalls spiderpool and cleans spiderpool CRD
213+
- name: uninstalls spiderpool
214214
id: clean
215215
if: ${{ inputs.run_e2e == 'true' }}
216216
run: |

charts/spiderpool/README.md

+1
Original file line numberDiff line numberDiff line change
@@ -370,6 +370,7 @@ helm install spiderpool spiderpool/spiderpool --wait --namespace kube-system \
370370
| `spiderpoolController.tls.auto.certExpiration` | server cert expiration for auto method | `73000` |
371371
| `spiderpoolController.tls.auto.extraIpAddresses` | extra IP addresses of server certificate for auto method | `[]` |
372372
| `spiderpoolController.tls.auto.extraDnsNames` | extra DNS names of server cert for auto method | `[]` |
373+
| `spiderpoolController.cleanup.enable` | clean up resources when helm uninstall | `true` |
373374

374375
### spiderpoolInit parameters
375376

Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
{{- if .Values.spiderpoolController.cleanup.enable }}
2+
apiVersion: batch/v1
3+
kind: Job
4+
metadata:
5+
name: {{ .Values.spiderpoolController.name | trunc 48 | trimSuffix "-" }}-hook-pre-delete
6+
annotations:
7+
"helm.sh/hook": pre-delete
8+
"helm.sh/hook-delete-policy": hook-succeeded,hook-failed
9+
spec:
10+
template:
11+
spec:
12+
serviceAccountName: {{ .Values.spiderpoolController.name | trunc 63 | trimSuffix "-" }}
13+
containers:
14+
- name: pre-delete
15+
image: {{ include "spiderpool.spiderpoolController.image" . | quote }}
16+
command:
17+
- {{ .Values.spiderpoolController.binName }}
18+
- clean
19+
- --validate
20+
- {{ .Values.spiderpoolController.name | trunc 63 | trimSuffix "-" }}
21+
- --mutating
22+
- {{ .Values.spiderpoolController.name | trunc 63 | trimSuffix "-" }}
23+
env:
24+
- name: SPIDERPOOL_POD_NAME
25+
valueFrom:
26+
fieldRef:
27+
fieldPath: metadata.name
28+
- name: SPIDERPOOL_POD_NAMESPACE
29+
valueFrom:
30+
fieldRef:
31+
fieldPath: metadata.namespace
32+
restartPolicy: Never
33+
backoffLimit: 2
34+
{{- end }}

charts/spiderpool/templates/role.yaml

+25
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,31 @@ rules:
4141
- get
4242
- list
4343
- watch
44+
- apiGroups:
45+
- admissionregistration.k8s.io
46+
resources:
47+
- mutatingwebhookconfigurations
48+
- validatingwebhookconfigurations
49+
verbs:
50+
- create
51+
- delete
52+
- get
53+
- list
54+
- patch
55+
- update
56+
- watch
57+
- apiGroups:
58+
- apiextensions.k8s.io
59+
resources:
60+
- customresourcedefinitions
61+
verbs:
62+
- create
63+
- delete
64+
- get
65+
- list
66+
- patch
67+
- update
68+
- watch
4469
- apiGroups:
4570
- apps
4671
resources:

charts/spiderpool/values.yaml

+4
Original file line numberDiff line numberDiff line change
@@ -757,6 +757,10 @@ spiderpoolController:
757757
## @param spiderpoolController.tls.auto.extraDnsNames extra DNS names of server cert for auto method
758758
extraDnsNames: []
759759

760+
cleanup:
761+
## @param spiderpoolController.cleanup.enable clean up resources when helm uninstall
762+
enable: true
763+
760764
## @section spiderpoolInit parameters
761765
##
762766
spiderpoolInit:
+221
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,221 @@
1+
// Copyright 2022 Authors of spidernet-io
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package cmd
5+
6+
import (
7+
"context"
8+
"os"
9+
10+
"github.com/hashicorp/go-multierror"
11+
"github.com/spf13/cobra"
12+
"github.com/spidernet-io/spiderpool/pkg/constant"
13+
spiderpoolv2beta1 "github.com/spidernet-io/spiderpool/pkg/k8s/apis/spiderpool.spidernet.io/v2beta1"
14+
webhook "k8s.io/api/admissionregistration/v1"
15+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
16+
ctrl "sigs.k8s.io/controller-runtime"
17+
"sigs.k8s.io/controller-runtime/pkg/client"
18+
)
19+
20+
var cleanCmd = &cobra.Command{
21+
Use: "clean",
22+
Short: "Clean resources",
23+
Long: "Clean resources with specified parameters.",
24+
Run: func(cmd *cobra.Command, args []string) {
25+
26+
validate, err := cmd.Flags().GetString("validate")
27+
if err != nil {
28+
logger.Fatal(err.Error())
29+
os.Exit(1)
30+
}
31+
mutating, err := cmd.Flags().GetString("mutating")
32+
if err != nil {
33+
logger.Fatal(err.Error())
34+
os.Exit(1)
35+
}
36+
logger.Sugar().Infof("validate %s\nmutating %s\n", validate, mutating)
37+
38+
client, err := NewCoreClient()
39+
if err != nil {
40+
logger.Fatal(err.Error())
41+
os.Exit(1)
42+
}
43+
err = client.clean(validate, mutating)
44+
if err != nil {
45+
logger.Fatal(err.Error())
46+
os.Exit(1)
47+
}
48+
},
49+
}
50+
51+
type CoreClient struct {
52+
client.Client
53+
}
54+
55+
func NewCoreClient() (*CoreClient, error) {
56+
c, err := client.New(
57+
ctrl.GetConfigOrDie(),
58+
client.Options{Scheme: scheme},
59+
)
60+
if err != nil {
61+
return nil, err
62+
}
63+
64+
return &CoreClient{Client: c}, nil
65+
}
66+
67+
func (c *CoreClient) clean(validate, mutating string) error {
68+
var jobResult *multierror.Error
69+
ctx := context.Background()
70+
vObj := &webhook.ValidatingWebhookConfiguration{}
71+
err := c.Get(ctx, client.ObjectKey{Name: validate}, vObj)
72+
if err == nil {
73+
err := c.Delete(ctx, vObj)
74+
if err != nil {
75+
logger.Sugar().Errorf("failed to delete ValidatingWebhookConfiguration: %v.", err)
76+
jobResult = multierror.Append(jobResult, err)
77+
}
78+
logger.Sugar().Infof("succeeded to delete ValidatingWebhookConfiguration")
79+
}
80+
81+
mObj := &webhook.MutatingWebhookConfiguration{}
82+
err = c.Get(ctx, client.ObjectKey{Name: mutating}, mObj)
83+
if err == nil {
84+
err := c.Delete(ctx, mObj)
85+
if err != nil {
86+
logger.Sugar().Errorf("failed to delete MutatingWebhookConfiguration: %v.", err)
87+
jobResult = multierror.Append(jobResult, err)
88+
}
89+
logger.Sugar().Infof("succeeded to delete MutatingWebhookConfiguration")
90+
}
91+
92+
ipPoolList := new(spiderpoolv2beta1.SpiderIPPoolList)
93+
err = c.List(ctx, ipPoolList)
94+
if err == nil {
95+
for _, item := range ipPoolList.Items {
96+
item.Finalizers = make([]string, 0)
97+
err := c.Update(ctx, &item)
98+
if err != nil {
99+
logger.Sugar().Errorf("failed to clean the finalizers of ippool: %v, %v", &item.Name, err)
100+
jobResult = multierror.Append(jobResult, err)
101+
continue
102+
}
103+
logger.Sugar().Infof("succeeded to clean the finalizers of ippool %v", &item.Name)
104+
105+
err = c.Delete(ctx, &item)
106+
if err != nil {
107+
logger.Sugar().Errorf("failed to delete ippool: %v, %v ", &item.Name, err)
108+
jobResult = multierror.Append(jobResult, err)
109+
continue
110+
}
111+
logger.Sugar().Infof("succeeded to delete ippool: %v", &item.Name)
112+
}
113+
}
114+
115+
subnetList := new(spiderpoolv2beta1.SpiderSubnetList)
116+
err = c.List(ctx, subnetList)
117+
if err == nil {
118+
for _, item := range subnetList.Items {
119+
err = c.Delete(ctx, &item)
120+
if err != nil {
121+
logger.Sugar().Errorf("failed to delete subnet: %v, %v ", &item.Name, err)
122+
jobResult = multierror.Append(jobResult, err)
123+
continue
124+
}
125+
logger.Sugar().Infof("succeeded to delete subnet: %v", &item.Name)
126+
}
127+
}
128+
129+
spiderEndpointList := new(spiderpoolv2beta1.SpiderEndpointList)
130+
err = c.List(ctx, spiderEndpointList)
131+
if err == nil {
132+
for _, item := range spiderEndpointList.Items {
133+
item.Finalizers = make([]string, 0)
134+
err := c.Update(ctx, &item)
135+
if err != nil {
136+
logger.Sugar().Errorf("failed to clean the finalizers of spiderEndpoint: %v, %v ", &item.Name, err)
137+
jobResult = multierror.Append(jobResult, err)
138+
continue
139+
}
140+
logger.Sugar().Infof("succeeded to clean the finalizers of spiderEndpoint %v", &item.Name)
141+
142+
err = c.Delete(ctx, &item)
143+
if err != nil {
144+
logger.Sugar().Errorf("failed to delete spiderEndpoint: %v, %v ", &item.Name, err)
145+
jobResult = multierror.Append(jobResult, err)
146+
continue
147+
}
148+
logger.Sugar().Infof("succeeded to delete spiderEndpoint: %v", &item.Name)
149+
}
150+
}
151+
152+
reservedIPList := new(spiderpoolv2beta1.SpiderReservedIPList)
153+
err = c.List(ctx, reservedIPList)
154+
if err == nil {
155+
for _, item := range reservedIPList.Items {
156+
err = c.Delete(ctx, &item)
157+
if err != nil {
158+
logger.Sugar().Errorf("failed to delete spiderReservedIP: %v, %v ", &item.Name, err)
159+
jobResult = multierror.Append(jobResult, err)
160+
continue
161+
}
162+
logger.Sugar().Infof("succeeded to delete spiderReservedIP: %v", &item.Name)
163+
}
164+
}
165+
166+
spiderMultusConfigList := new(spiderpoolv2beta1.SpiderMultusConfigList)
167+
err = c.List(ctx, spiderMultusConfigList)
168+
if err == nil {
169+
for _, item := range spiderMultusConfigList.Items {
170+
err = c.Delete(ctx, &item)
171+
if err != nil {
172+
logger.Sugar().Errorf("failed to delete spiderMultusConfig: %v, %v ", &item.Name, err)
173+
jobResult = multierror.Append(jobResult, err)
174+
continue
175+
}
176+
logger.Sugar().Infof("succeeded to delete spiderMultusConfig: %v", &item.Name)
177+
}
178+
}
179+
180+
spiderCoordinatorList := new(spiderpoolv2beta1.SpiderCoordinatorList)
181+
err = c.List(ctx, spiderCoordinatorList)
182+
if err == nil {
183+
for _, item := range spiderCoordinatorList.Items {
184+
item.Finalizers = make([]string, 0)
185+
err := c.Update(ctx, &item)
186+
if err != nil {
187+
logger.Sugar().Errorf("failed to clean the finalizers of spiderCoordinator: %v, %v ", &item.Name, err)
188+
jobResult = multierror.Append(jobResult, err)
189+
continue
190+
}
191+
logger.Sugar().Infof("succeeded to clean the finalizers of spiderCoordinator %v", &item.Name)
192+
193+
err = c.Delete(ctx, &item)
194+
if err != nil {
195+
logger.Sugar().Errorf("failed to delete spiderCoordinator: %v, %v ", &item.Name, err)
196+
jobResult = multierror.Append(jobResult, err)
197+
continue
198+
}
199+
logger.Sugar().Infof("succeeded to delete spiderCoordinator: %v", &item.Name)
200+
}
201+
}
202+
203+
// Delete all crds of spiderpool
204+
customResourceDefinitionList := new(apiextensionsv1.CustomResourceDefinitionList)
205+
err = c.List(ctx, customResourceDefinitionList)
206+
if err == nil {
207+
for _, item := range customResourceDefinitionList.Items {
208+
if item.Spec.Group == constant.SpiderpoolAPIGroup {
209+
err = c.Delete(ctx, &item)
210+
if err != nil {
211+
logger.Sugar().Errorf("failed to delete customResourceDefinitionList: %v, %v ", &item.Name, err)
212+
jobResult = multierror.Append(jobResult, err)
213+
continue
214+
}
215+
logger.Sugar().Infof("succeeded to delete customResourceDefinitionList: %v", &item.Name)
216+
}
217+
}
218+
}
219+
220+
return jobResult.ErrorOrNil()
221+
}

cmd/spiderpool-controller/cmd/command_root.go

+4
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,10 @@ var rootCmd = &cobra.Command{
2424
// Execute adds all child commands to the root command sets flags appropriately.
2525
// This is called by main.main(). It only needs to happen once to the rootCmd.
2626
func Execute() {
27+
cleanCmd.Flags().String("validate", "", "Specify validate parameter")
28+
cleanCmd.Flags().String("mutating", "", "Specify mutating parameter")
29+
30+
rootCmd.AddCommand(cleanCmd)
2731
if err := rootCmd.Execute(); err != nil {
2832
logger.Fatal(err.Error())
2933
}

docs/mkdocs.yml

+1
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ nav:
7272
- VWware vSphere: usage/install/cloud/get-started-vmware.md
7373
- OpenStack: usage/install/cloud/get-started-openstack.md
7474
- Upgrading: usage/install/upgrade.md
75+
- Uninstalling: usage/install/uninstall.md
7576
- System requirements: usage/install/system-requirements.md
7677
- Concepts:
7778
- Architecture: concepts/arch.md

docs/usage/install/uninstall-zh_CN.md

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# 卸载指南
2+
3+
[**English**](./uninstall.md) | **简体中文**
4+
5+
本卸载指南适用于在 Kubernetes 上运行的 Spiderpool。如果您有任何疑问,请随时通过 [Spiderpool Community](../../README-zh_CN.md#_6) 联系我们。
6+
7+
## 注意事项
8+
9+
- 在执行卸载之前,请阅读完整的卸载指南以了解所有必要的步骤。
10+
11+
## 卸载
12+
13+
了解正在运行应用,理解卸载 Spiderpool 可能对其他相关组件(如中间件) 产生的影响,请确保完全了解风险后,才开始执行卸载步骤。
14+
15+
1. 通过 `helm ls` 查询集群所安装的 Spiderpool
16+
17+
```bash
18+
helm ls -A | grep -i spiderpool
19+
```
20+
21+
2. 通过 `helm uninstall` 卸载 Spiderpool
22+
23+
```bash
24+
helm uninstall <spiderpool-name> --namespace <spiderpool-namespace>
25+
```
26+
27+
`<spiderpool-name>` 替换为要卸载的 Spiderpool 的名称,将 `<spiderpool-namespace>` 替换为 Spiderpool 所在的命名空间。
28+
29+
### v0.10.0 以上版本
30+
31+
在 v0.10.0 之后引入了自动清理 Spiderpool 资源的功能,它通过 `spiderpoolController.cleanup.enabled` 配置项来启用,该值默认为 `true`,您可以通过如下方式验证与 Spiderpool 相关的资源数量是否自动被清理。
32+
33+
```bash
34+
kubectl get spidersubnets.spiderpool.spidernet.io -o name | wc -l
35+
kubectl get spiderips.spiderpool.spidernet.io -o name | wc -l
36+
kubectl get spiderippools.spiderpool.spidernet.io -o name | wc -l
37+
kubectl get spiderreservedips.spiderpool.spidernet.io -o name | wc -l
38+
kubectl get spiderendpoints.spiderpool.spidernet.io -o name | wc -l
39+
kubectl get spidercoordinators.spiderpool.spidernet.io -o name | wc -l
40+
```
41+
42+
### v0.10.0 以下版本
43+
44+
在低于 v0.10.0 的版本中,由于 Spiderpool 的某些 CR 资源中存在 [finalizers](https://kubernetes.io/docs/concepts/overview/working-with-objects/finalizers/) ,导致 `helm uninstall` 命令无法清理干净,您需要手动清理。可获取如下清理脚本来完成清理,以确保下次部署 Spiderpool 时不会出现意外错误。
45+
46+
```bash
47+
wget https://raw.githubusercontent.com/spidernet-io/spiderpool/main/tools/scripts/cleanCRD.sh
48+
chmod +x cleanCRD.sh && ./cleanCRD.sh
49+
```
50+

0 commit comments

Comments
 (0)