Skip to content

Commit cebb288

Browse files
haoqing0110haoqing0110
committed
helm chart for kueue addon
Signed-off-by: Qing Hao <[email protected]>
1 parent 233b4bf commit cebb288

File tree

6 files changed

+286
-0
lines changed

6 files changed

+286
-0
lines changed

‎kueue-addon/charts/kueue-addon/Chart.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
apiVersion: v2
2+
name: kueue-addon
3+
description: A Helm chart for Open Cluster Management Kueue Addon
4+
type: application
5+
version: 0.1.0
6+
appVersion: "1.0.0"

‎kueue-addon/charts/kueue-addon/templates/addon-template-hub.yaml

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
apiVersion: addon.open-cluster-management.io/v1alpha1
2+
kind: AddOnTemplate
3+
metadata:
4+
name: kueue-addon-hub-{{ .Chart.Version }}
5+
spec:
6+
addonName: kueue-addon
7+
agentSpec:
8+
workload:
9+
manifests:
10+
- apiVersion: v1
11+
kind: Namespace
12+
metadata:
13+
annotations:
14+
addon.open-cluster-management.io/deletion-orphan: ""
15+
labels:
16+
addon.open-cluster-management.io/namespace: "true"
17+
name: open-cluster-management-agent-addon
18+
- kind: ClusterRole
19+
apiVersion: rbac.authorization.k8s.io/v1
20+
metadata:
21+
name: open-cluster-management:kueue-addon:controller
22+
rules:
23+
- apiGroups: [ "" ]
24+
resources: [ "nodes","configmaps", "pods", "events", "secrets"]
25+
verbs: [ "get", "list", "watch", "create", "update", "delete", "deletecollection", "patch" ]
26+
- apiGroups: [ "coordination.k8s.io" ]
27+
resources: [ "leases" ]
28+
verbs: [ "create", "get", "list", "update", "watch", "patch" ]
29+
# Allow hub to managedclusters, placements, placementdecisions
30+
- apiGroups: ["cluster.open-cluster-management.io"]
31+
resources: ["managedclusters", "placements", "placementdecisions"]
32+
verbs: ["get", "list", "watch"]
33+
# Allow hub to clusterpermission
34+
- apiGroups: ["rbac.open-cluster-management.io"]
35+
resources: ["clusterpermissions"]
36+
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
37+
# Allow hub to managedserviceaccount
38+
- apiGroups: ["authentication.open-cluster-management.io"]
39+
resources: ["managedserviceaccounts"]
40+
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
41+
# Allow hub to manage multikueueconfig
42+
- apiGroups: ["kueue.x-k8s.io"]
43+
resources: ["multikueueconfigs"]
44+
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
45+
# Allow hub to manage multikueueclusters
46+
- apiGroups: ["kueue.x-k8s.io"]
47+
resources: ["multikueueclusters"]
48+
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
49+
# Allow hub to manage admissionchecks
50+
- apiGroups: ["kueue.x-k8s.io"]
51+
resources: ["admissionchecks"]
52+
verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
53+
- apiGroups: ["kueue.x-k8s.io"]
54+
resources: ["admissionchecks/status"]
55+
verbs: ["update", "patch"]
56+
- kind: ClusterRoleBinding
57+
apiVersion: rbac.authorization.k8s.io/v1
58+
metadata:
59+
name: open-cluster-management:kueue-addon:controller
60+
roleRef:
61+
apiGroup: rbac.authorization.k8s.io
62+
kind: ClusterRole
63+
name: open-cluster-management:kueue-addon:controller
64+
subjects:
65+
- kind: ServiceAccount
66+
name: kueue-addon-controller-sa
67+
namespace: open-cluster-management-agent-addon
68+
- kind: Deployment
69+
apiVersion: apps/v1
70+
metadata:
71+
name: kueue-addon-controller
72+
namespace: open-cluster-management-agent-addon
73+
labels:
74+
app: kueue-addon-controller
75+
spec:
76+
replicas: 1
77+
selector:
78+
matchLabels:
79+
app: kueue-addon-controller
80+
template:
81+
metadata:
82+
labels:
83+
app: kueue-addon-controller
84+
spec:
85+
serviceAccount: kueue-addon-controller-sa
86+
containers:
87+
- name: kueue-addon-controller
88+
image: {{ .Values.image }}:{{ .Values.tag | default (print "v" .Chart.Version) }}
89+
imagePullPolicy: Always
90+
args:
91+
- "/kueue-addon-controller"
92+
- "hub"
93+
- kind: ServiceAccount
94+
apiVersion: v1
95+
metadata:
96+
name: kueue-addon-controller-sa
97+
namespace: open-cluster-management-agent-addon

‎kueue-addon/charts/kueue-addon/templates/addon-template-spoke.yaml

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
apiVersion: addon.open-cluster-management.io/v1alpha1
2+
kind: AddOnTemplate
3+
metadata:
4+
name: kueue-addon-spoke-{{ .Chart.Version }}
5+
spec:
6+
addonName: kueue-addon
7+
agentSpec:
8+
workload:
9+
manifests:
10+
- apiVersion: rbac.authorization.k8s.io/v1
11+
kind: ClusterRoleBinding
12+
metadata:
13+
name: kueue-resourceflavor-ocm-rolebinding
14+
roleRef:
15+
apiGroup: rbac.authorization.k8s.io
16+
kind: ClusterRole
17+
name: kueue-resourceflavor-editor-role
18+
subjects:
19+
- kind: ServiceAccount
20+
name: klusterlet-work-sa
21+
namespace: open-cluster-management-agent
22+
- apiVersion: rbac.authorization.k8s.io/v1
23+
kind: ClusterRoleBinding
24+
metadata:
25+
name: kueue-clusterqueue-ocm-rolebinding
26+
roleRef:
27+
apiGroup: rbac.authorization.k8s.io
28+
kind: ClusterRole
29+
name: kueue-clusterqueue-editor-role
30+
subjects:
31+
- kind: ServiceAccount
32+
name: klusterlet-work-sa
33+
namespace: open-cluster-management-agent
34+
- apiVersion: rbac.authorization.k8s.io/v1
35+
kind: ClusterRoleBinding
36+
metadata:
37+
name: kueue-localqueue-ocm-rolebinding
38+
roleRef:
39+
apiGroup: rbac.authorization.k8s.io
40+
kind: ClusterRole
41+
name: kueue-localqueue-editor-role
42+
subjects:
43+
- kind: ServiceAccount
44+
name: klusterlet-work-sa
45+
namespace: open-cluster-management-agent
46+
- apiVersion: kueue.x-k8s.io/v1beta1
47+
kind: ResourceFlavor
48+
metadata:
49+
name: {{ .Values.resourceFlavor.name}}
50+
- apiVersion: kueue.x-k8s.io/v1beta1
51+
kind: ClusterQueue
52+
metadata:
53+
name: {{ .Values.clusterQueue.name}}
54+
spec:
55+
namespaceSelector: {{ .Values.clusterQueue.namespaceSelector }}
56+
resourceGroups:
57+
{{- toYaml .Values.clusterQueue.resourceGroups | nindent 12 }}
58+
- apiVersion: kueue.x-k8s.io/v1beta1
59+
kind: LocalQueue
60+
metadata:
61+
namespace: {{ .Values.localQueue.namespace}}
62+
name: {{ .Values.localQueue.name}}
63+
spec:
64+
clusterQueue: {{ .Values.clusterQueue.name}}

‎kueue-addon/charts/kueue-addon/templates/cluster-management-addon.yaml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
apiVersion: addon.open-cluster-management.io/v1alpha1
2+
kind: ClusterManagementAddOn
3+
metadata:
4+
name: kueue-addon
5+
spec:
6+
addOnMeta:
7+
displayName: kueue-addon
8+
description: kueue-addon
9+
supportedConfigs:
10+
- group: addon.open-cluster-management.io
11+
resource: addontemplates
12+
installStrategy:
13+
type: Placements
14+
placements:
15+
- name: {{ .Values.placement.hub.name }}
16+
namespace: {{ .Release.Namespace }}
17+
configs:
18+
- name: kueue-addon-hub-{{ .Chart.Version }}
19+
group: addon.open-cluster-management.io
20+
resource: addontemplates
21+
- name: {{ .Values.placement.spoke.name }}
22+
namespace: {{ .Release.Namespace }}
23+
configs:
24+
- name: kueue-addon-spoke-{{ .Chart.Version }}
25+
group: addon.open-cluster-management.io
26+
resource: addontemplates

‎kueue-addon/charts/kueue-addon/templates/placement.yaml

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{{ if not .Values.skipClusterSetBinding }}
2+
apiVersion: cluster.open-cluster-management.io/v1beta2
3+
kind: ManagedClusterSetBinding
4+
metadata:
5+
name: global
6+
namespace: {{ .Release.Namespace }}
7+
spec:
8+
clusterSet: global
9+
{{- end }}
10+
11+
---
12+
13+
apiVersion: cluster.open-cluster-management.io/v1beta1
14+
kind: Placement
15+
metadata:
16+
name: {{ .Values.placement.hub.name }}
17+
namespace: {{ .Release.Namespace }}
18+
spec:
19+
clusterSets:
20+
- global
21+
tolerations:
22+
- key: cluster.open-cluster-management.io/unreachable
23+
operator: Equal
24+
- key: cluster.open-cluster-management.io/unavailable
25+
operator: Equal
26+
predicates:
27+
- requiredClusterSelector:
28+
labelSelector:
29+
matchLabels:
30+
local-cluster: "true"
31+
32+
---
33+
34+
apiVersion: cluster.open-cluster-management.io/v1beta1
35+
kind: Placement
36+
metadata:
37+
name: {{ .Values.placement.spoke.name }}
38+
namespace: {{ .Release.Namespace }}
39+
spec:
40+
clusterSets:
41+
- global
42+
tolerations:
43+
- key: cluster.open-cluster-management.io/unreachable
44+
operator: Equal
45+
- key: cluster.open-cluster-management.io/unavailable
46+
operator: Equal
47+
predicates:
48+
- requiredClusterSelector:
49+
labelSelector:
50+
matchExpressions:
51+
- key: local-cluster
52+
operator: DoesNotExist

‎kueue-addon/charts/kueue-addon/values.yaml

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# Default values for kueue-addon.
2+
# This is a YAML-formatted file.
3+
4+
# Image configuration
5+
image: quay.io/open-cluster-management/kueue-addon
6+
tag: latest
7+
8+
# Skip cluster set binding
9+
skipClusterSetBinding: false
10+
11+
# Placement configuration
12+
placement:
13+
hub:
14+
name: hub
15+
spoke:
16+
name: spoke
17+
18+
# Resource flavor
19+
resourceFlavor:
20+
name: default-flavor
21+
22+
# Cluster queue
23+
clusterQueue:
24+
name: cluster-queue
25+
namespaceSelector: "{}" # match all namespaces
26+
resourceGroups:
27+
- coveredResources: ["cpu", "memory", "nvidia.com/gpu"]
28+
flavors:
29+
- name: "default-flavor"
30+
resources:
31+
- name: "cpu"
32+
nominalQuota: 9
33+
- name: "memory"
34+
nominalQuota: 36Gi
35+
- name: "nvidia.com/gpu"
36+
nominalQuota: 3
37+
38+
# Local queue
39+
localQueue:
40+
name: user-queue
41+
namespace: default

0 commit comments

Comments
 (0)