Skip to content

Commit 94d314d

Browse files
dspodspo
authored
feature: migrate the API GetProject from core-services to dop (#2714)
* feature: migrate the API GetProject from core-servcies to dop * feature: license * feature: migrate the API fetch-resource from core-services to dop * feature: remove cmp client from core-services * feature: remove cmp dependencies from core-services * fmt: project * feature: check new quota is less than request * migration: update old project quota * fix: i18n; migration for quota * fix: i18n context * feature: remove dop dependency on quota table * fix: compatible with old quota * fmt: project * fix: invalid memory address or nil pointer dereference * fix: ut * fix: typo * feature: add ut * resolve conflic * fmt: project * feature: accuracy
1 parent c58190e commit 94d314d

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+1249
-596
lines changed

‎.erda/migrations/cmdb/20211101-update-project-quota.sql

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
UPDATE ps_group_projects
2+
SET cpu_quota=0,
3+
mem_quota=0;
4+
5+
UPDATE ps_group_projects a, ps_group_projects_quota b
6+
SET a.cpu_quota=b.prod_cpu_quota + b.staging_cpu_quota + b.test_cpu_quota + b.dev_cpu_quota,
7+
a.mem_quota=b.prod_mem_quota + b.staging_mem_quota + b.test_mem_quota + b.dev_mem_quota
8+
WHERE a.id = b.project_id
9+
AND b.project_id IS NOT NULL
10+
;

‎apistructs/project.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,7 @@ type ProjectDTO struct {
248248
ClusterConfig map[string]string `json:"clusterConfig"`
249249
// ResourceConfig shows the relationship between clusters and workspaces,
250250
// and contains the quota info for every workspace .
251-
ResourceConfig *ResourceConfigsInfo `json:"resourceConfig"`
251+
ResourceConfig *ResourceConfigsInfo `json:"resourceConfig,omitempty"`
252252
RollbackConfig map[string]int `json:"rollbackConfig"`
253253
// Deprecated: to retrieve the quota for every workspace, prefer to use ResourceConfig
254254
CpuQuota float64 `json:"cpuQuota"`
@@ -304,15 +304,15 @@ type ResourceConfigInfo struct {
304304
CPURequestByAddonRate float64 `json:"cpuRequestByAddonRate"`
305305
CPURequestByService float64 `json:"cpuRequestByService"`
306306
CPURequestByServiceRate float64 `json:"cpuRequestByServiceRate"`
307-
CPUAvailable float64 `json:"cpuAvailable"`
307+
CPUAvailable float64 `json:"cpuAvailable,omitempty"`
308308
MemQuota float64 `json:"memQuota"`
309309
MemRequest float64 `json:"memRequest"`
310310
MemRequestRate float64 `json:"memRequestRate"`
311311
MemRequestByAddon float64 `json:"memRequestByAddon"`
312312
MemRequestByAddonRate float64 `json:"memRequestByAddonRate"`
313313
MemRequestByService float64 `json:"memRequestByService"`
314314
MemRequestByServiceRate float64 `json:"memRequestByServiceRate"`
315-
MemAvailable float64 `json:"memAvailable"`
315+
MemAvailable float64 `json:"memAvailable,omitempty"`
316316
Tips string `json:"tips"`
317317
}
318318

‎apistructs/project_quota.go

+69-9
Original file line numberDiff line numberDiff line change
@@ -14,8 +14,14 @@
1414

1515
package apistructs
1616

17-
import "time"
17+
import (
18+
"strings"
19+
"time"
20+
)
1821

22+
// ProjectQuota is the table "ps_group_projects_quota"
23+
// CPU quota unit is Core * 10^-3
24+
// Mem quota uint is Byte
1925
type ProjectQuota struct {
2026
ID uint64 `json:"id" gorm:"id"`
2127
UpdatedAt time.Time `json:"updated_at" gorm:"updated_at"`
@@ -26,18 +32,72 @@ type ProjectQuota struct {
2632
StagingClusterName string `json:"staging_cluster_name" gorm:"staging_cluster_name"`
2733
TestClusterName string `json:"test_cluster_name" gorm:"test_cluster_name"`
2834
DevClusterName string `json:"dev_cluster_name" gorm:"dev_cluster_name"`
29-
ProdCPUQuota int64 `json:"prod_cpu_quota" gorm:"prod_cpu_quota"`
30-
ProdMemQuota int64 `json:"prod_mem_quota" gorm:"prod_mem_quota"`
31-
StagingCPUQuota int64 `json:"staging_cpu_quota" gorm:"staging_cpu_quota"`
32-
StagingMemQuota int64 `json:"staging_mem_quota" gorm:"staging_mem_quota"`
33-
TestCPUQuota int64 `json:"test_cpu_quota" gorm:"test_cpu_quota"`
34-
TestMemQuota int64 `json:"test_mem_quota" gorm:"test_mem_quota"`
35-
DevCPUQuota int64 `json:"dev_cpu_quota" gorm:"dev_cpu_quota"`
36-
DevMemQuota int64 `json:"dev_mem_quota" gorm:"dev_mem_quota"`
35+
ProdCPUQuota uint64 `json:"prod_cpu_quota" gorm:"prod_cpu_quota"`
36+
ProdMemQuota uint64 `json:"prod_mem_quota" gorm:"prod_mem_quota"`
37+
StagingCPUQuota uint64 `json:"staging_cpu_quota" gorm:"staging_cpu_quota"`
38+
StagingMemQuota uint64 `json:"staging_mem_quota" gorm:"staging_mem_quota"`
39+
TestCPUQuota uint64 `json:"test_cpu_quota" gorm:"test_cpu_quota"`
40+
TestMemQuota uint64 `json:"test_mem_quota" gorm:"test_mem_quota"`
41+
DevCPUQuota uint64 `json:"dev_cpu_quota" gorm:"dev_cpu_quota"`
42+
DevMemQuota uint64 `json:"dev_mem_quota" gorm:"dev_mem_quota"`
3743
CreatorID uint64 `json:"creator_id" gorm:"creator_id"`
3844
UpdaterID uint64 `json:"updater_id" gorm:"updater_id"`
3945
}
4046

47+
// TableName returns the model's name "ps_group_projects_quota"
4148
func (ProjectQuota) TableName() string {
4249
return "ps_group_projects_quota"
4350
}
51+
52+
func (p ProjectQuota) GetClusterName(workspace string) string {
53+
switch strings.ToLower(workspace) {
54+
case "prod":
55+
return p.ProdClusterName
56+
case "staging":
57+
return p.StagingClusterName
58+
case "test":
59+
return p.TestClusterName
60+
case "dev":
61+
return p.DevClusterName
62+
default:
63+
return ""
64+
}
65+
}
66+
67+
// GetCPUQuota returns the CPU quota on the workspace.
68+
// The unit is Core * 10^-3
69+
func (p ProjectQuota) GetCPUQuota(workspace string) uint64 {
70+
switch strings.ToLower(workspace) {
71+
case "prod":
72+
return p.ProdCPUQuota
73+
case "staging":
74+
return p.StagingCPUQuota
75+
case "test":
76+
return p.TestCPUQuota
77+
case "dev":
78+
return p.DevCPUQuota
79+
default:
80+
return 0
81+
}
82+
}
83+
84+
// GetMemQuota returns the Mem quota on the workspace.
85+
// The unit is Byte
86+
func (p ProjectQuota) GetMemQuota(workspace string) uint64 {
87+
switch strings.ToLower(workspace) {
88+
case "prod":
89+
return p.ProdMemQuota
90+
case "staging":
91+
return p.StagingMemQuota
92+
case "test":
93+
return p.TestMemQuota
94+
case "dev":
95+
return p.DevMemQuota
96+
default:
97+
return 0
98+
}
99+
}
100+
101+
func (p ProjectQuota) ClustersNames() []string {
102+
return []string{p.ProdClusterName, p.StagingClusterName, p.TestClusterName, p.DevClusterName}
103+
}

‎apistructs/project_test.go

+3-4
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,6 @@ import (
1919
"testing"
2020

2121
"github.com/erda-project/erda/apistructs"
22-
"github.com/erda-project/erda/modules/core-services/model"
2322
calcu "github.com/erda-project/erda/pkg/resourcecalculator"
2423
)
2524

@@ -59,7 +58,7 @@ func TestProjectCreateRequest(t *testing.T) {
5958
}
6059
t.Logf("project: %+v", project)
6160

62-
quota := &model.ProjectQuota{
61+
quota := &apistructs.ProjectQuota{
6362
ProjectID: 0,
6463
ProjectName: project.Name,
6564
ProdClusterName: project.ResourceConfigs.PROD.ClusterName,
@@ -74,8 +73,8 @@ func TestProjectCreateRequest(t *testing.T) {
7473
TestMemQuota: calcu.GibibyteToByte(project.ResourceConfigs.TEST.MemQuota),
7574
DevCPUQuota: calcu.CoreToMillcore(project.ResourceConfigs.DEV.CPUQuota),
7675
DevMemQuota: calcu.GibibyteToByte(project.ResourceConfigs.DEV.MemQuota),
77-
CreatorID: "0",
78-
UpdaterID: "0",
76+
CreatorID: 0,
77+
UpdaterID: 0,
7978
}
8079
data, _ := json.MarshalIndent(quota, "", " ")
8180
t.Log(string(data))

‎apistructs/testplan_test.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -63,4 +63,4 @@ func TestTestPlanCreateRequest_Check(t *testing.T) {
6363
for _, v := range tt {
6464
assert.Equal(t, v.want, v.req.Check() == nil)
6565
}
66-
}
66+
}

‎bundle/quota.go

+23
Original file line numberDiff line numberDiff line change
@@ -84,3 +84,26 @@ func (b *Bundle) FetchNamespacesBelongsTo() (*apistructs.GetProjectsNamesapcesRe
8484
}
8585
return resp.Data, nil
8686
}
87+
88+
func (b *Bundle) ListQuotaRecords() ([]*apistructs.ProjectQuota, error) {
89+
host, err := b.urls.CoreServices()
90+
if err != nil {
91+
return nil, err
92+
}
93+
type response struct {
94+
apistructs.Header
95+
Data struct {
96+
Total uint64 `json:"total"`
97+
List []*apistructs.ProjectQuota `json:"list"`
98+
}
99+
}
100+
var resp response
101+
httpResp, err := b.hc.Get(host).Path("/api/quota-records").Do().JSON(&resp)
102+
if err != nil {
103+
return nil, err
104+
}
105+
if !httpResp.IsOK() {
106+
return nil, toAPIError(httpResp.StatusCode(), resp.Error)
107+
}
108+
return resp.Data.List, nil
109+
}

‎cmd/core-services/main.go

-1
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@ import (
2020

2121
// providers
2222
_ "github.com/erda-project/erda-infra/providers/grpcclient"
23-
_ "github.com/erda-project/erda-proto-go/cmp/dashboard/client"
2423
_ "github.com/erda-project/erda-proto-go/core/pipeline/cms/client"
2524
_ "github.com/erda-project/erda/modules/core-services"
2625
_ "github.com/erda-project/erda/modules/core-services/services/notify/channel"

‎cmd/dop/main.go

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
_ "github.com/erda-project/erda-infra/providers/i18n"
2525
_ "github.com/erda-project/erda-infra/providers/mysql"
2626
_ "github.com/erda-project/erda-infra/providers/serviceregister"
27+
_ "github.com/erda-project/erda-proto-go/cmp/dashboard/client"
2728
_ "github.com/erda-project/erda-proto-go/core/pipeline/cms/client"
2829
_ "github.com/erda-project/erda-proto-go/core/pipeline/definition/client"
2930
_ "github.com/erda-project/erda/modules/dop"

‎conf/core-services/core-services.yaml

-5
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,3 @@ mysql:
2727

2828
erda.core.services.authentication.credentials.accesskey:
2929
erda.core.services.notify.channel:
30-
31-
[email protected]:
32-
addr: "${CMP_GRPC_ADDR:cmp:9029}"
33-
block: false
34-
erda.cmp.dashboard.resource-client:

‎conf/core-services/i18n/i18n.yaml

+9-1
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ zh:
1010
TEST: '测试'
1111
DEV: '开发'
1212
OwnerUnknown: '无'
13+
CPU: ' CPU '
14+
Mem: ' 内存 '
15+
CPUQuotaIsLessThanRequest: '调整后的%s环境的 CPU 配额（%.2f 核）小于请求值（%.2f 核），请重新设置'
16+
MemQuotaIsLessThanRequest: '调整后的%s环境的内存配额（%.2f G）小于请求值（%.2f G），请重新设置'
1317
en:
1418
AvailableIsLessThanQuota: The actual available resource on this workspace in the cluster is less than the quota. Please ask the Ops to allocate project resources reasonably
1519
NoResourceForTheWorkspace: No allocatable resources on this workspace in the cluster, please check the workspace labels for the nodes
@@ -21,4 +25,8 @@ en:
2125
STAGING: 'STAGING'
2226
TEST: 'TEST'
2327
DEV: 'DEV'
24-
OwnerUnknown: 'Unknown'
28+
OwnerUnknown: 'Unknown'
29+
CPU: ' CPU '
30+
Mem: ' Mem '
31+
CPUQuotaIsLessThanRequest: "The env %s's new CPU quota (%.2f Core) is less than the actual request (%.2f Core), please reset it"
32+
MemQuotaIsLessThanRequest: "The env %s's new Mem quota (%.2f G) is less than the actual request (%.2f G), please reset it"

‎conf/dop/dop.yaml

+6
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ grpc-server:
1010
i18n:
1111
files:
1212
- conf/dop/i18n/component-protocol.yaml
13+
- conf/dop/i18n/resource-trans.yaml
1314
mysql:
1415
host: "${MYSQL_HOST}"
1516
port: "${MYSQL_PORT}"
@@ -33,6 +34,11 @@ [email protected]:
3334
erda.core.pipeline.definition-client:
3435
erda.core.pipeline.definition-process-client:
3536

37+
[email protected]:
38+
addr: "${CMP_GRPC_ADDR:cmp:9029}"
39+
block: false
40+
erda.cmp.dashboard.resource-client:
41+
3642
erda.core.dop.autotest.testplan:
3743

3844
# component-protocol framework

‎conf/dop/i18n/resource-trans.yaml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
zh:
2+
AvailableIsLessThanQuota: 本集群上该环境实际可用资源已小于配额，请询运维人员合理分配资源
3+
NoResourceForTheWorkspace: 本集群上该环境的可用资源为 0, 请检查是否未正确为节点标注环境标签
4+
NoNodesInTheCluster: 此集群没有可用的节点，请添加节点
5+
ResourceSqueeze: '注意：该集群在%s环境已分配资源已经超过该集群在%s环境总资源，项目资源分配会按先用先得的原则进行抢占'
6+
CPUResourceSqueeze: '注意：该集群在%s环境已分配资源已经超过该集群在%s环境总资源，项目资源分配会按先用先得的原则进行抢占'
7+
MmeResourceSqueeze: '注意：该集群在%s环境已分配资源已经超过该集群在%s环境总资源，项目资源分配会按先用先得的原则进行抢占'
8+
PROD: '生产'
9+
STAGING: '预发'
10+
TEST: '测试'
11+
DEV: '开发'
12+
OwnerUnknown: '无'
13+
en:
14+
AvailableIsLessThanQuota: The actual available resource on this workspace in the cluster is less than the quota. Please ask the Ops to allocate project resources reasonably
15+
NoResourceForTheWorkspace: No allocatable resources on this workspace in the cluster, please check the workspace labels for the nodes
16+
NoNodesInTheCluster: No allocatable nodes in the cluster, please add nodes for it",
17+
ResourceSqueeze: 'Note: The current allocated resources of the cluster on the workspace %s have exceeded the total resources of the cluster on the workspace %s, the project resource allocation will be based on the principle of first use first serve Preempt'
18+
CPUResourceSqueeze: 'Note: The current allocated resources of the cluster on the workspace %s have exceeded the total resources of the cluster on the workspace %s, the project resource allocation will be based on the principle of first use first serve Preempt'
19+
MemResourceSqueeze: 'Note: The current allocated resources of the cluster on the workspace %s have exceeded the total resources of the cluster on the workspace %s, the project resource allocation will be based on the principle of first use first serve Preempt'
20+
PROD: 'PROD'
21+
STAGING: 'STAGING'
22+
TEST: 'TEST'
23+
DEV: 'DEV'
24+
OwnerUnknown: 'Unknown'

‎modules/core-services/dao/project.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ func (client *DBClient) DeleteProject(projectID int64) error {
4141
}
4242

4343
func (client *DBClient) DeleteProjectQutoa(projectID int64) error {
44-
return client.Delete(new(model.ProjectQuota), map[string]interface{}{"project_id": projectID}).Error
44+
return client.Delete(new(apistructs.ProjectQuota), map[string]interface{}{"project_id": projectID}).Error
4545
}
4646

4747
// GetProjectByID 根据projectID获取项目信息

‎modules/core-services/dao/quota.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import (
2222

2323
// GetWorkspaceQuota get project workspace quota
2424
// Valid workspace: prod, staging, test, dev
25-
func (client *DBClient) GetWorkspaceQuota(projectID, workspace string) (int64, int64, error) {
25+
func (client *DBClient) GetWorkspaceQuota(projectID, workspace string) (uint64, uint64, error) {
2626
var projectQuota apistructs.ProjectQuota
2727
if err := client.Find(&projectQuota, map[string]interface{}{
2828
"project_id": projectID,

‎modules/core-services/endpoints/endpoints.go

+25-29
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ import (
2121
"github.com/aliyun/aliyun-oss-go-sdk/oss"
2222
"github.com/gorilla/schema"
2323

24-
dashboardPb "github.com/erda-project/erda-proto-go/cmp/dashboard/pb"
2524
"github.com/erda-project/erda/bundle"
2625
"github.com/erda-project/erda/modules/core-services/dao"
2726
"github.com/erda-project/erda/modules/core-services/services/activity"
@@ -50,31 +49,30 @@ import (
5049

5150
// Endpoints 定义 endpoint 方法
5251
type Endpoints struct {
53-
store jsonstore.JsonStore
54-
etcdStore *etcd.Store
55-
ossClient *oss.Client
56-
db *dao.DBClient
57-
uc *ucauth.UCClient
58-
bdl *bundle.Bundle
59-
clusterResourceClient dashboardPb.ClusterResourceClient
60-
org *org.Org
61-
project *project.Project
62-
approve *approve.Approve
63-
app *application.Application
64-
member *member.Member
65-
ManualReview *manual_review.ManualReview
66-
activity *activity.Activity
67-
permission *permission.Permission
68-
license *license.License
69-
notifyGroup *notify.NotifyGroup
70-
mbox *mbox.MBox
71-
label *label.Label
72-
notice *notice.Notice
73-
queryStringDecoder *schema.Decoder
74-
audit *audit.Audit
75-
errorbox *errorbox.ErrorBox
76-
fileSvc *filesvc.FileService
77-
user *user.User
52+
store jsonstore.JsonStore
53+
etcdStore *etcd.Store
54+
ossClient *oss.Client
55+
db *dao.DBClient
56+
uc *ucauth.UCClient
57+
bdl *bundle.Bundle
58+
org *org.Org
59+
project *project.Project
60+
approve *approve.Approve
61+
app *application.Application
62+
member *member.Member
63+
ManualReview *manual_review.ManualReview
64+
activity *activity.Activity
65+
permission *permission.Permission
66+
license *license.License
67+
notifyGroup *notify.NotifyGroup
68+
mbox *mbox.MBox
69+
label *label.Label
70+
notice *notice.Notice
71+
queryStringDecoder *schema.Decoder
72+
audit *audit.Audit
73+
errorbox *errorbox.ErrorBox
74+
fileSvc *filesvc.FileService
75+
user *user.User
7876
}
7977

8078
type Option func(*Endpoints)
@@ -293,9 +291,6 @@ func (e *Endpoints) Routes() []httpserver.Endpoint {
293291
{Path: "/api/orgs/clusters/relations/{orgID}", Method: http.MethodGet, Handler: e.GetOrgClusterRelationsByOrg},
294292
{Path: "/api/clusters/actions/dereference", Method: http.MethodPut, Handler: e.DereferenceCluster},
295293

296-
// 获取企业可用资源
297-
{Path: "/api/orgs/actions/fetch-resources", Method: http.MethodGet, Handler: e.FetchOrgResources},
298-
299294
// the interface of project
300295
{Path: "/api/projects", Method: http.MethodPost, Handler: e.CreateProject},
301296
{Path: "/api/projects/{projectID}", Method: http.MethodPut, Handler: e.UpdateProject},
@@ -320,6 +315,7 @@ func (e *Endpoints) Routes() []httpserver.Endpoint {
320315
// cmp dependencies
321316
{Path: "/api/projects-quota", Method: http.MethodGet, Handler: e.GetProjectQuota},
322317
{Path: "/api/projects-namespaces", Method: http.MethodGet, Handler: e.GetNamespacesBelongsTo},
318+
{Path: "/api/quota-records", Method: http.MethodGet, Handler: e.ListQuotaRecords},
323319

324320
// the interface of application
325321
{Path: "/api/applications", Method: http.MethodPost, Handler: e.CreateApplication},

0 commit comments

Comments
 (0)