Skip to content

Commit 6708fa9

Browse files
sfwnsnakorse
authored andcommitted
refactor: manual test case&plan-case-relations paging use sql join (erda-project#2910)
* refacotr: manual test case&plan-case-relations paging use sql join * limit testset's directory length * fix: manual-test fix paging by updatedAt * migration: manual test add migration for slow sql * add unit test for mt-case-paging refactor
1 parent a984556 commit 6708fa9

15 files changed

+997
-355
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# dice_test_cases
2+
ALTER TABLE `dice_test_cases` ADD INDEX `idx_proj_testset_recycle_priority_updater_name` (`project_id`, `test_set_id`, `recycled`, `priority`, `updater_id`, `name`);
3+
4+
# dice_test_plan_case_relations
5+
ALTER TABLE `dice_test_plan_case_relations` ADD INDEX `idx_testcaseid` (`test_case_id`);
6+
ALTER TABLE `dice_test_plan_case_relations` ADD INDEX `idx_plan_set_case_status_updater` (`test_plan_id`, `test_set_id`, `test_case_id`, `exec_status`, `updater_id`);
7+
8+
# dice_test_sets
9+
## related to: modules/dop/dao/testset.go:28
10+
ALTER TABLE `dice_test_sets` MODIFY COLUMN `directory` varchar(5000) NOT NULL DEFAULT '' COMMENT '当前节点+所有父级节点的name集合(参考值:新建测试集1/新建测试集2/测试集名称3),这里冗余是为了方便界面展示。';
11+
ALTER TABLE `dice_test_sets` ADD INDEX `idx_directory` (`directory`(191));
12+
13+
# dice_test_plan_members
14+
ALTER TABLE `dice_test_plan_members` MODIFY COLUMN `user_id` varchar(191) NOT NULL DEFAULT '' COMMENT 'user_id';
15+
ALTER TABLE `dice_test_plan_members` ADD INDEX `idx_plan_role_userid` (`test_plan_id`, `role`, `user_id`);

apistructs/testplan.go

+16-8
Original file line numberDiff line numberDiff line change
@@ -205,14 +205,22 @@ type TestPlanCaseRelPagingRequest struct {
205205
UpdatedAtEndInclude *time.Time `schema:"-"`
206206

207207
// order by field
208-
OrderByPriorityAsc *bool `schema:"orderByPriorityAsc"`
209-
OrderByPriorityDesc *bool `schema:"orderByPriorityDesc"`
210-
OrderByUpdaterIDAsc *bool `schema:"orderByUpdaterIDAsc"`
211-
OrderByUpdaterIDDesc *bool `schema:"orderByUpdaterIDDesc"`
212-
OrderByUpdatedAtAsc *bool `schema:"orderByUpdatedAtAsc"`
213-
OrderByUpdatedAtDesc *bool `schema:"orderByUpdatedAtDesc"`
214-
OrderByIDAsc *bool `schema:"orderByIDAsc"`
215-
OrderByIDDesc *bool `schema:"orderByIDDesc"`
208+
OrderFields []string `schema:"orderField"`
209+
OrderByPriorityAsc *bool `schema:"orderByPriorityAsc"`
210+
OrderByPriorityDesc *bool `schema:"orderByPriorityDesc"`
211+
OrderByUpdaterIDAsc *bool `schema:"orderByUpdaterIDAsc"`
212+
OrderByUpdaterIDDesc *bool `schema:"orderByUpdaterIDDesc"`
213+
OrderByUpdatedAtAsc *bool `schema:"orderByUpdatedAtAsc"`
214+
OrderByUpdatedAtDesc *bool `schema:"orderByUpdatedAtDesc"`
215+
OrderByIDAsc *bool `schema:"orderByIDAsc"`
216+
OrderByIDDesc *bool `schema:"orderByIDDesc"`
217+
OrderByTestSetIDAsc *bool `schema:"orderByTestSetIDAsc"`
218+
OrderByTestSetIDDesc *bool `schema:"orderByTestSetIDDesc"`
219+
OrderByTestSetNameAsc *bool `schema:"orderByTestSetNameAsc"`
220+
OrderByTestSetNameDesc *bool `schema:"orderByTestSetNameDesc"`
221+
222+
// tp
223+
TestPlan *TestPlan `schema:"-"` // internal use
216224

217225
IdentityInfo
218226
}

go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ require (
3333
github.com/dsnet/compress v0.0.1 // indirect
3434
github.com/dustin/go-humanize v1.0.0
3535
github.com/elastic/cloud-on-k8s v0.0.0-20210205172912-5ce0eca90c60
36-
github.com/erda-project/erda-infra v0.0.0-20211103123843-dd949cf5688a
36+
github.com/erda-project/erda-infra v0.0.0-20211108014909-ee0141361213
3737
github.com/erda-project/erda-oap-thirdparty-protocol v0.0.0-20210907135609-15886a136d5b
3838
github.com/erda-project/erda-proto-go v0.0.0
3939
github.com/erda-project/erda-sourcecov v0.1.0

go.sum

+4
Original file line numberDiff line numberDiff line change
@@ -452,6 +452,10 @@ github.com/erda-project/elastic v0.0.1-ex h1:5ajfxQ5S5YjpzFqY9LzL9hiKWCn6q/JDT4n
452452
github.com/erda-project/elastic v0.0.1-ex/go.mod h1:iAVsas6fcmt9pxtge1+dErMhecv+RLSXlD4rnZRJVW0=
453453
github.com/erda-project/erda-infra v0.0.0-20211103123843-dd949cf5688a h1:NdAhlhv5Lq12rBAp8YJQp+A7vXo4ybJyesOJYlSGwc0=
454454
github.com/erda-project/erda-infra v0.0.0-20211103123843-dd949cf5688a/go.mod h1:YWl4gg86UMFLrr160Jba/4yhovqC/t7Vi6cpGqx1TrY=
455+
github.com/erda-project/erda-infra v0.0.0-20211105091917-e0f611a1311f h1:VCyqn5HYc+F90b4mh3oTn4tmDrpS+xrzCNAKmqAEAZs=
456+
github.com/erda-project/erda-infra v0.0.0-20211105091917-e0f611a1311f/go.mod h1:YWl4gg86UMFLrr160Jba/4yhovqC/t7Vi6cpGqx1TrY=
457+
github.com/erda-project/erda-infra v0.0.0-20211108014909-ee0141361213 h1:v8Xf/qKCDDEQ2VJtsybTufa/xJ1pNwZDx7ToPDciNFQ=
458+
github.com/erda-project/erda-infra v0.0.0-20211108014909-ee0141361213/go.mod h1:YWl4gg86UMFLrr160Jba/4yhovqC/t7Vi6cpGqx1TrY=
455459
github.com/erda-project/erda-oap-thirdparty-protocol v0.0.0-20210907135609-15886a136d5b h1:GWf2ChasZFerFwQoTokIvjJLWH57ligTSLD2hUb7UWk=
456460
github.com/erda-project/erda-oap-thirdparty-protocol v0.0.0-20210907135609-15886a136d5b/go.mod h1:H/f81Thef2Tnz4nUeLt0r4VwHdOznthpyXBwT9vDWo0=
457461
github.com/erda-project/erda-sourcecov v0.1.0 h1:iLvoMsQ1xX81KNOW98BKr85Vs7sSazrIDEphLYdmgP4=

modules/dop/dao/testcase.go

+239
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,15 @@ import (
1818
"database/sql/driver"
1919
"encoding/json"
2020
"fmt"
21+
"sync"
22+
"time"
2123

2224
"github.com/pkg/errors"
2325

2426
"github.com/erda-project/erda/apistructs"
27+
"github.com/erda-project/erda/modules/dop/services/apierrors"
2528
"github.com/erda-project/erda/pkg/database/dbengine"
29+
"github.com/erda-project/erda/pkg/strutil"
2630
)
2731

2832
// TestCase 测试用例
@@ -188,3 +192,238 @@ func (client *DBClient) CleanTestCasesByTestSetID(projectID, testSetID uint64) e
188192
func (client *DBClient) BatchDeleteTestCases(ids []uint64) error {
189193
return client.Where("`id` IN (?)", ids).Delete(TestCase{}).Error
190194
}
195+
196+
// order
197+
const (
198+
tcFieldPriority = "priority"
199+
tcFieldID = "id"
200+
tcFieldTestSetID = "test_set_id"
201+
tcFieldTestSetIDV2 = "testSetID"
202+
tcFieldUpdaterID = "updater_id"
203+
tcFieldUpdaterIDV2 = "updaterID"
204+
tcFieldUpdatedAt = "updated_at"
205+
tcFieldUpdatedAtV2 = "updatedAt"
206+
)
207+
208+
func (client *DBClient) PagingTestCases(req apistructs.TestCasePagingRequest) ([]TestCase, uint64, error) {
209+
// validate request
210+
if err := validateTestCasePagingRequest(req); err != nil {
211+
return nil, 0, err
212+
}
213+
// set default for request
214+
setDefaultForTestCasePagingRequest(&req)
215+
// query base test set if necessary, then use `directory` to do `like` query
216+
var baseTestSet TestSet
217+
if req.TestSetID > 0 {
218+
ts, err := client.GetTestSetByID(req.TestSetID)
219+
if err != nil {
220+
return nil, 0, err
221+
}
222+
baseTestSet = *ts
223+
}
224+
225+
baseSQL := client.DB.Table(TestCase{}.TableName() + " AS `tc`").Select("*")
226+
227+
// left join test_plan_case_relations
228+
if len(req.NotInTestPlanIDs) > 0 {
229+
baseSQL = baseSQL.Joins(
230+
"LEFT JOIN ("+
231+
" SELECT * FROM "+TestPlanCaseRel{}.TableName()+" WHERE `test_plan_id` IN (?) GROUP BY `test_case_id`"+
232+
") AS `rel` ON `tc`.`id` = `rel`.`test_case_id`",
233+
req.NotInTestPlanIDs,
234+
)
235+
baseSQL = baseSQL.Where("`rel`.`test_plan_id` IS NULL OR `rel`.`test_plan_id` NOT IN (?)", req.NotInTestPlanIDs)
236+
}
237+
238+
// left join test_sets
239+
// use left join because test_set with id = 0 is not exists in test_sets table
240+
baseSQL = baseSQL.Joins("LEFT JOIN " + TestSet{}.TableName() + " AS `ts` ON `tc`.`test_set_id` = `ts`.`id`")
241+
242+
// where clauses
243+
// project id
244+
baseSQL = baseSQL.Where("`tc`.`project_id` = ?", req.ProjectID)
245+
// test set id
246+
if req.TestSetID > 0 {
247+
baseSQL = baseSQL.Where("`ts`.`directory` LIKE '" + baseTestSet.Directory + "%'")
248+
}
249+
// recycled
250+
baseSQL = baseSQL.Where("`tc`.`recycled` = ?", req.Recycled)
251+
// name
252+
if req.Query != "" {
253+
baseSQL = baseSQL.Where("`tc`.`name` LIKE ?", strutil.Concat("%", req.Query, "%"))
254+
}
255+
// priority
256+
if len(req.Priorities) > 0 {
257+
baseSQL = baseSQL.Where("`tc`.`priority` IN (?)", req.Priorities)
258+
}
259+
// updater
260+
if len(req.UpdaterIDs) > 0 {
261+
baseSQL = baseSQL.Where("`tc`.`updater_id` IN (?)", req.UpdaterIDs)
262+
}
263+
// updatedAtBegin (Left closed Section)
264+
if req.TimestampSecUpdatedAtBegin != nil {
265+
t := time.Unix(int64(*req.TimestampSecUpdatedAtBegin), 0)
266+
req.UpdatedAtBeginInclude = &t
267+
}
268+
if req.UpdatedAtBeginInclude != nil {
269+
baseSQL = baseSQL.Where("`tc`.`updated_at` >= ?", req.UpdatedAtBeginInclude)
270+
}
271+
// updatedAtEnd (Right closed Section)
272+
if req.TimestampSecUpdatedAtEnd != nil {
273+
t := time.Unix(int64(*req.TimestampSecUpdatedAtEnd), 0)
274+
req.UpdatedAtEndInclude = &t
275+
}
276+
if req.UpdatedAtEndInclude != nil {
277+
baseSQL = baseSQL.Where("`tc`.`updated_at` <= ?", req.UpdatedAtEndInclude)
278+
}
279+
// testCaseIDs
280+
if len(req.TestCaseIDs) > 0 {
281+
baseSQL = baseSQL.Where("`tc`.`id` IN (?)", req.TestCaseIDs)
282+
}
283+
if len(req.NotInTestCaseIDs) > 0 {
284+
baseSQL = baseSQL.Where("`tc`.`id` NOT IN (?)", req.NotInTestCaseIDs)
285+
}
286+
287+
pagingSQL := baseSQL.NewScope(nil).DB()
288+
countSQL := baseSQL.NewScope(nil).DB()
289+
290+
// order by fields
291+
for _, orderField := range req.OrderFields {
292+
switch orderField {
293+
case tcFieldID:
294+
if req.OrderByIDAsc != nil {
295+
pagingSQL = pagingSQL.Order("`tc`.`id` ASC")
296+
}
297+
if req.OrderByIDDesc != nil {
298+
pagingSQL = pagingSQL.Order("`tc`.`id` DESC")
299+
}
300+
case tcFieldTestSetID, tcFieldTestSetIDV2:
301+
if req.OrderByTestSetIDAsc != nil {
302+
pagingSQL = pagingSQL.Order("`tc`.`test_set_id` ASC")
303+
}
304+
if req.OrderByTestSetIDDesc != nil {
305+
pagingSQL = pagingSQL.Order("`tc`.`test_set_id` DESC")
306+
}
307+
case tcFieldPriority:
308+
if req.OrderByPriorityAsc != nil {
309+
pagingSQL = pagingSQL.Order("`tc`.`priority` ASC")
310+
}
311+
if req.OrderByPriorityDesc != nil {
312+
pagingSQL = pagingSQL.Order("`tc`.`priority` DESC")
313+
}
314+
case tcFieldUpdaterID, tcFieldUpdaterIDV2:
315+
if req.OrderByUpdaterIDAsc != nil {
316+
pagingSQL = pagingSQL.Order("`tc`.`updater_id` ASC")
317+
}
318+
if req.OrderByUpdaterIDDesc != nil {
319+
pagingSQL = pagingSQL.Order("`tc`.`updater_id` DESC")
320+
}
321+
case tcFieldUpdatedAt, tcFieldUpdatedAtV2:
322+
if req.OrderByUpdatedAtAsc != nil {
323+
pagingSQL = pagingSQL.Order("`tc`.`updated_at` ASC")
324+
}
325+
if req.OrderByUpdatedAtDesc != nil {
326+
pagingSQL = pagingSQL.Order("`tc`.`updated_at` DESC")
327+
}
328+
}
329+
}
330+
331+
// concurrent do paging and count
332+
var wg sync.WaitGroup
333+
wg.Add(2)
334+
335+
// result
336+
var (
337+
testCases []TestCase
338+
total uint64
339+
pagingErr, countErr error
340+
)
341+
342+
// do paging
343+
go func() {
344+
defer wg.Done()
345+
346+
// offset, limit
347+
offset := (req.PageNo - 1) * req.PageSize
348+
limit := req.PageSize
349+
pagingErr = pagingSQL.Offset(offset).Limit(limit).Find(&testCases).Error
350+
}()
351+
352+
// do count
353+
go func() {
354+
defer wg.Done()
355+
356+
// reset offset & limit before count
357+
countErr = countSQL.Offset(0).Limit(-1).Count(&total).Error
358+
}()
359+
360+
// wait
361+
wg.Wait()
362+
363+
if pagingErr != nil {
364+
return nil, 0, apierrors.ErrPagingTestCases.InternalError(pagingErr)
365+
}
366+
if countErr != nil {
367+
return nil, 0, apierrors.ErrPagingTestCases.InternalError(countErr)
368+
}
369+
370+
return testCases, total, nil
371+
}
372+
373+
func validateTestCasePagingRequest(req apistructs.TestCasePagingRequest) error {
374+
if req.ProjectID == 0 {
375+
return apierrors.ErrPagingTestCases.MissingParameter("projectID")
376+
}
377+
for _, priority := range req.Priorities {
378+
if !priority.IsValid() {
379+
return apierrors.ErrPagingTestCases.InvalidParameter(fmt.Sprintf("priority: %s", priority))
380+
}
381+
}
382+
if req.OrderByPriorityAsc != nil && req.OrderByPriorityDesc != nil {
383+
return apierrors.ErrPagingTestCases.InvalidParameter("order by priority ASC or DESC?")
384+
}
385+
if req.OrderByUpdaterIDAsc != nil && req.OrderByUpdaterIDDesc != nil {
386+
return apierrors.ErrPagingTestCases.InvalidParameter("order by updaterID ASC or DESC?")
387+
}
388+
if req.OrderByUpdatedAtAsc != nil && req.OrderByUpdatedAtDesc != nil {
389+
return apierrors.ErrPagingTestCases.InvalidParameter("order by updatedAt ASC or DESC?")
390+
}
391+
if req.OrderByIDAsc != nil && req.OrderByIDDesc != nil {
392+
return apierrors.ErrPagingTestCases.InvalidParameter("order by id ASC or DESC?")
393+
}
394+
if req.OrderByTestSetIDAsc != nil && req.OrderByTestSetIDDesc != nil {
395+
return apierrors.ErrPagingTestCases.InvalidParameter("order by testSetID ASC or DESC?")
396+
}
397+
if req.OrderByTestSetNameAsc != nil && req.OrderByTestSetNameDesc != nil {
398+
return apierrors.ErrPagingTestCases.InvalidParameter("order by testSetName ASC or DESC?")
399+
}
400+
401+
return nil
402+
}
403+
404+
func setDefaultForTestCasePagingRequest(req *apistructs.TestCasePagingRequest) {
405+
// must order by testSet
406+
if req.OrderByTestSetIDAsc == nil && req.OrderByTestSetIDDesc == nil &&
407+
req.OrderByTestSetNameAsc == nil && req.OrderByTestSetNameDesc == nil {
408+
// default order by `test_set_id` ASC
409+
req.OrderByTestSetIDAsc = &[]bool{true}[0]
410+
req.OrderFields = append(req.OrderFields, tcFieldTestSetID)
411+
}
412+
413+
// set default order inside a testSet
414+
if req.OrderByPriorityAsc == nil && req.OrderByPriorityDesc == nil &&
415+
req.OrderByUpdaterIDAsc == nil && req.OrderByUpdaterIDDesc == nil &&
416+
req.OrderByUpdatedAtAsc == nil && req.OrderByUpdatedAtDesc == nil &&
417+
req.OrderByIDAsc == nil && req.OrderByIDDesc == nil {
418+
// default order by `id` ASC
419+
req.OrderByIDAsc = &[]bool{true}[0]
420+
req.OrderFields = append(req.OrderFields, tcFieldID)
421+
}
422+
423+
if req.PageNo == 0 {
424+
req.PageNo = 1
425+
}
426+
if req.PageSize == 0 {
427+
req.PageSize = 20
428+
}
429+
}

0 commit comments

Comments
 (0)