Skip to content

Commit 8e21d40

Browse files
BornChangerBornChanger
authored
ebs br: fail the ebs volume backup when some backup member tc is invalid (#6087)
1 parent dc33d06 commit 8e21d40

File tree

7 files changed

+81
-13
lines changed

7 files changed

+81
-13
lines changed

‎pkg/backup/backup/backup_manager.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,7 @@ func (bm *backupManager) syncBackupJob(backup *v1alpha1.Backup) error {
116116
}
117117
}
118118

119-
if v1alpha1.IsBackupComplete(backup) || v1alpha1.IsBackupFailed(backup) {
119+
if v1alpha1.IsBackupComplete(backup) || v1alpha1.IsBackupFailed(backup) || v1alpha1.IsBackupInvalid(backup) {
120120
return nil
121121
}
122122

@@ -1161,8 +1161,8 @@ func (bm *backupManager) teardownVolumeBackup(backup *v1alpha1.Backup) (err erro
11611161
return
11621162
}
11631163

1164-
// if backup is failed or complete, just delete job, not modify status
1165-
if v1alpha1.IsBackupFailed(backup) || v1alpha1.IsBackupComplete(backup) {
1164+
// if backup is failed, complete or invalid, just delete job, not modify status
1165+
if v1alpha1.IsBackupFailed(backup) || v1alpha1.IsBackupComplete(backup) || v1alpha1.IsBackupInvalid(backup) {
11661166
return
11671167
}
11681168
backupCondition := v1alpha1.BackupComplete

‎pkg/backup/backupschedule/backup_schedule_manager.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -237,7 +237,7 @@ func (bm *backupScheduleManager) canPerformNextBackup(bs *v1alpha1.BackupSchedul
237237
return fmt.Errorf("backup schedule %s/%s, get backup %s failed, err: %v", ns, bsName, bs.Status.LastBackup, err)
238238
}
239239

240-
if v1alpha1.IsBackupComplete(backup) || (v1alpha1.IsBackupScheduled(backup) && v1alpha1.IsBackupFailed(backup)) {
240+
if v1alpha1.IsBackupComplete(backup) || (v1alpha1.IsBackupScheduled(backup) && v1alpha1.IsBackupFailed(backup)) || v1alpha1.IsBackupInvalid(backup) {
241241
return nil
242242
}
243243
// skip this sync round of the backup schedule and waiting the last backup.
@@ -752,8 +752,8 @@ func separateSnapshotBackupsAndLogBackup(backupsList []*v1alpha1.Backup) ([]*v1a
752752
logBackup = backup
753753
continue
754754
}
755-
// Completed or failed backups will be GC'ed
756-
if !(v1alpha1.IsBackupFailed(backup) || v1alpha1.IsBackupComplete(backup)) {
755+
// Completed, failed or invalid backups will be GC'ed
756+
if !(v1alpha1.IsBackupFailed(backup) || v1alpha1.IsBackupComplete(backup) || v1alpha1.IsBackupInvalid(backup)) {
757757
continue
758758
}
759759
ascBackupList = append(ascBackupList, backup)

‎pkg/backup/backupschedule/backup_schedule_manager_test.go

+11
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,17 @@ func TestManager(t *testing.T) {
7070
g.Expect(err).Should(BeNil())
7171
helper.deleteBackup(bk)
7272

73+
// test last backup invalid state
74+
bk.Status.Conditions = nil
75+
bk.Status.Conditions = append(bk.Status.Conditions, v1alpha1.BackupCondition{
76+
Type: v1alpha1.BackupInvalid,
77+
Status: v1.ConditionTrue,
78+
})
79+
helper.createBackup(bk)
80+
err = m.canPerformNextBackup(bs)
81+
g.Expect(err).Should(BeNil())
82+
helper.deleteBackup(bk)
83+
7384
// test last backup failed state and not scheduled yet
7485
bk.Status.Conditions = nil
7586
bk.Status.Conditions = append(bk.Status.Conditions, v1alpha1.BackupCondition{

‎pkg/controller/backup/backup_controller.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -234,8 +234,8 @@ func (c *Controller) updateBackup(cur interface{}) {
234234
return
235235
}
236236

237-
if v1alpha1.IsBackupScheduled(newBackup) || v1alpha1.IsBackupRunning(newBackup) || v1alpha1.IsBackupPrepared(newBackup) || v1alpha1.IsBackupFailed(newBackup) {
238-
klog.V(4).Infof("backup %s/%s is already Scheduled, Running, Preparing or Failed, skipping.", ns, name)
237+
if v1alpha1.IsBackupScheduled(newBackup) || v1alpha1.IsBackupRunning(newBackup) || v1alpha1.IsBackupPrepared(newBackup) || v1alpha1.IsBackupFailed(newBackup) || v1alpha1.IsBackupInvalid(newBackup) {
238+
klog.V(4).Infof("backup %s/%s is already Scheduled, Running, Preparing, Failed or InValid, skipping.", ns, name)
239239
return
240240
}
241241

‎pkg/fedvolumebackup/backup/backup_manager.go

+11-4
Original file line numberDiff line numberDiff line change
@@ -263,7 +263,7 @@ func (bm *backupManager) waitBackupMemberInitialized(ctx context.Context, volume
263263
Message: errMsg,
264264
}
265265
}
266-
if pingcapv1alpha1.IsBackupFailed(backupMember.backup) {
266+
if pingcapv1alpha1.IsBackupFailed(backupMember.backup) || pingcapv1alpha1.IsBackupInvalid(backupMember.backup) {
267267
errMsg := fmt.Sprintf("backup member %s of cluster %s failed", backupMember.backup.Name, backupMember.k8sClusterName)
268268
return &fedvolumebackup.BRDataPlaneFailedError{
269269
Reason: reasonVolumeBackupMemberFailed,
@@ -343,7 +343,8 @@ func (bm *backupManager) waitVolumeSnapshotsCreated(backupMembers []*volumeBacku
343343
for _, backupMember := range backupMembers {
344344
if pingcapv1alpha1.IsVolumeBackupInitializeFailed(backupMember.backup) ||
345345
pingcapv1alpha1.IsVolumeBackupFailed(backupMember.backup) ||
346-
pingcapv1alpha1.IsBackupFailed(backupMember.backup) {
346+
pingcapv1alpha1.IsBackupFailed(backupMember.backup) ||
347+
pingcapv1alpha1.IsBackupInvalid(backupMember.backup) {
347348
errMsg := fmt.Sprintf("backup member %s of cluster %s failed", backupMember.backup.Name, backupMember.k8sClusterName)
348349
return &fedvolumebackup.BRDataPlaneFailedError{
349350
Reason: reasonVolumeBackupMemberFailed,
@@ -363,7 +364,8 @@ func (bm *backupManager) waitBackupMemberInitializeComplete(volumeBackup *v1alph
363364
for _, backupMember := range backupMembers {
364365
if pingcapv1alpha1.IsVolumeBackupInitializeFailed(backupMember.backup) ||
365366
pingcapv1alpha1.IsVolumeBackupFailed(backupMember.backup) ||
366-
pingcapv1alpha1.IsBackupFailed(backupMember.backup) {
367+
pingcapv1alpha1.IsBackupFailed(backupMember.backup) ||
368+
pingcapv1alpha1.IsBackupInvalid(backupMember.backup) {
367369
errMsg := fmt.Sprintf("backup member %s of cluster %s failed", backupMember.backup.Name, backupMember.k8sClusterName)
368370
return &fedvolumebackup.BRDataPlaneFailedError{
369371
Reason: reasonVolumeBackupMemberFailed,
@@ -385,7 +387,8 @@ func (bm *backupManager) waitVolumeSnapshotsComplete(backupMembers []*volumeBack
385387
for _, backupMember := range backupMembers {
386388
if pingcapv1alpha1.IsVolumeBackupInitializeFailed(backupMember.backup) ||
387389
pingcapv1alpha1.IsVolumeBackupFailed(backupMember.backup) ||
388-
pingcapv1alpha1.IsBackupFailed(backupMember.backup) {
390+
pingcapv1alpha1.IsBackupFailed(backupMember.backup) ||
391+
pingcapv1alpha1.IsBackupInvalid(backupMember.backup) {
389392
errMsg := fmt.Sprintf("backup member %s of cluster %s failed", backupMember.backup.Name, backupMember.k8sClusterName)
390393
return &fedvolumebackup.BRDataPlaneFailedError{
391394
Reason: reasonVolumeBackupMemberFailed,
@@ -437,6 +440,10 @@ func (bm *backupManager) waitVolumeBackupComplete(ctx context.Context, volumeBac
437440
failedBackups = append(failedBackups, backupMember)
438441
klog.Errorf("VolumeBackup %s/%s backup member %s of cluster %s is failed",
439442
volumeBackup.Namespace, volumeBackup.Name, backupMember.backup.Name, backupMember.k8sClusterName)
443+
} else if pingcapv1alpha1.IsBackupInvalid(backupMember.backup) {
444+
failedBackups = append(failedBackups, backupMember)
445+
klog.Errorf("VolumeBackup %s/%s backup member %s of cluster %s is invalid",
446+
volumeBackup.Namespace, volumeBackup.Name, backupMember.backup.Name, backupMember.k8sClusterName)
440447
} else if !pingcapv1alpha1.IsBackupComplete(backupMember.backup) {
441448
isBackupRunning = true
442449
klog.Infof(

‎pkg/fedvolumebackup/backup/backup_test.go

+37
Original file line numberDiff line numberDiff line change
@@ -294,6 +294,17 @@ func (h *helper) setDataPlaneFailed(ctx context.Context) {
294294
h.g.Expect(err).To(gomega.BeNil())
295295
}
296296

297+
func (h *helper) setDataPlaneFailed4Invalid(ctx context.Context) {
298+
backupMember1, err := h.dataPlaneClient1.PingcapV1alpha1().Backups(fakeTcNamespace1).Get(ctx, h.backupMemberName1, metav1.GetOptions{})
299+
h.g.Expect(err).To(gomega.BeNil())
300+
pingcapv1alpha1.UpdateBackupCondition(&backupMember1.Status, &pingcapv1alpha1.BackupCondition{
301+
Status: corev1.ConditionTrue,
302+
Type: pingcapv1alpha1.BackupInvalid,
303+
})
304+
_, err = h.dataPlaneClient1.PingcapV1alpha1().Backups(fakeTcNamespace1).UpdateStatus(ctx, backupMember1, metav1.UpdateOptions{})
305+
h.g.Expect(err).To(gomega.BeNil())
306+
}
307+
297308
func (h *helper) setAllDataPlanesFailed(ctx context.Context) {
298309
backupMember1, err := h.dataPlaneClient1.PingcapV1alpha1().Backups(fakeTcNamespace1).Get(ctx, h.backupMemberName1, metav1.GetOptions{})
299310
h.g.Expect(err).To(gomega.BeNil())
@@ -450,6 +461,32 @@ func TestVolumeBackupVolumeFailed(t *testing.T) {
450461
h.assertFailed(volumeBackup)
451462
}
452463

464+
func TestVolumeBackupFailed4Invalid(t *testing.T) {
465+
ctx := context.Background()
466+
backupName := "backup-2"
467+
backupNamespace := "ns-2"
468+
h := newHelper(t, backupName, backupNamespace)
469+
470+
// create volume backup
471+
volumeBackup := h.createVolumeBackup(ctx)
472+
473+
// run initialize phase
474+
err := h.bm.Sync(volumeBackup)
475+
h.g.Expect(err).To(gomega.BeNil())
476+
h.assertRunInitialize(ctx, volumeBackup)
477+
478+
// invalid member backup, run teardown
479+
h.setDataPlaneFailed4Invalid(ctx)
480+
err = h.bm.Sync(volumeBackup)
481+
h.g.Expect(err).To(gomega.BeNil())
482+
h.assertRunTeardown(ctx, volumeBackup, true)
483+
484+
// volume backup failed
485+
err = h.bm.Sync(volumeBackup)
486+
h.g.Expect(err).To(gomega.BeNil())
487+
h.assertFailed(volumeBackup)
488+
}
489+
453490
func generateVolumeBackup(backupName, backupNamespace string) *v1alpha1.VolumeBackup {
454491
return &v1alpha1.VolumeBackup{
455492
ObjectMeta: metav1.ObjectMeta{

‎pkg/fedvolumebackup/backupschedule/backup_schedule_manager_test.go

+14-1
Original file line numberDiff line numberDiff line change
@@ -398,7 +398,7 @@ func TestCalculateExpiredBackups(t *testing.T) {
398398
// 3 backups should be deleted
399399
{
400400
backups: []*v1alpha1.VolumeBackup{
401-
fakeBackup(&last3Day),
401+
fakeFailedBackup(&last3Day),
402402
fakeBackup(&last2Day),
403403
fakeBackup(&last1Day),
404404
fakeBackup(&last10Min),
@@ -573,6 +573,19 @@ func fakeBackup(ts *time.Time) *v1alpha1.VolumeBackup {
573573
return backup
574574
}
575575

576+
func fakeFailedBackup(ts *time.Time) *v1alpha1.VolumeBackup {
577+
backup := &v1alpha1.VolumeBackup{}
578+
if ts == nil {
579+
return backup
580+
}
581+
backup.CreationTimestamp = metav1.Time{Time: *ts}
582+
backup.Status.Conditions = append(backup.Status.Conditions, v1alpha1.VolumeBackupCondition{
583+
Type: v1alpha1.VolumeBackupFailed,
584+
Status: v1.ConditionTrue,
585+
})
586+
return backup
587+
}
588+
576589
func getTSOStr(ts int64) string {
577590
tso := getTSO(ts)
578591
return strconv.FormatUint(tso, 10)

0 commit comments

Comments
 (0)