Skip to content

Commit 40b4c6a

Browse files
RidRisRRidRisR
authored
br: fix compact bugs (#6103)
1 parent 8e21d40 commit 40b4c6a

File tree

14 files changed

+310
-115
lines changed

14 files changed

+310
-115
lines changed

‎cmd/backup-manager/app/compact/manager.go

+40-18
Original file line numberDiff line numberDiff line change
@@ -35,8 +35,9 @@ import (
3535
)
3636

3737
const (
38-
messageCompactionDone = "Finishing compaction."
39-
messageCompactAborted = "Compaction aborted."
38+
messageCompactionDone = "Finishing compaction."
39+
messageCompactionSpawn = "Spawning compaction."
40+
messageCompactAborted = "Compaction aborted."
4041
)
4142

4243
// logLine is line of JSON log.
@@ -82,8 +83,7 @@ func (cm *Manager) kvCtlBin() string {
8283
}
8384

8485
// ProcessBackup used to process the backup logic
85-
func (cm *Manager) ProcessCompact() error {
86-
var err error
86+
func (cm *Manager) ProcessCompact() (err error) {
8787
ctx, cancel := backuputil.GetContextForTerminationSignals(cm.options.ResourceName)
8888
defer cancel()
8989

@@ -152,10 +152,15 @@ func (cm *Manager) runCompaction(ctx context.Context, base64Storage string) (err
152152
cm.statusUpdater.OnStart(ctx, cm.compact)
153153
err = cm.processCompactionLogs(ctx, io.TeeReader(tikvLog, os.Stdout))
154154
if err != nil {
155+
cmd.Process.Kill()
155156
return err
156157
}
157158

158-
return cmd.Wait()
159+
if waitErr := cmd.Wait(); waitErr != nil {
160+
klog.Errorf("Command exited with error: %v", waitErr)
161+
return waitErr
162+
}
163+
return nil
159164
}
160165

161166
func (cm *Manager) compactCmd(ctx context.Context, base64Storage string) *exec.Cmd {
@@ -181,7 +186,7 @@ func (cm *Manager) compactCmd(ctx context.Context, base64Storage string) *exec.C
181186

182187
func (cm *Manager) processCompactionLogs(ctx context.Context, logStream io.Reader) error {
183188
dec := json.NewDecoder(logStream)
184-
189+
currentEndTS, _ := strconv.ParseUint(cm.compact.Status.EndTs, 10, 64)
185190
for dec.More() {
186191
if ctx.Err() != nil {
187192
return ctx.Err()
@@ -198,32 +203,49 @@ func (cm *Manager) processCompactionLogs(ctx context.Context, logStream io.Reade
198203
}
199204
line.Raw = raw
200205

201-
if err := cm.processLogLine(ctx, line); err != nil {
206+
if err := cm.processLogLine(ctx, line, &currentEndTS); err != nil {
202207
return err
203208
}
204209
}
205-
210+
cm.statusUpdater.OnProgress(ctx, cm.compact, nil, strconv.FormatUint(currentEndTS, 10))
206211
return nil
207212
}
208213

209-
func (cm *Manager) processLogLine(ctx context.Context, l logLine) error {
214+
func (cm *Manager) processLogLine(ctx context.Context, l logLine, currentEndTS *uint64) error {
215+
fmtError := func(err error, format string) error {
216+
return errors.Annotatef(err, format, string(l.Raw))
217+
}
218+
210219
switch l.Message {
211220
case messageCompactionDone:
212221
var prog controller.Progress
213222
if err := json.Unmarshal(l.Raw, &prog); err != nil {
214-
return errors.Annotatef(err, "failed to decode progress message: %s", string(l.Raw))
223+
return fmtError(err, "failed to decode progress message: %s")
215224
}
216-
cm.statusUpdater.OnProgress(ctx, cm.compact, prog)
217-
return nil
225+
cm.statusUpdater.OnProgress(ctx, cm.compact, &prog, "")
226+
227+
case messageCompactionSpawn:
228+
var ts struct {
229+
Input_max_ts uint64 `json:"input_max_ts"`
230+
}
231+
if err := json.Unmarshal(l.Raw, &ts); err != nil {
232+
return fmtError(err, "failed to decode input_max_ts message: %s")
233+
}
234+
235+
if ts.Input_max_ts > *currentEndTS {
236+
*currentEndTS = ts.Input_max_ts
237+
}
238+
218239
case messageCompactAborted:
219-
errContainer := struct {
240+
var errMsg struct {
220241
Err string `json:"err"`
221-
}{}
222-
if err := json.Unmarshal(l.Raw, &errContainer); err != nil {
223-
return errors.Annotatef(err, "failed to decode error message: %s", string(l.Raw))
224242
}
225-
return errors.New(errContainer.Err)
243+
if err := json.Unmarshal(l.Raw, &errMsg); err != nil {
244+
return fmtError(err, "failed to decode error message: %s")
245+
}
246+
return errors.New(errMsg.Err)
247+
226248
default:
227-
return nil
228249
}
250+
return nil
229251
}

‎docs/api-references/docs.md

+20-9
Original file line numberDiff line numberDiff line change
@@ -651,13 +651,13 @@ string
651651
</tr>
652652
<tr>
653653
<td>
654-
<code>compactSpan</code></br>
654+
<code>compactInterval</code></br>
655655
<em>
656656
string
657657
</em>
658658
</td>
659659
<td>
660-
<p>CompactSpan is to specify how long backups we want to compact.</p>
660+
<p>CompactInterval is to specify how long backups we want to compact.</p>
661661
</td>
662662
</tr>
663663
<tr>
@@ -4009,13 +4009,13 @@ string
40094009
</tr>
40104010
<tr>
40114011
<td>
4012-
<code>compactSpan</code></br>
4012+
<code>compactInterval</code></br>
40134013
<em>
40144014
string
40154015
</em>
40164016
</td>
40174017
<td>
4018-
<p>CompactSpan is to specify how long backups we want to compact.</p>
4018+
<p>CompactInterval is to specify how long backups we want to compact.</p>
40194019
</td>
40204020
</tr>
40214021
<tr>
@@ -4207,28 +4207,28 @@ Kubernetes meta/v1.Time
42074207
</tr>
42084208
<tr>
42094209
<td>
4210-
<code>lastCompactTs</code></br>
4210+
<code>lastCompactProgress</code></br>
42114211
<em>
42124212
<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta">
42134213
Kubernetes meta/v1.Time
42144214
</a>
42154215
</em>
42164216
</td>
42174217
<td>
4218-
<p>LastCompactTs represents the endTs of the last compact</p>
4218+
<p>LastCompactProgress represents the endTs of the last compact</p>
42194219
</td>
42204220
</tr>
42214221
<tr>
42224222
<td>
4223-
<code>nextCompactEndTs</code></br>
4223+
<code>lastCompactExecutionTs</code></br>
42244224
<em>
42254225
<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#time-v1-meta">
42264226
Kubernetes meta/v1.Time
42274227
</a>
42284228
</em>
42294229
</td>
42304230
<td>
4231-
<p>NextCompactEndTs represents the scheduled endTs of next compact</p>
4231+
<p>LastCompactExecutionTs represents the execution time of the last compact</p>
42324232
</td>
42334233
</tr>
42344234
<tr>
@@ -6186,7 +6186,7 @@ string
61866186
</em>
61876187
</td>
61886188
<td>
6189-
<p>Progress is the progress of the backup</p>
6189+
<p>Progress is the detailed progress of a running backup</p>
61906190
</td>
61916191
</tr>
61926192
<tr>
@@ -6202,6 +6202,17 @@ string
62026202
</tr>
62036203
<tr>
62046204
<td>
6205+
<code>endTs</code></br>
6206+
<em>
6207+
string
6208+
</em>
6209+
</td>
6210+
<td>
6211+
<p>endTs is the real endTs processed by the compact backup</p>
6212+
</td>
6213+
</tr>
6214+
<tr>
6215+
<td>
62056216
<code>backoffRetryStatus</code></br>
62066217
<em>
62076218
<a href="#compactretryrecord">

‎manifests/crd.yaml

+12-6
Original file line numberDiff line numberDiff line change
@@ -4491,7 +4491,7 @@ spec:
44914491
useKMS:
44924492
type: boolean
44934493
type: object
4494-
compactSpan:
4494+
compactInterval:
44954495
type: string
44964496
gcs:
44974497
properties:
@@ -7560,17 +7560,17 @@ spec:
75607560
type: string
75617561
lastCompact:
75627562
type: string
7563-
lastCompactTs:
7563+
lastCompactExecutionTs:
7564+
format: date-time
7565+
type: string
7566+
lastCompactProgress:
75647567
format: date-time
75657568
type: string
75667569
logBackup:
75677570
type: string
75687571
logBackupStartTs:
75697572
format: date-time
75707573
type: string
7571-
nextCompactEndTs:
7572-
format: date-time
7573-
type: string
75747574
type: object
75757575
required:
75767576
- metadata
@@ -10065,7 +10065,11 @@ spec:
1006510065
jsonPath: .status.state
1006610066
name: Status
1006710067
type: string
10068-
- description: The progress of the compact backup
10068+
- description: The endTs of the compact backup
10069+
jsonPath: .status.endTs
10070+
name: EndTs
10071+
type: string
10072+
- description: The detailed progress of a running compact backup
1006910073
jsonPath: .status.progress
1007010074
name: Progress
1007110075
type: string
@@ -12238,6 +12242,8 @@ spec:
1223812242
type: string
1223912243
type: object
1224012244
type: array
12245+
endTs:
12246+
type: string
1224112247
message:
1224212248
type: string
1224312249
progress:

‎manifests/crd/v1/pingcap.com_backupschedules.yaml

+5-5
Original file line numberDiff line numberDiff line change
@@ -4491,7 +4491,7 @@ spec:
44914491
useKMS:
44924492
type: boolean
44934493
type: object
4494-
compactSpan:
4494+
compactInterval:
44954495
type: string
44964496
gcs:
44974497
properties:
@@ -7560,17 +7560,17 @@ spec:
75607560
type: string
75617561
lastCompact:
75627562
type: string
7563-
lastCompactTs:
7563+
lastCompactExecutionTs:
7564+
format: date-time
7565+
type: string
7566+
lastCompactProgress:
75647567
format: date-time
75657568
type: string
75667569
logBackup:
75677570
type: string
75687571
logBackupStartTs:
75697572
format: date-time
75707573
type: string
7571-
nextCompactEndTs:
7572-
format: date-time
7573-
type: string
75747574
type: object
75757575
required:
75767576
- metadata

‎manifests/crd/v1/pingcap.com_compactbackups.yaml

+7-1
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,11 @@ spec:
2121
jsonPath: .status.state
2222
name: Status
2323
type: string
24-
- description: The progress of the compact backup
24+
- description: The endTs of the compact backup
25+
jsonPath: .status.endTs
26+
name: EndTs
27+
type: string
28+
- description: The detailed progress of a running compact backup
2529
jsonPath: .status.progress
2630
name: Progress
2731
type: string
@@ -2194,6 +2198,8 @@ spec:
21942198
type: string
21952199
type: object
21962200
type: array
2201+
endTs:
2202+
type: string
21972203
message:
21982204
type: string
21992205
progress:

‎pkg/apis/pingcap/v1alpha1/openapi_generated.go

+2-2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎pkg/apis/pingcap/v1alpha1/types.go

+11-8
Original file line numberDiff line numberDiff line change
@@ -2564,8 +2564,8 @@ type BackupScheduleSpec struct {
25642564
MaxBackups *int32 `json:"maxBackups,omitempty"`
25652565
// MaxReservedTime is to specify how long backups we want to keep.
25662566
MaxReservedTime *string `json:"maxReservedTime,omitempty"`
2567-
// CompactSpan is to specify how long backups we want to compact.
2568-
CompactSpan *string `json:"compactSpan,omitempty"`
2567+
// CompactInterval is to specify how long backups we want to compact.
2568+
CompactInterval *string `json:"compactInterval,omitempty"`
25692569
// BackupTemplate is the specification of the backup structure to get scheduled.
25702570
BackupTemplate BackupSpec `json:"backupTemplate"`
25712571
// LogBackupTemplate is the specification of the log backup structure to get scheduled.
@@ -2603,10 +2603,10 @@ type BackupScheduleStatus struct {
26032603
LogBackupStartTs *metav1.Time `json:"logBackupStartTs,omitempty"`
26042604
// LastBackupTime represents the last time the backup was successfully created.
26052605
LastBackupTime *metav1.Time `json:"lastBackupTime,omitempty"`
2606-
// LastCompactTs represents the endTs of the last compact
2607-
LastCompactTs *metav1.Time `json:"lastCompactTs,omitempty"`
2608-
// NextCompactEndTs represents the scheduled endTs of next compact
2609-
NextCompactEndTs *metav1.Time `json:"nextCompactEndTs,omitempty"`
2606+
// LastCompactProgress represents the endTs of the last compact
2607+
LastCompactProgress *metav1.Time `json:"lastCompactProgress,omitempty"`
2608+
// LastCompactExecutionTs represents the execution time of the last compact
2609+
LastCompactExecutionTs *metav1.Time `json:"lastCompactExecutionTs,omitempty"`
26102610
// AllBackupCleanTime represents the time when all backup entries are cleaned up
26112611
AllBackupCleanTime *metav1.Time `json:"allBackupCleanTime,omitempty"`
26122612
}
@@ -3493,7 +3493,8 @@ type ScalePolicy struct {
34933493
// +k8s:openapi-gen=true
34943494
// +kubebuilder:resource:shortName="cpbk"
34953495
// +kubebuilder:printcolumn:name="Status",type=string,JSONPath=`.status.state`,description="The current status of the compact backup"
3496-
// +kubebuilder:printcolumn:name="Progress",type=string,JSONPath=`.status.progress`,description="The progress of the compact backup"
3496+
// +kubebuilder:printcolumn:name="EndTs",type=string,JSONPath=`.status.endTs`,description="The endTs of the compact backup"
3497+
// +kubebuilder:printcolumn:name="Progress",type=string,JSONPath=`.status.progress`,description="The detailed progress of a running compact backup"
34973498
// +kubebuilder:printcolumn:name="Message",type=string,JSONPath=`.status.message`,description="The message of the compact backup"
34983499
type CompactBackup struct {
34993500
metav1.TypeMeta `json:",inline"`
@@ -3598,10 +3599,12 @@ type CompactRetryRecord struct {
35983599
type CompactStatus struct {
35993600
// State is the current state of the backup
36003601
State string `json:"state,omitempty"`
3601-
// Progress is the progress of the backup
3602+
// Progress is the detailed progress of a running backup
36023603
Progress string `json:"progress,omitempty"`
36033604
// Message is the error message of the backup
36043605
Message string `json:"message,omitempty"`
3606+
// endTs is the real endTs processed by the compact backup
3607+
EndTs string `json:"endTs,omitempty"`
36053608
// RetryStatus is status of the backoff retry, it will be used when backup pod or job exited unexpectedly
36063609
RetryStatus []CompactRetryRecord `json:"backoffRetryStatus,omitempty"`
36073610
}

‎pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go

+6-6
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)