Skip to content

Commit 78c35ee

Browse files
kichikkichik
authored
feat: Centralized logs with structured JSON (#140)
Resolves #138
1 parent e493780 commit 78c35ee

14 files changed

+424
-1032
lines changed

‎.projen/deps.json

+1-1
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎.projenrc.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ const project = new awscdk.AwsCdkConstructLibrary({
55
author: 'Amir Szekely',
66
authorAddress: '[email protected]',
77
stability: Stability.EXPERIMENTAL,
8-
cdkVersion: '2.85.0', // for no more deprecated nodejs 14 in integration test
8+
cdkVersion: '2.127.0', // 2.85.0 for no more deprecated nodejs 14 in integration test, 2.217.0 for lambda log settings
99
defaultReleaseBranch: 'main',
1010
name: '@cloudsnorkel/cdk-rds-sanitized-snapshots',
1111
repositoryUrl: 'https://github.com/CloudSnorkel/cdk-rds-sanitized-snapshots.git',

‎package.json

+2-2
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/index.ts

+18-12
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import {
66
aws_events_targets as events_targets,
77
aws_iam as iam,
88
aws_kms as kms,
9+
aws_lambda as lambda,
910
aws_logs as logs,
1011
aws_rds as rds,
1112
aws_stepfunctions as stepfunctions,
@@ -141,6 +142,7 @@ export class RdsSanitizedSnapshotter extends Construct {
141142
private readonly sqlScript: string;
142143
private readonly reencrypt: boolean;
143144
private readonly useExistingSnapshot: boolean;
145+
private readonly logGroup: logs.ILogGroup;
144146

145147
private readonly generalTags: {Key: string; Value: string}[];
146148
private readonly finalSnapshotTags: {Key: string; Value: string}[];
@@ -201,6 +203,11 @@ export class RdsSanitizedSnapshotter extends Construct {
201203
this.reencrypt = props.snapshotKey !== undefined;
202204
this.useExistingSnapshot = props.useExistingSnapshot ?? false;
203205

206+
this.logGroup = new logs.LogGroup(this, 'Logs', {
207+
removalPolicy: cdk.RemovalPolicy.DESTROY,
208+
retention: logs.RetentionDays.ONE_MONTH,
209+
});
210+
204211
this.dbClusterArn = cdk.Stack.of(this).formatArn({
205212
service: 'rds',
206213
resource: 'cluster',
@@ -329,7 +336,7 @@ export class RdsSanitizedSnapshotter extends Construct {
329336
}
330337

331338
private dbParametersTask(databaseKey?: kms.IKey) {
332-
const parametersFn = new ParametersFunction(this, 'parameters', { logRetention: logs.RetentionDays.ONE_MONTH });
339+
const parametersFn = new ParametersFunction(this, 'parameters', { logGroup: this.logGroup, loggingFormat: lambda.LoggingFormat.JSON });
333340
const parametersState = new stepfunctions_tasks.LambdaInvoke(this, 'Get Parameters', {
334341
lambdaFunction: parametersFn,
335342
payload: stepfunctions.TaskInput.fromObject({
@@ -362,13 +369,15 @@ export class RdsSanitizedSnapshotter extends Construct {
362369

363370
private findLatestSnapshot(id: string, databaseId: string) {
364371
const findFn = new FindSnapshotFunction(this, 'find-snapshot', {
365-
logRetention: logs.RetentionDays.ONE_MONTH,
372+
logGroup: this.logGroup,
373+
loggingFormat: lambda.LoggingFormat.JSON,
366374
initialPolicy: [
367375
new iam.PolicyStatement({
368376
actions: ['rds:DescribeDBClusterSnapshots', 'rds:DescribeDBSnapshots'],
369377
resources: [this.dbClusterArn, this.dbInstanceArn],
370378
}),
371379
],
380+
timeout: cdk.Duration.minutes(1),
372381
});
373382

374383
let payload = {
@@ -402,7 +411,8 @@ export class RdsSanitizedSnapshotter extends Construct {
402411

403412
private waitForOperation(id: string, resourceType: 'snapshot' | 'cluster' | 'instance', databaseIdentifier: string, snapshotId?: string) {
404413
this.waitFn = this.waitFn ?? new WaitFunction(this, 'wait', {
405-
logRetention: logs.RetentionDays.ONE_MONTH,
414+
logGroup: this.logGroup,
415+
loggingFormat: lambda.LoggingFormat.JSON,
406416
initialPolicy: [
407417
new iam.PolicyStatement({
408418
actions: ['rds:DescribeDBClusters', 'rds:DescribeDBClusterSnapshots', 'rds:DescribeDBSnapshots', 'rds:DescribeDBInstances'],
@@ -545,11 +555,6 @@ export class RdsSanitizedSnapshotter extends Construct {
545555
}
546556

547557
private sanitize(): stepfunctions.IChainable {
548-
const logGroup = new logs.LogGroup(this, 'Logs', {
549-
removalPolicy: cdk.RemovalPolicy.DESTROY,
550-
retention: logs.RetentionDays.ONE_MONTH,
551-
});
552-
553558
const mysqlTask = new ecs.FargateTaskDefinition(this, 'MySQL Task', {
554559
volumes: [{ name: 'config', host: {} }],
555560
});
@@ -561,7 +566,7 @@ export class RdsSanitizedSnapshotter extends Construct {
561566
image: ecs.AssetImage.fromRegistry('public.ecr.aws/docker/library/bash:4-alpine3.15'),
562567
command: ['bash', '-c', `echo "${mycnf}" > ~/.my.cnf && chmod 700 ~/.my.cnf`],
563568
logging: ecs.LogDriver.awsLogs({
564-
logGroup,
569+
logGroup: this.logGroup,
565570
streamPrefix: 'mysql-config',
566571
}),
567572
essential: false,
@@ -571,7 +576,7 @@ export class RdsSanitizedSnapshotter extends Construct {
571576
image: ecs.AssetImage.fromRegistry('public.ecr.aws/lts/mysql:latest'),
572577
command: ['mysql', '-e', this.sqlScript],
573578
logging: ecs.LogDriver.awsLogs({
574-
logGroup,
579+
logGroup: this.logGroup,
575580
streamPrefix: 'mysql-sanitize',
576581
}),
577582
});
@@ -583,7 +588,7 @@ export class RdsSanitizedSnapshotter extends Construct {
583588
image: ecs.AssetImage.fromRegistry('public.ecr.aws/lts/postgres:latest'),
584589
command: ['psql', '-c', this.sqlScript],
585590
logging: ecs.LogDriver.awsLogs({
586-
logGroup,
591+
logGroup: this.logGroup,
587592
streamPrefix: 'psql-sanitize',
588593
}),
589594
});
@@ -726,7 +731,8 @@ export class RdsSanitizedSnapshotter extends Construct {
726731

727732
private deleteOldSnapshots(historyLimit: number) {
728733
const deleteOldFn = new DeleteOldFunction(this, 'delete-old', {
729-
logRetention: logs.RetentionDays.ONE_MONTH,
734+
logGroup: this.logGroup,
735+
loggingFormat: lambda.LoggingFormat.JSON,
730736
timeout: cdk.Duration.minutes(5),
731737
});
732738
deleteOldFn.addToRolePolicy(new iam.PolicyStatement({

‎test/default.integ.snapshot/RDS-Sanitized-Snapshotter-RDS.assets.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
{
2-
"version": "32.0.0",
2+
"version": "36.0.0",
33
"files": {
4-
"43ed48878c85e3ce0324881d365f12acd6963f302748e21cd1257a713163350b": {
4+
"da7ebb4e6882fcf053b199e0417772188b1369e252d27b7cf23ae1743467b6b0": {
55
"source": {
66
"path": "RDS-Sanitized-Snapshotter-RDS.template.json",
77
"packaging": "file"
88
},
99
"destinations": {
1010
"current_account-current_region": {
1111
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
12-
"objectKey": "43ed48878c85e3ce0324881d365f12acd6963f302748e21cd1257a713163350b.json",
12+
"objectKey": "da7ebb4e6882fcf053b199e0417772188b1369e252d27b7cf23ae1743467b6b0.json",
1313
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
1414
}
1515
}

‎test/default.integ.snapshot/RDS-Sanitized-Snapshotter-RDS.template.json

+18-18
Original file line numberDiff line numberDiff line change
@@ -78,27 +78,27 @@
7878
},
7979
"DeleteAutomatedBackups": true,
8080
"Engine": "mysql",
81-
"MasterUsername": {
81+
"MasterUserPassword": {
8282
"Fn::Join": [
8383
"",
8484
[
8585
"{{resolve:secretsmanager:",
8686
{
8787
"Ref": "MySQLInstanceSecret84563F6F"
8888
},
89-
":SecretString:username::}}"
89+
":SecretString:password::}}"
9090
]
9191
]
9292
},
93-
"MasterUserPassword": {
93+
"MasterUsername": {
9494
"Fn::Join": [
9595
"",
9696
[
9797
"{{resolve:secretsmanager:",
9898
{
9999
"Ref": "MySQLInstanceSecret84563F6F"
100100
},
101-
":SecretString:password::}}"
101+
":SecretString:username::}}"
102102
]
103103
]
104104
},
@@ -190,27 +190,27 @@
190190
"Ref": "MySQLClusterSubnets30A4ABD4"
191191
},
192192
"Engine": "aurora-mysql",
193-
"MasterUsername": {
193+
"MasterUserPassword": {
194194
"Fn::Join": [
195195
"",
196196
[
197197
"{{resolve:secretsmanager:",
198198
{
199199
"Ref": "MySQLClusterSecret06B35C31"
200200
},
201-
":SecretString:username::}}"
201+
":SecretString:password::}}"
202202
]
203203
]
204204
},
205-
"MasterUserPassword": {
205+
"MasterUsername": {
206206
"Fn::Join": [
207207
"",
208208
[
209209
"{{resolve:secretsmanager:",
210210
{
211211
"Ref": "MySQLClusterSecret06B35C31"
212212
},
213-
":SecretString:password::}}"
213+
":SecretString:username::}}"
214214
]
215215
]
216216
},
@@ -242,6 +242,7 @@
242242
"Key961B73FD": {
243243
"Type": "AWS::KMS::Key",
244244
"Properties": {
245+
"Description": "RDS sanitize test source key",
245246
"KeyPolicy": {
246247
"Statement": [
247248
{
@@ -269,8 +270,7 @@
269270
}
270271
],
271272
"Version": "2012-10-17"
272-
},
273-
"Description": "RDS sanitize test source key"
273+
}
274274
},
275275
"UpdateReplacePolicy": "Delete",
276276
"DeletionPolicy": "Delete"
@@ -359,27 +359,27 @@
359359
"Arn"
360360
]
361361
},
362-
"MasterUsername": {
362+
"MasterUserPassword": {
363363
"Fn::Join": [
364364
"",
365365
[
366366
"{{resolve:secretsmanager:",
367367
{
368368
"Ref": "PostgresInstanceSecret47B7DD5E"
369369
},
370-
":SecretString:username::}}"
370+
":SecretString:password::}}"
371371
]
372372
]
373373
},
374-
"MasterUserPassword": {
374+
"MasterUsername": {
375375
"Fn::Join": [
376376
"",
377377
[
378378
"{{resolve:secretsmanager:",
379379
{
380380
"Ref": "PostgresInstanceSecret47B7DD5E"
381381
},
382-
":SecretString:password::}}"
382+
":SecretString:username::}}"
383383
]
384384
]
385385
},
@@ -478,27 +478,27 @@
478478
"Arn"
479479
]
480480
},
481-
"MasterUsername": {
481+
"MasterUserPassword": {
482482
"Fn::Join": [
483483
"",
484484
[
485485
"{{resolve:secretsmanager:",
486486
{
487487
"Ref": "PostgresClusterSecretEB353FC9"
488488
},
489-
":SecretString:username::}}"
489+
":SecretString:password::}}"
490490
]
491491
]
492492
},
493-
"MasterUserPassword": {
493+
"MasterUsername": {
494494
"Fn::Join": [
495495
"",
496496
[
497497
"{{resolve:secretsmanager:",
498498
{
499499
"Ref": "PostgresClusterSecretEB353FC9"
500500
},
501-
":SecretString:password::}}"
501+
":SecretString:username::}}"
502502
]
503503
]
504504
},

‎test/default.integ.snapshot/RDS-Sanitized-Snapshotter-SFN.assets.json

+3-16
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
{
2-
"version": "32.0.0",
2+
"version": "36.0.0",
33
"files": {
44
"73ae9c6df09ad4bfbc13c9d4e1f9695a2de8da41bbb8b0037182a8cda9a710c6": {
55
"source": {
@@ -14,19 +14,6 @@
1414
}
1515
}
1616
},
17-
"5fa1330271b8967d9254ba2d4a07144f8acefe8b77e6d6bba38261373a50d5f8": {
18-
"source": {
19-
"path": "asset.5fa1330271b8967d9254ba2d4a07144f8acefe8b77e6d6bba38261373a50d5f8",
20-
"packaging": "zip"
21-
},
22-
"destinations": {
23-
"current_account-current_region": {
24-
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
25-
"objectKey": "5fa1330271b8967d9254ba2d4a07144f8acefe8b77e6d6bba38261373a50d5f8.zip",
26-
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
27-
}
28-
}
29-
},
3017
"66486f7e33c34ceaae0d26eda8231c31f462018de9b6f34e598b3cc0df48b44f": {
3118
"source": {
3219
"path": "asset.66486f7e33c34ceaae0d26eda8231c31f462018de9b6f34e598b3cc0df48b44f.lambda",
@@ -40,15 +27,15 @@
4027
}
4128
}
4229
},
43-
"4d342d6a3f6400ba76ff90f4cd6140b4d7b6f7a8a8c14189a2b75f634544caac": {
30+
"3961ed00ffe563e995f9d7e4fa79705f773ae1376c3cd5b54e937b4bb5f047c9": {
4431
"source": {
4532
"path": "RDS-Sanitized-Snapshotter-SFN.template.json",
4633
"packaging": "file"
4734
},
4835
"destinations": {
4936
"current_account-current_region": {
5037
"bucketName": "cdk-hnb659fds-assets-${AWS::AccountId}-${AWS::Region}",
51-
"objectKey": "4d342d6a3f6400ba76ff90f4cd6140b4d7b6f7a8a8c14189a2b75f634544caac.json",
38+
"objectKey": "3961ed00ffe563e995f9d7e4fa79705f773ae1376c3cd5b54e937b4bb5f047c9.json",
5239
"assumeRoleArn": "arn:${AWS::Partition}:iam::${AWS::AccountId}:role/cdk-hnb659fds-file-publishing-role-${AWS::AccountId}-${AWS::Region}"
5340
}
5441
}

0 commit comments

Comments
 (0)