Skip to content

Commit e923a0f

Browse files
KSDaemonKSDaemon
authored
feat(schema-compiler): Use LRUCache for js data models compiled vm.Scripts (#9424)
* fix types * update lru-cache to the latest * fix lru-cache after upgrade * introduce compiledScriptCache * add proactive cache cleanup * call compilerCache.clear() on shutdown
1 parent a4fc5ec commit e923a0f

File tree

12 files changed

+80
-42
lines changed

12 files changed

+80
-42
lines changed

‎packages/cubejs-query-orchestrator/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
"@cubejs-backend/shared": "1.2.30",
3535
"csv-write-stream": "^2.0.0",
3636
"generic-pool": "^3.8.2",
37-
"lru-cache": "^5.1.1",
37+
"lru-cache": "^11.1.0",
3838
"ramda": "^0.27.2"
3939
},
4040
"devDependencies": {

‎packages/cubejs-query-orchestrator/src/orchestrator/PreAggregations.ts

+4-4
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { getEnv, } from '@cubejs-backend/shared';
44

55
import { BaseDriver, InlineTable, } from '@cubejs-backend/base-driver';
66
import { CubeStoreDriver } from '@cubejs-backend/cubestore-driver';
7-
import LRUCache from 'lru-cache';
7+
import { LRUCache } from 'lru-cache';
88

99
import { PreAggTableToTempTable, Query, QueryBody, QueryCache, QueryWithParams } from './QueryCache';
1010
import { DriverFactory, DriverFactoryByDataSource } from './DriverFactory';
@@ -282,8 +282,8 @@ export class PreAggregations {
282282
this.getQueueEventsBus = options.getQueueEventsBus;
283283
this.touchCache = new LRUCache({
284284
max: getEnv('touchPreAggregationCacheMaxCount'),
285-
maxAge: getEnv('touchPreAggregationCacheMaxAge') * 1000,
286-
stale: false,
285+
ttl: getEnv('touchPreAggregationCacheMaxAge') * 1000,
286+
allowStale: false,
287287
updateAgeOnGet: false
288288
});
289289
}
@@ -330,7 +330,7 @@ export class PreAggregations {
330330
this.touchTablePersistTime
331331
);
332332
} catch (e: unknown) {
333-
this.touchCache.del(tableName);
333+
this.touchCache.delete(tableName);
334334

335335
throw e;
336336
}

‎packages/cubejs-query-orchestrator/src/orchestrator/QueryCache.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import crypto from 'crypto';
22
import csvWriter from 'csv-write-stream';
3-
import LRUCache from 'lru-cache';
3+
import { LRUCache } from 'lru-cache';
44
import { pipeline } from 'stream';
55
import { getEnv, MaybeCancelablePromise, streamToArray } from '@cubejs-backend/shared';
66
import { CubeStoreCacheDriver, CubeStoreDriver } from '@cubejs-backend/cubestore-driver';
@@ -909,7 +909,7 @@ export class QueryCache {
909909
inMemoryValue.renewalKey !== renewalKey
910910
) || renewedAgo > expiration * 1000 || renewedAgo > inMemoryCacheDisablePeriod
911911
) {
912-
this.memoryCache.del(redisKey);
912+
this.memoryCache.delete(redisKey);
913913
} else {
914914
this.logger('Found in memory cache entry', {
915915
cacheKey,

‎packages/cubejs-schema-compiler/package.json

+1-2
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@
4949
"inflection": "^1.12.0",
5050
"joi": "^17.8.3",
5151
"js-yaml": "^4.1.0",
52-
"lru-cache": "^5.1.1",
52+
"lru-cache": "^11.1.0",
5353
"moment-timezone": "^0.5.46",
5454
"node-dijkstra": "^2.5.0",
5555
"ramda": "^0.27.2",
@@ -66,7 +66,6 @@
6666
"@types/babel__traverse": "^7.20.5",
6767
"@types/inflection": "^1.5.28",
6868
"@types/jest": "^27",
69-
"@types/lru-cache": "^5.1.0",
7069
"@types/node": "^18",
7170
"@types/ramda": "^0.27.34",
7271
"@types/sqlstring": "^2.3.0",

‎packages/cubejs-schema-compiler/src/compiler/CompilerCache.ts

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
import LRUCache from 'lru-cache';
1+
import { LRUCache } from 'lru-cache';
22
import { QueryCache } from '../adapter/QueryCache';
33

44
export class CompilerCache extends QueryCache {
@@ -11,13 +11,13 @@ export class CompilerCache extends QueryCache {
1111

1212
this.queryCache = new LRUCache({
1313
max: maxQueryCacheSize || 10000,
14-
maxAge: (maxQueryCacheAge * 1000) || 1000 * 60 * 10,
14+
ttl: (maxQueryCacheAge * 1000) || 1000 * 60 * 10,
1515
updateAgeOnGet: true
1616
});
1717

1818
this.rbacCache = new LRUCache({
1919
max: 10000,
20-
maxAge: 1000 * 60 * 5, // 5 minutes
20+
ttl: 1000 * 60 * 5, // 5 minutes
2121
});
2222
}
2323

‎packages/cubejs-schema-compiler/src/compiler/DataSchemaCompiler.js

+17-1
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import crypto from 'crypto';
12
import vm from 'vm';
23
import fs from 'fs';
34
import path from 'path';
@@ -47,6 +48,7 @@ export class DataSchemaCompiler {
4748
this.pythonContext = null;
4849
this.workerPool = null;
4950
this.compilerId = options.compilerId;
51+
this.compiledScriptCache = options.compiledScriptCache;
5052
}
5153

5254
compileObjects(compileServices, objects, errorsReport) {
@@ -370,6 +372,18 @@ export class DataSchemaCompiler {
370372
}
371373
}
372374

375+
getJsScript(file) {
376+
const cacheKey = crypto.createHash('md5').update(JSON.stringify(file.content)).digest('hex');
377+
378+
if (this.compiledScriptCache.has(cacheKey)) {
379+
return this.compiledScriptCache.get(cacheKey);
380+
}
381+
382+
const script = new vm.Script(file.content, { filename: file.fileName, timeout: 15000 });
383+
this.compiledScriptCache.set(cacheKey, script);
384+
return script;
385+
}
386+
373387
compileJsFile(file, errorsReport, cubes, contexts, exports, asyncModules, toCompile, compiledFiles, { doSyntaxCheck } = { doSyntaxCheck: false }) {
374388
if (doSyntaxCheck) {
375389
// There is no need to run syntax check for data model files
@@ -382,7 +396,9 @@ export class DataSchemaCompiler {
382396
}
383397

384398
try {
385-
vm.runInNewContext(file.content, {
399+
const script = this.getJsScript(file);
400+
401+
script.runInNewContext({
386402
view: (name, cube) => (
387403
!cube ?
388404
this.cubeFactory({ ...name, fileName: file.fileName, isView: true }) :

‎packages/cubejs-schema-compiler/src/compiler/PrepareCompiler.ts

+6
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
import { SchemaFileRepository } from '@cubejs-backend/shared';
22
import { NativeInstance } from '@cubejs-backend/native';
33
import { v4 as uuidv4 } from 'uuid';
4+
import { LRUCache } from 'lru-cache';
5+
import vm from 'vm';
46

57
import { CubeValidator } from './CubeValidator';
68
import { DataSchemaCompiler } from './DataSchemaCompiler';
@@ -32,6 +34,7 @@ export type PrepareCompilerOptions = {
3234
standalone?: boolean;
3335
headCommitId?: string;
3436
adapter?: string;
37+
compiledScriptCache?: LRUCache<string, vm.Script>;
3538
};
3639

3740
export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareCompilerOptions = {}) => {
@@ -49,6 +52,8 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
4952
const compilerCache = new CompilerCache({ maxQueryCacheSize, maxQueryCacheAge });
5053
const yamlCompiler = new YamlCompiler(cubeSymbols, cubeDictionary, nativeInstance, viewCompiler);
5154

55+
const compiledScriptCache = options.compiledScriptCache || new LRUCache<string, vm.Script>({ max: 250 });
56+
5257
const transpilers: TranspilerInterface[] = [
5358
new ValidationTranspiler(),
5459
new ImportExportTranspiler(),
@@ -66,6 +71,7 @@ export const prepareCompiler = (repo: SchemaFileRepository, options: PrepareComp
6671
preTranspileCubeCompilers: [cubeSymbols, cubeValidator],
6772
transpilers,
6873
viewCompilationGate,
74+
compiledScriptCache,
6975
viewCompilers: [viewCompiler],
7076
cubeCompilers: [cubeEvaluator, joinGraph, metaTransformer],
7177
contextCompilers: [contextEvaluator],

‎packages/cubejs-server-core/package.json

+1-2
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@
4646
"joi": "^17.8.3",
4747
"jsonwebtoken": "^9.0.2",
4848
"lodash.clonedeep": "^4.5.0",
49-
"lru-cache": "^5.1.1",
49+
"lru-cache": "^11.1.0",
5050
"moment": "^2.29.1",
5151
"node-fetch": "^2.6.0",
5252
"p-limit": "^3.1.0",
@@ -67,7 +67,6 @@
6767
"@types/fs-extra": "^9.0.8",
6868
"@types/jest": "^27",
6969
"@types/jsonwebtoken": "^9.0.2",
70-
"@types/lru-cache": "^5.1.0",
7170
"@types/node": "^18",
7271
"@types/node-fetch": "^2.5.7",
7372
"@types/ramda": "^0.27.34",

‎packages/cubejs-server-core/src/core/CompilerApi.js

+22-1
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,8 @@
11
import crypto from 'crypto';
22
import R from 'ramda';
33
import { createQuery, compile, queryClass, PreAggregations, QueryFactory } from '@cubejs-backend/schema-compiler';
4-
import { v4 as uuidv4, parse as uuidParse, stringify as uuidStringify } from 'uuid';
4+
import { v4 as uuidv4, parse as uuidParse } from 'uuid';
5+
import { LRUCache } from 'lru-cache';
56
import { NativeInstance } from '@cubejs-backend/native';
67

78
export class CompilerApi {
@@ -29,6 +30,25 @@ export class CompilerApi {
2930
this.sqlCache = options.sqlCache;
3031
this.standalone = options.standalone;
3132
this.nativeInstance = this.createNativeInstance();
33+
this.compiledScriptCache = new LRUCache({
34+
max: options.compilerCacheSize || 250,
35+
ttl: options.maxCompilerCacheKeepAlive,
36+
updateAgeOnGet: options.updateCompilerCacheKeepAlive
37+
});
38+
39+
// proactively free up old cache values occasionally
40+
if (this.options.maxCompilerCacheKeepAlive) {
41+
this.compiledScriptCacheInterval = setInterval(
42+
() => this.compiledScriptCache.purgeStale(),
43+
this.options.maxCompilerCacheKeepAlive
44+
);
45+
}
46+
}
47+
48+
dispose() {
49+
if (this.compiledScriptCacheInterval) {
50+
clearInterval(this.compiledScriptCacheInterval);
51+
}
3252
}
3353

3454
setGraphQLSchema(schema) {
@@ -83,6 +103,7 @@ export class CompilerApi {
83103
allowJsDuplicatePropsInSchema: this.allowJsDuplicatePropsInSchema,
84104
standalone: this.standalone,
85105
nativeInstance: this.nativeInstance,
106+
compiledScriptCache: this.compiledScriptCache,
86107
});
87108
this.queryFactory = await this.createQueryFactory(compilers);
88109

‎packages/cubejs-server-core/src/core/server.ts

+14-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
/* eslint-disable global-require,no-return-assign */
22
import crypto from 'crypto';
33
import fs from 'fs-extra';
4-
import LRUCache from 'lru-cache';
4+
import { LRUCache } from 'lru-cache';
55
import isDocker from 'is-docker';
66
import pLimit from 'p-limit';
77

@@ -203,8 +203,10 @@ export class CubejsServerCore {
203203

204204
this.compilerCache = new LRUCache<string, CompilerApi>({
205205
max: this.options.compilerCacheSize || 250,
206-
maxAge: this.options.maxCompilerCacheKeepAlive,
207-
updateAgeOnGet: this.options.updateCompilerCacheKeepAlive
206+
ttl: this.options.maxCompilerCacheKeepAlive,
207+
updateAgeOnGet: this.options.updateCompilerCacheKeepAlive,
208+
// needed to clear the setInterval timer for proactive cache internal cleanups
209+
dispose: (v) => v.dispose(),
208210
});
209211

210212
if (this.options.contextToAppId) {
@@ -224,7 +226,7 @@ export class CubejsServerCore {
224226
// proactively free up old cache values occasionally
225227
if (this.options.maxCompilerCacheKeepAlive) {
226228
this.maxCompilerCacheKeep = setInterval(
227-
() => this.compilerCache.prune(),
229+
() => this.compilerCache.purgeStale(),
228230
this.options.maxCompilerCacheKeepAlive
229231
);
230232
}
@@ -554,7 +556,7 @@ export class CubejsServerCore {
554556
await this.orchestratorStorage.releaseConnections();
555557

556558
this.orchestratorStorage.clear();
557-
this.compilerCache.reset();
559+
this.compilerCache.clear();
558560

559561
this.reloadEnvVariables();
560562

@@ -714,6 +716,9 @@ export class CubejsServerCore {
714716
standalone: this.standalone,
715717
allowNodeRequire: options.allowNodeRequire,
716718
fastReload: options.fastReload || getEnv('fastReload'),
719+
compilerCacheSize: this.options.compilerCacheSize || 250,
720+
maxCompilerCacheKeepAlive: this.options.maxCompilerCacheKeepAlive,
721+
updateCompilerCacheKeepAlive: this.options.updateCompilerCacheKeepAlive
717722
},
718723
);
719724
}
@@ -871,6 +876,8 @@ export class CubejsServerCore {
871876
clearInterval(this.maxCompilerCacheKeep);
872877
}
873878

879+
this.compilerCache.clear();
880+
874881
if (this.scheduledRefreshTimerInterval) {
875882
await this.scheduledRefreshTimerInterval.cancel();
876883
}
@@ -914,6 +921,8 @@ export class CubejsServerCore {
914921
};
915922

916923
public async shutdown() {
924+
this.compilerCache.clear();
925+
917926
if (this.devServer) {
918927
if (!process.env.CI) {
919928
process.removeListener('uncaughtException', this.onUncaughtException);

‎packages/cubejs-templates/src/PackageFetcher.ts

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ export class PackageFetcher {
1616

1717
protected repoArchivePath: string;
1818

19-
public constructor(private repo: Repository) {
19+
public constructor(private readonly repo: Repository) {
2020
this.tmpFolderPath = path.resolve('.', 'node_modules', '.tmp');
2121

2222
this.init();
@@ -53,7 +53,7 @@ export class PackageFetcher {
5353
(await proxyFetch(url)).body.pipe(writer);
5454

5555
return new Promise((resolve, reject) => {
56-
writer.on('finish', resolve);
56+
writer.on('finish', resolve as () => void);
5757
writer.on('error', reject);
5858
});
5959
}

‎yarn.lock

+7-19
Original file line numberDiff line numberDiff line change
@@ -9601,11 +9601,6 @@
96019601
resolved "https://registry.yarnpkg.com/@types/jwk-to-pem/-/jwk-to-pem-2.0.1.tgz#ba6949f447e02cb7bebf101551e3a4dea5f4fde4"
96029602
integrity sha512-QXmRPhR/LPzvXBHTPfG2BBfMTkNLUD7NyRcPft8m5xFCeANa1BZyLgT0Gw+OxdWx6i1WCpT27EqyggP4UUHMrA==
96039603

9604-
"@types/lru-cache@^5.1.0":
9605-
version "5.1.1"
9606-
resolved "https://registry.yarnpkg.com/@types/lru-cache/-/lru-cache-5.1.1.tgz#c48c2e27b65d2a153b19bfc1a317e30872e01eef"
9607-
integrity sha512-ssE3Vlrys7sdIzs5LOxCzTVMsU7i9oa/IaW92wF32JFb3CVczqOkru2xspuKczHEbG3nvmPY7IFqVmGGHdNbYw==
9608-
96099604
"@types/mime@^1":
96109605
version "1.3.2"
96119606
resolved "https://registry.yarnpkg.com/@types/mime/-/mime-1.3.2.tgz#93e25bf9ee75fe0fd80b594bc4feb0e862111b5a"
@@ -17154,7 +17149,7 @@ generate-object-property@^1.0.0:
1715417149
dependencies:
1715517150
is-property "^1.0.0"
1715617151

17157-
generic-pool@*, generic-pool@^3.6.0, generic-pool@^3.8.2:
17152+
generic-pool@*, generic-pool@^3.8.2:
1715817153
version "3.9.0"
1715917154
resolved "https://registry.yarnpkg.com/generic-pool/-/generic-pool-3.9.0.tgz#36f4a678e963f4fdb8707eab050823abc4e8f5e4"
1716017155
integrity sha512-hymDOu5B53XvN4QT9dBmZxPX4CWhBPPLguTZ9MMFeFa/Kg0xWVfylOVNlJji/E7yTZWFd/q9GO5TxDLq156D7g==
@@ -20685,16 +20680,16 @@ lower-case@^2.0.2:
2068520680
dependencies:
2068620681
tslib "^2.0.3"
2068720682

20688-
lru-cache@^10.0.1:
20689-
version "10.1.0"
20690-
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.1.0.tgz#2098d41c2dc56500e6c88584aa656c84de7d0484"
20691-
integrity sha512-/1clY/ui8CzjKFyjdvwPWJUYKiFVXG2I2cY0ssG7h4+hwk+XOIX7ZSG9Q7TW8TW3Kp3BUSqgFWBLgL4PJ+Blag==
20692-
20693-
lru-cache@^10.2.0, lru-cache@^10.2.2:
20683+
lru-cache@^10.0.1, lru-cache@^10.2.0, lru-cache@^10.2.2, "lru-cache@^9.1.1 || ^10.0.0":
2069420684
version "10.4.3"
2069520685
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.4.3.tgz#410fc8a17b70e598013df257c2446b7f3383f119"
2069620686
integrity sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==
2069720687

20688+
lru-cache@^11.1.0:
20689+
version "11.1.0"
20690+
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-11.1.0.tgz#afafb060607108132dbc1cf8ae661afb69486117"
20691+
integrity sha512-QIXZUBJUx+2zHUdQujWejBkcD9+cs94tLn0+YL8UrCh+D5sCXZ4c7LaEH48pNwRY3MLDgqUFyhlCyjJPf1WP0A==
20692+
2069820693
lru-cache@^4.0.1, lru-cache@^4.1.2, lru-cache@^4.1.5:
2069920694
version "4.1.5"
2070020695
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-4.1.5.tgz#8bbe50ea85bed59bc9e33dcab8235ee9bcf443cd"
@@ -20722,13 +20717,6 @@ lru-cache@^7.14.1, lru-cache@^7.7.1:
2072220717
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-7.18.3.tgz#f793896e0fd0e954a59dfdd82f0773808df6aa89"
2072320718
integrity sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA==
2072420719

20725-
"lru-cache@^9.1.1 || ^10.0.0":
20726-
version "10.0.2"
20727-
resolved "https://registry.yarnpkg.com/lru-cache/-/lru-cache-10.0.2.tgz#34504678cc3266b09b8dfd6fab4e1515258271b7"
20728-
integrity sha512-Yj9mA8fPiVgOUpByoTZO5pNrcl5Yk37FcSHsUINpAsaBIEZIuqcCclDZJCVxqQShDsmYX8QG63svJiTbOATZwg==
20729-
dependencies:
20730-
semver "^7.3.5"
20731-
2073220720
lru.min@^1.0.0:
2073320721
version "1.1.1"
2073420722
resolved "https://registry.yarnpkg.com/lru.min/-/lru.min-1.1.1.tgz#146e01e3a183fa7ba51049175de04667d5701f0e"

0 commit comments

Comments
 (0)