feat(runtime): impl cloud function import and cache function (#1005)

* feat(runtime): impl func import and cache func

* feat: opt init and cache

* feat: opt func cache & add pkg types

* style: opt style

Author: skyoct

runtimes/nodejs/package.json

@@ -12,7 +12,8 @@
     "watch": "tsc -p tsconfig.json -w",
     "prepublishOnly": "npm run build",
     "trace-gc": "node --trace_gc --trace_gc_verbose ./dist/index.js",
-    "init": "node ./dist/init.js"
+    "init": "node ./dist/init.js",
+    "prettier": "npx prettier --write ./src"
   },
   "keywords": [
     "laf",

runtimes/nodejs/src/config.ts

@@ -1,9 +1,9 @@
-import * as dotenv from "dotenv";
+import * as dotenv from 'dotenv'
 
 /**
  * parse environment vars from the `.env` file if existing
  */
-dotenv.config();
+dotenv.config()
 
 /**
  * configuration management
@@ -13,80 +13,78 @@ export default class Config {
    * mongodb connection configuration
    */
   static get DB_URI() {
-    if (!process.env["DB_URI"]) {
-      throw new Error("env: `DB_URI` is missing");
+    if (!process.env['DB_URI']) {
+      throw new Error('env: `DB_URI` is missing')
     }
-    return process.env["DB_URI"];
+    return process.env['DB_URI']
   }
 
   /**
    * the server secret salt, mainly used for generating tokens
    */
   static get SERVER_SECRET(): string {
-    const secret_salt = process.env["SERVER_SECRET"];
+    const secret_salt = process.env['SERVER_SECRET']
     if (!secret_salt) {
-      throw new Error("env: `SERVER_SECRET` is missing");
+      throw new Error('env: `SERVER_SECRET` is missing')
     }
-    return secret_salt;
+    return secret_salt
   }
 
   /**
    * the logger level : 'fatal', 'error', 'warning', 'info', 'debug', 'trace'
    */
   static get LOG_LEVEL():
-    | "fatal"
-    | "error"
-    | "warning"
-    | "info"
-    | "debug"
-    | "trace" {
-    return (
-      (process.env["LOG_LEVEL"] as any) ?? (this.isProd ? "info" : "debug")
-    );
+    | 'fatal'
+    | 'error'
+    | 'warning'
+    | 'info'
+    | 'debug'
+    | 'trace' {
+    return (process.env['LOG_LEVEL'] as any) ?? (this.isProd ? 'info' : 'debug')
   }
 
   /**
    * the serving port, default is 8000
    */
   static get PORT(): number {
-    return (process.env.PORT ?? 8000) as number;
+    return (process.env.PORT ?? 8000) as number
   }
 
   /**
    * in production deploy or not
    */
   static get isProd(): boolean {
-    return process.env.NODE_ENV === "production";
+    return process.env.NODE_ENV === 'production'
   }
 
   /**
    * Expired time of function logs, in seconds
    */
   static get FUNCTION_LOG_EXPIRED_TIME(): number {
-    return (process.env.FUNCTION_LOG_EXPIRED_TIME ?? 3600 * 24 * 3) as number;
+    return (process.env.FUNCTION_LOG_EXPIRED_TIME ?? 3600 * 24 * 3) as number
   }
 
   static get RUNTIME_IMAGE(): string {
-    return process.env.RUNTIME_IMAGE;
+    return process.env.RUNTIME_IMAGE
   }
 
   static get RUNTIME_VERSION(): string {
-    return require("../package.json")?.version;
+    return require('../package.json')?.version
   }
 
   static get APP_ID(): string {
-    return process.env.APP_ID;
+    return process.env.APP_ID
   }
 
   static get NPM_INSTALL_FLAGS(): string {
-    return process.env.NPM_INSTALL_FLAGS || "";
+    return process.env.NPM_INSTALL_FLAGS || ''
   }
 
   static get REQUEST_LIMIT_SIZE(): string {
-    return process.env.REQUEST_LIMIT_SIZE || "10mb";
+    return process.env.REQUEST_LIMIT_SIZE || '10mb'
   }
 
   static get PACKAGES(): string {
-    return process.env.PACKAGES || "";
+    return process.env.PACKAGES || ''
   }
 }

runtimes/nodejs/src/constants.ts

@@ -1,4 +1,3 @@
-
 /**
  * Collection names
  */
@@ -7,8 +6,7 @@ export const POLICY_COLLECTION = '__policies__'
 export const FUNCTION_LOG_COLLECTION = '__function_logs__'
 export const CONFIG_COLLECTION = '__config__'
 
-
 export const WEBSOCKET_FUNCTION_NAME = '__websocket__'
 export const INTERCEPTOR_FUNCTION_NAME = '__interceptor__'
 export const DEFAULT_FUNCTION_NAME = '__default__'
-export const INIT_FUNCTION_NAME = '__init__'
+export const INIT_FUNCTION_NAME = '__init__'

runtimes/nodejs/src/db.ts

@@ -2,15 +2,14 @@
  * @Author: Maslow<[email protected]>
  * @Date: 2021-08-16 15:29:15
  * @LastEditTime: 2022-02-03 00:42:33
- * @Description: 
+ * @Description:
  */
 
-
 import { MongoAccessor } from 'database-proxy'
 import Config from './config'
 import { createLogger, logger } from './support/logger'
 import * as mongodb_uri from 'mongodb-uri'
-
+import { FunctionCache } from './support/function-engine/cache'
 
 /**
  * Database Management
@@ -34,18 +33,20 @@ export class DatabaseAgent {
 
   /**
    * Create MongoAccessor instance
-   * @returns 
+   * @returns
    */
   private static _createAccessor() {
     const { database } = mongodb_uri.parse(Config.DB_URI)
     const accessor = new MongoAccessor(database, Config.DB_URI)
 
     accessor.setLogger(createLogger('accessor', 'warning'))
-    accessor.init()
+    accessor
+      .init()
       .then(async () => {
         logger.info('db connected')
+        FunctionCache.initialize()
       })
-      .catch(error => {
+      .catch((error) => {
         logger.error(error)
         setTimeout(() => process.exit(101), 0)
       })

runtimes/nodejs/src/handler/db-proxy.ts

@@ -1,4 +1,3 @@
-
 import { Response } from 'express'
 import { Proxy } from 'database-proxy'
 import Config from '../config'
@@ -7,7 +6,6 @@ import { logger } from '../support/logger'
 import { PolicyAgent } from '../support/policy'
 import { IRequest } from '../support/types'
 
-
 export async function handleDatabaseProxy(req: IRequest, res: Response) {
   const accessor = DatabaseAgent.accessor
 
@@ -37,7 +35,7 @@ export async function handleDatabaseProxy(req: IRequest, res: Response) {
     return res.status(403).send({
       code: 'permission denied',
       error: result.errors,
-      injections: Config.isProd ? undefined : injections
+      injections: Config.isProd ? undefined : injections,
     })
   }
 
@@ -49,14 +47,14 @@ export async function handleDatabaseProxy(req: IRequest, res: Response) {
 
     return res.send({
       code: 0,
-      data
+      data,
     })
   } catch (error) {
     logger.error(requestId, 'execute query got error:  ', error)
     return res.send({
       code: 1,
       error: error.toString(),
-      injections: Config.isProd ? undefined : injections
+      injections: Config.isProd ? undefined : injections,
     })
   }
-}
+}

runtimes/nodejs/src/handler/debug-func.ts

@@ -26,11 +26,10 @@ export async function handleDebugFunction(req: IRequest, res: Response) {
     return res.status(400).send('x-laf-func-data is required')
   }
 
-
   // parse func_data
   let func_data: ICloudFunctionData
   try {
-    const decoded =  decodeURIComponent(func_str)
+    const decoded = decodeURIComponent(func_str)
     func_data = JSON.parse(decoded)
   } catch (error) {
     return res.status(400).send('x-laf-func-data is invalid')

runtimes/nodejs/src/handler/invoke-func.ts

@@ -32,10 +32,10 @@ export async function handleInvokeFunction(req: IRequest, res: Response) {
   const func_name = req.params?.name
 
   // load function data from db
-  let funcData = await CloudFunction.getFunctionByName(func_name)
+  let funcData = CloudFunction.getFunctionByName(func_name)
   if (!funcData) {
     // load default function from db
-    funcData = await CloudFunction.getFunctionByName(DEFAULT_FUNCTION_NAME)
+    funcData = CloudFunction.getFunctionByName(DEFAULT_FUNCTION_NAME)
     if (!funcData) {
       return res.status(404).send('Function Not Found')
     }
@@ -103,7 +103,7 @@ async function invokeInterceptor(req: IRequest, res: Response) {
   const func_name = INTERCEPTOR_FUNCTION_NAME
 
   // load function data from db
-  const funcData = await CloudFunction.getFunctionByName(func_name)
+  const funcData = CloudFunction.getFunctionByName(func_name)
   // pass if no interceptor
   if (!funcData) {
     return true

runtimes/nodejs/src/handler/router.ts

@@ -2,7 +2,7 @@
  * @Author: Maslow<[email protected]>
  * @Date: 2021-07-30 10:30:29
  * @LastEditTime: 2022-02-03 00:32:16
- * @Description: 
+ * @Description:
  */
 
 import { Router } from 'express'
@@ -13,7 +13,6 @@ import { handlePackageTypings } from './typings'
 import { generateUUID } from '../support/utils'
 import { handleInvokeFunction } from './invoke-func'
 
-
 /**
  * multer uploader config
  */
@@ -22,18 +21,19 @@ const uploader = multer({
     filename: (_req, file, cb) => {
       const { ext } = path.parse(file.originalname)
       cb(null, generateUUID() + ext)
-    }
+    },
   }),
   fileFilter(_req, file, callback) {
     // solve the problem of garbled unicode names
-    file.originalname = Buffer.from(file.originalname, 'latin1').toString('utf8')
+    file.originalname = Buffer.from(file.originalname, 'latin1').toString(
+      'utf8',
+    )
     callback(null, true)
   },
 })
 
 export const router = Router()
 
-
 router.post('/proxy/:policy', handleDatabaseProxy)
 router.get('/_/typing/package', handlePackageTypings)
 router.get('/_/healthz', (_req, res) => res.status(200).send('ok'))
@@ -42,4 +42,4 @@ router.get('/_/healthz', (_req, res) => res.status(200).send('ok'))
  * Invoke cloud function through HTTP request.
  * @method *
  */
-router.all('/:name', uploader.any(), handleInvokeFunction)
+router.all('/:name', uploader.any(), handleInvokeFunction)

runtimes/nodejs/src/handler/typings.ts

@@ -2,14 +2,15 @@
  * @Author: Maslow<[email protected]>
  * @Date: 2021-07-30 10:30:29
  * @LastEditTime: 2021-08-18 16:49:35
- * @Description: 
+ * @Description:
  */
 
 import { Response } from 'express'
 import { PackageDeclaration, NodePackageDeclarations } from 'node-modules-utils'
 import path = require('path')
 import { logger } from '../support/logger'
 import { IRequest } from '../support/types'
+import { FunctionCache } from '../support/function-engine/cache'
 
 const nodeModulesRoot = path.resolve(__dirname, '../../node_modules')
 
@@ -35,7 +36,7 @@ export async function handlePackageTypings(req: IRequest, res: Response) {
 
     return res.send({
       code: 0,
-      data: rets
+      data: rets,
     })
   }
 
@@ -46,7 +47,22 @@ export async function handlePackageTypings(req: IRequest, res: Response) {
 
     return res.send({
       code: 0,
-      data: [r]
+      data: [r],
+    })
+  }
+
+  // get cloud function types
+  if (packageName.startsWith('@/')) {
+    const func = FunctionCache.getFunctionByName(packageName.replace('@/', ''))
+    const r = {
+      packageName: packageName,
+      content: func.source.code,
+      path: `${packageName}/index.ts`,
+      from: 'node',
+    }
+    return res.send({
+      code: 0,
+      data: [r],
     })
   }
 
@@ -56,13 +72,13 @@ export async function handlePackageTypings(req: IRequest, res: Response) {
     await pkd.load()
     return res.send({
       code: 0,
-      data: pkd.declarations
+      data: pkd.declarations,
     })
   } catch (error) {
     logger.error(requestId, 'failed to get package typings', error)
     return res.send({
       code: 1,
-      error: error.toString()
+      error: error.toString(),
     })
   }
 }

runtimes/nodejs/src/support/cloud-sdk.ts

@@ -52,7 +52,7 @@ function createCloudSdk() {
  * @returns
  */
 async function invokeInFunction(name: string, param?: FunctionContext) {
-  const data = await CloudFunction.getFunctionByName(name)
+  const data = CloudFunction.getFunctionByName(name)
   const func = new CloudFunction(data)
 
   if (!func) {