feat: gen-schema-view updates

firestore-bigquery-export/scripts/gen-schema-view/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@firebaseextensions/fs-bq-schema-views",
-  "version": "0.4.9",
+  "version": "0.4.10",
   "description": "Generate strongly-typed BigQuery Views based on raw JSON",
   "main": "./lib/index.js",
   "repository": {
@@ -31,13 +31,15 @@
   "author": "Jan Wyszynski <[email protected]>",
   "license": "Apache-2.0",
   "dependencies": {
-    "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.36",
+    "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.38",
+    "@genkit-ai/googleai": "^1.1.0",
     "@google-cloud/bigquery": "^6.0.3",
     "commander": "5.0.0",
     "firebase-admin": "^12.1.0",
     "firebase-functions": "^4.2.0",
     "fs-find": "^0.4.0",
     "generate-schema": "^2.6.0",
+    "genkit": "^1.1.0",
     "glob": "7.1.5",
     "inquirer": "^6.4.0",
     "sql-formatter": "^2.3.3"
@@ -46,15 +48,16 @@
     "@types/chai": "^4.1.6",
     "@types/express": "^4.17.14",
     "@types/express-serve-static-core": "4.17.30",
+    "@types/inquirer": "^9.0.7",
+    "@types/jest": "29.5.0",
     "chai": "^4.2.0",
     "exec": "^0.2.1",
-    "nyc": "^14.0.0",
-    "rimraf": "^2.6.3",
-    "ts-node": "^7.0.1",
-    "typescript": "^4.9.3",
-    "@types/jest": "29.5.0",
     "jest": "29.5.0",
     "mocked-env": "^1.3.2",
-    "ts-jest": "29.1.2"
+    "nyc": "^17.1.0",
+    "rimraf": "^2.6.3",
+    "ts-jest": "29.1.2",
+    "ts-node": "^7.0.1",
+    "typescript": "^4.9.3"
   }
 }

firestore-bigquery-export/scripts/gen-schema-view/src/__tests__/config/index.test.ts

@@ -0,0 +1,261 @@
+import { parseConfig } from "../../../src/config";
+import { promptInquirer } from "../../../src/config/interactive";
+import {
+  parseProgram,
+  validateNonInteractiveParams,
+} from "../../../src/config/non-interactive";
+import { readSchemas } from "../../../src/schema-loader-utils";
+
+// Mock dependencies
+jest.mock("../../../src/config/interactive", () => ({
+  promptInquirer: jest.fn(),
+}));
+
+jest.mock("../../../src/config/non-interactive", () => ({
+  parseProgram: jest.fn(),
+  validateNonInteractiveParams: jest.fn(),
+}));
+
+jest.mock("../../../src/schema-loader-utils", () => ({
+  readSchemas: jest.fn(),
+}));
+
+// Mock process.exit to prevent tests from actually exiting
+const mockExit = jest.spyOn(process, "exit").mockImplementation((code) => {
+  throw new Error(`Process exited with code ${code}`);
+});
+
+describe("parseConfig", () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe("Non-interactive mode", () => {
+    it("should return CLI config from command line arguments", async () => {
+      // Setup mocks for non-interactive mode
+      const mockProgram = {
+        nonInteractive: true,
+        project: "test-project",
+        bigQueryProject: "test-bq-project",
+        dataset: "test-dataset",
+        tableNamePrefix: "test-prefix",
+        schemaFiles: ["schema1.json", "schema2.json"],
+        outputHelp: jest.fn(),
+      };
+
+      const mockSchemas = {
+        schema1: { fields: { field1: { type: "string" } } },
+        schema2: { fields: { field2: { type: "number" } } },
+      };
+
+      (parseProgram as jest.Mock).mockReturnValue(mockProgram);
+      (validateNonInteractiveParams as jest.Mock).mockReturnValue(true);
+      (readSchemas as jest.Mock).mockReturnValue(mockSchemas);
+
+      const result = await parseConfig();
+
+      expect(parseProgram).toHaveBeenCalled();
+      expect(validateNonInteractiveParams).toHaveBeenCalledWith(mockProgram);
+      expect(readSchemas).toHaveBeenCalledWith(mockProgram.schemaFiles);
+      expect(result).toEqual({
+        agentSampleSize: 100,
+        projectId: "test-project",
+        bigQueryProjectId: "test-bq-project",
+        datasetId: "test-dataset",
+        tableNamePrefix: "test-prefix",
+        schemas: mockSchemas,
+        geminiAnalyzeCollectionPath: undefined,
+        googleAiKey: undefined,
+        schemaDirectory: undefined,
+        useGemini: false,
+      });
+    });
+
+    it("should use project as bigQueryProject if not specified", async () => {
+      // Setup mocks with missing bigQueryProject
+      const mockProgram = {
+        nonInteractive: true,
+        project: "test-project",
+        bigQueryProject: undefined,
+        dataset: "test-dataset",
+        tableNamePrefix: "test-prefix",
+        schemaFiles: ["schema.json"],
+        outputHelp: jest.fn(),
+      };
+
+      const mockSchemas = { schema: { fields: { field: { type: "string" } } } };
+
+      (parseProgram as jest.Mock).mockReturnValue(mockProgram);
+      (validateNonInteractiveParams as jest.Mock).mockReturnValue(true);
+      (readSchemas as jest.Mock).mockReturnValue(mockSchemas);
+
+      const result = await parseConfig();
+
+      expect(result.bigQueryProjectId).toBe("test-project");
+    });
+
+    it("should use gemini if specified", async () => {
+      // Setup mocks with useGemini = true
+      const mockProgram = {
+        nonInteractive: true,
+        project: "test-project",
+        bigQueryProject: "test-bq-project",
+        dataset: "test-dataset",
+        tableNamePrefix: "test-prefix",
+        schemaFiles: ["schema.json"],
+        useGemini: "test-collection",
+        googleAiKey: "test-key",
+        geminiAnalyzeCollectionPath: "test-collection",
+        schemaDirectory: "test-directory",
+        outputHelp: jest.fn(),
+      };
+
+      (parseProgram as jest.Mock).mockReturnValue(mockProgram);
+      (validateNonInteractiveParams as jest.Mock).mockReturnValue(true);
+
+      const result = await parseConfig();
+
+      expect(result.useGemini).toBe(true);
+      expect(result.googleAiKey).toBe("test-key");
+      expect(result.geminiAnalyzeCollectionPath).toBe("test-collection");
+      expect(result.schemaDirectory).toBe("test-directory");
+      expect(result.agentSampleSize).toBe(100);
+    });
+
+    it("should exit if required parameters are missing", async () => {
+      const mockProgram = {
+        nonInteractive: true,
+        outputHelp: jest.fn(),
+      };
+
+      (parseProgram as jest.Mock).mockReturnValue(mockProgram);
+      (validateNonInteractiveParams as jest.Mock).mockReturnValue(false);
+
+      await expect(parseConfig()).rejects.toThrow("Process exited with code 1");
+      expect(mockProgram.outputHelp).toHaveBeenCalled();
+      expect(mockExit).toHaveBeenCalledWith(1);
+    });
+  });
+
+  describe("Interactive mode without Gemini", () => {
+    it("should return CLI config from inquirer prompts", async () => {
+      // Setup mocks for interactive mode
+      const mockProgram = {
+        nonInteractive: false,
+      };
+
+      const mockPromptResponse = {
+        projectId: "interactive-project",
+        bigQueryProjectId: "interactive-bq-project",
+        datasetId: "interactive-dataset",
+        tableNamePrefix: "interactive-prefix",
+        useGemini: false,
+        schemaFiles: "schema1.json, schema2.json",
+      };
+
+      const mockSchemas = {
+        schema1: { fields: { field1: { type: "string" } } },
+        schema2: { fields: { field2: { type: "number" } } },
+      };
+
+      (parseProgram as jest.Mock).mockReturnValue(mockProgram);
+      (promptInquirer as jest.Mock).mockResolvedValue(mockPromptResponse);
+      (readSchemas as jest.Mock).mockReturnValue(mockSchemas);
+
+      const result = await parseConfig();
+
+      expect(parseProgram).toHaveBeenCalled();
+      expect(promptInquirer).toHaveBeenCalled();
+      // Expect the schemaFiles string to be passed as-is in an array
+      expect(readSchemas).toHaveBeenCalledWith(["schema1.json, schema2.json"]);
+      expect(result).toEqual({
+        agentSampleSize: 100,
+        projectId: "interactive-project",
+        bigQueryProjectId: "interactive-bq-project",
+        datasetId: "interactive-dataset",
+        tableNamePrefix: "interactive-prefix",
+        schemas: mockSchemas,
+        geminiAnalyzeCollectionPath: undefined,
+        googleAiKey: undefined,
+        schemaDirectory: undefined,
+        useGemini: false,
+      });
+    });
+
+    it("should properly handle schema file paths without trimming or splitting", async () => {
+      const mockProgram = {
+        nonInteractive: false,
+      };
+
+      const mockPromptResponse = {
+        project: "test-project",
+        bigQueryProject: "test-bq-project",
+        dataset: "test-dataset",
+        tableNamePrefix: "test-prefix",
+        useGemini: false,
+        schemaFiles: " schema1.json,  schema2.json , schema3.json",
+      };
+
+      (parseProgram as jest.Mock).mockReturnValue(mockProgram);
+      (promptInquirer as jest.Mock).mockResolvedValue(mockPromptResponse);
+      (readSchemas as jest.Mock).mockReturnValue({});
+
+      await parseConfig();
+
+      // Verify that the schemaFiles string is passed as-is within an array
+      expect(readSchemas).toHaveBeenCalledWith([
+        " schema1.json,  schema2.json , schema3.json",
+      ]);
+    });
+  });
+
+  describe("Interactive mode with Gemini", () => {
+    it("should return CLI config from inquirer prompts", async () => {
+      // Setup mocks for interactive mode
+      const mockProgram = {
+        nonInteractive: false,
+      };
+
+      const mockPromptResponse = {
+        projectId: "interactive-project",
+        bigQueryProjectId: "interactive-bq-project",
+        datasetId: "interactive-dataset",
+        tableNamePrefix: "interactive-prefix",
+        useGemini: true,
+        googleAiKey: "test-key",
+        geminiAnalyzeCollectionPath: "test-collection",
+        schemaDirectory: "test-directory",
+      };
+
+      // Although we set up mockSchemas, in Gemini mode readSchemas is not called.
+      const mockSchemas = {
+        schema1: { fields: { field1: { type: "string" } } },
+        schema2: { fields: { field2: { type: "number" } } },
+      };
+
+      (parseProgram as jest.Mock).mockReturnValue(mockProgram);
+      (promptInquirer as jest.Mock).mockResolvedValue(mockPromptResponse);
+      (readSchemas as jest.Mock).mockReturnValue(mockSchemas);
+
+      const result = await parseConfig();
+
+      expect(parseProgram).toHaveBeenCalled();
+      expect(promptInquirer).toHaveBeenCalled();
+      // In Gemini mode, schemaFiles may not be provided so readSchemas should not be called
+      expect(readSchemas).not.toHaveBeenCalled();
+      // Expect schemas to be an empty object in Gemini mode
+      expect(result).toEqual({
+        agentSampleSize: 100,
+        projectId: "interactive-project",
+        bigQueryProjectId: "interactive-bq-project",
+        datasetId: "interactive-dataset",
+        tableNamePrefix: "interactive-prefix",
+        schemas: {},
+        geminiAnalyzeCollectionPath: "test-collection",
+        googleAiKey: "test-key",
+        schemaDirectory: "test-directory",
+        useGemini: true,
+      });
+    });
+  });
+});