use @dvirtz/parquets

Fixes llvm#19

Author: dvirtz

README.md

@@ -4,8 +4,6 @@
 
 Views [Apache Parquet](https://parquet.apache.org/) files as JSON.
 
-Requires [parquet-tools](https://mvnrepository.com/artifact/org.apache.parquet/parquet-tools).
-
 ## Features
 
 When opening a Parquet file, a JSON presentation of the file will open automatically:
@@ -18,8 +16,10 @@ After closing the JSON view, it is possible to reopen it by clicking on the link
 
 ## Requirements
 
-The extension requires [parquet-tools](https://github.com/apache/parquet-mr/tree/master/parquet-tools-deprecated).
-It should be in your PATH, or a path can be set in settings.
+The extension used to require [parquet-tools](https://mvnrepository.com/artifact/org.apache.parquet/parquet-tools).
+Now the extension uses the [parquets](https://github.com/dvirtz/parquets) TypeScript library to do parse the files.
+
+If you still want to use `parquet-tools`, you should set `parquet-viewer.useParquetTools` to `true` and `paruqet-tools` should be in your `PATH`, or pointed by the `parquet-viewer.parquetToolsPath` setting.
 
 ![settings](images/settings.png)
 
@@ -29,10 +29,11 @@ The following setting options are available:
 
 |name|default|description|
 |----|-------|-----------|
-|`parquet-viewer.parquetToolsPath`|`parquet-tools`| the name of the parquet-tools executable or a path to the parquet-tools jar|
-|`parquet-viewer.logPanel`|`false`|whether to write diagnostic logs to an output panel|
+|`parquet-viewer.parquetToolsPath`|`parquet-tools`|The name of the parquet-tools executable or a path to the parquet-tools jar|
+|`parquet-viewer.logPanel`|`false`|Whether to write diagnostic logs to an output panel|
 |`parquet-viewer.logFolder`|empty|Write diagnostic logs under the given directory|
 |`parquet-viewer.logLevel`|info|Diagnostic log level. Choose between: `off`, `fatal`, `error`, `warn`, `info`, `debug` or `trace`|
+|`parquet-viewer.useParquetTools`|`false`|Use the legacy `parquet-tools` application for reading the files|
 
 ### What's new
 

package-lock.json

@@ -66,6 +66,10 @@
               "trace"
             ],
             "default": "info"
+          },
+          "parquet-viewer.useParquetTools": {
+            "description": "use legacy parquet-tools application for reading parquet files",
+            "type": "boolean"
           }
         }
       }
@@ -97,7 +101,8 @@
     "@types/chai-as-promised": "^7.1.3",
     "@types/glob": "^7.1.3",
     "@types/mocha": "^8.0.3",
-    "@types/node": "^14.14.6",
+    "@types/node": "^12.20.11",
+    "@types/thrift": "^0.10.10",
     "@types/vscode": "^1.46.0",
     "@typescript-eslint/eslint-plugin": "^4.16.1",
     "@typescript-eslint/parser": "^4.16.1",
@@ -113,6 +118,7 @@
   },
   "dependencies": {
     "@async-generators/to-array": "^0.1.0",
+    "@dvirtz/parquets": "^0.11.2-develop.1",
     "@vscode-logging/logger": "^1.2.2",
     "@vscode-logging/wrapper": "^1.0.0"
   }

package.json

@@ -3,7 +3,6 @@
 // Import the module and reference it with the alias vscode in your code below
 import * as vscode from 'vscode';
 import { ParquetEditorProvider } from './parquet-editor-provider';
-import { ParquetToolsRunner } from './parquet-tools-runner';
 import { getLogger, initLogger } from './logger';
 
 // this method is called when your extension is activated
@@ -15,12 +14,5 @@ export async function activate(context: vscode.ExtensionContext): Promise<void>
   // restart logger on configuration change
   vscode.workspace.onDidChangeConfiguration(() => initLogger(context));
 
-  const parquetTools = await ParquetToolsRunner.spawnParquetTools(['-h']);
-  parquetTools.on('error', async (err: string) => {
-    const message = `parquet-tools not in PATH ('${err}')`;
-    getLogger().error(message);
-    await vscode.window.showErrorMessage(message);
-  });
-
   context.subscriptions.push(ParquetEditorProvider.register(context));
 }

src/extension.ts

@@ -2,10 +2,12 @@ import * as vscode from "vscode";
 import * as path from 'path';
 import { getNonce } from './util';
 import { Disposable } from "./dispose";
-import { ParquetTextDocumentContentProvider } from "./parquet-document-provider";
-import { ParquetToolsRunner } from "./parquet-tools-runner";
+import { ParquetTextDocumentContentProvider } from './parquet-document-provider';
+import { ParquetToolsBackend } from './parquet-tools-backend';
+import { ParquetsBackend } from './parquets-backend';
 import toArray from '@async-generators/to-array';
 import { getLogger } from './logger';
+import { useParquetTools } from "./settings";
 
 class DummyDocument extends Disposable implements vscode.CustomDocument {
   uri: vscode.Uri;
@@ -24,29 +26,36 @@ class DummyDocument extends Disposable implements vscode.CustomDocument {
     );
   }
 
+  private async * toJson(parquetPath: string, token?: vscode.CancellationToken): AsyncGenerator<string> {
+    if (useParquetTools()) {
+      yield* ParquetToolsBackend.toJson(parquetPath, token);
+    }
+
+    yield* ParquetsBackend.toJson(parquetPath, token);
+  }
+
   public async show() {
     getLogger().info(`showing ${this.path}.as.json`);
     if (ParquetTextDocumentContentProvider.has(this.path)) {
       return await this.open();
     }
 
-    return await vscode.window.withProgress({
-      location: vscode.ProgressLocation.Notification,
-      title: `opening ${path.basename(this.path)}`,
-      cancellable: true
-    },
-      async (progress, token) => {
-        try {
-          const json = await toArray(ParquetToolsRunner.toJson(this.path, token));
+    try {
+      return await vscode.window.withProgress({
+        location: vscode.ProgressLocation.Notification,
+        title: `opening ${path.basename(this.path)}`,
+        cancellable: true
+      },
+        async (progress, token) => {
+          const json = await toArray(this.toJson(this.path, token));
           if (!token.isCancellationRequested) {
             ParquetTextDocumentContentProvider.add(this.path, json.join(''));
             await this.open();
           }
-        } catch (err) {
-          getLogger().error(err.message);
-          await vscode.window.showErrorMessage(err.message);
-        }
-      });
+        });
+    } catch (err) {
+      await vscode.window.showErrorMessage(err);
+    }
   }
 
   dispose(): void {
@@ -86,11 +95,19 @@ export class ParquetEditorProvider implements vscode.CustomReadonlyEditorProvide
 
     webviewPanel.webview.html = this.getHtmlForWebview(webviewPanel.webview, document);
 
-    webviewPanel.webview.onDidReceiveMessage(_ => document.show());
+    webviewPanel.webview.onDidReceiveMessage(e => this.onMessage(document, e));
 
     await document.show();
   }
 
+  private async onMessage(document: DummyDocument, message: string) {
+    switch (message) {
+      case 'clicked':
+        await document.show();
+        break;
+    }
+  }
+
   private getHtmlForWebview(webview: vscode.Webview, document: DummyDocument): string {
     // Use a nonce to whitelist which scripts can be run
     const nonce = getNonce();
@@ -109,7 +126,7 @@ export class ParquetEditorProvider implements vscode.CustomReadonlyEditorProvide
       //# sourceURL=to-json.js
       const vscode = acquireVsCodeApi();
       document.getElementById('here').addEventListener('click', _ => {
-        vscode.postMessage();
+        vscode.postMessage('clicked');
       });
     </script>
   </body>