Compiler rewrite (#68)

* Move compilation sources to lib/compiler

* Extract findInputs to a separate function

* Extract findImports to separate function

* Remove overrides from compiler

* Move loadConfig into separate file

* Remove the Compiler class

* Remove solcjs custom saveOutput and extract buildCommand in nativeWrapper

* Remove saving output from the Wrappers

* Add formatErrors helper

* Replace SolcjsWrapper with a higher order function

* Replace DockerWrapper with a higher order function

* Fix compilation output

* Replace NativeWrapper with a higher order function

* Move some logic to ImportMappingBuilder

* Extract buildInputObject to a separate function

* Extract buildSources to a separate function

* Remove wrappers entirely

* Prepare a 2.0.3 release

* Fix linting errors

Author: sz-piotr

bin/waffle

@@ -1,5 +1,10 @@
 #!/usr/bin/env node
 
 'use strict';
-const Waffle = require('../dist/compiler.js');
-Waffle.compile(process.argv[2]);
+const Waffle = require('../dist/compiler/compiler.js');
+Waffle
+  .compileProject(process.argv[2])
+  .catch(e => {
+    console.error(e);
+    process.exit(1);
+  });

docs/release-notes/2.0.3.md

@@ -0,0 +1,10 @@
+# Summary
+This release introduces TypeScript declaration files for custom matchers and
+a rewrite of the compiler.
+
+## Type declarations for custom matchers 
+You can now enjoy full type safety when writing your tests with waffle.
+
+## Compiler rewrite
+While no external facing api is changing the internals of the compiler have been
+rewritten to enable further developments.

lib/compiler.ts

@@ -0,0 +1,18 @@
+export function buildInputObject(sources: any, remappings?: any) {
+  return {
+    language: 'Solidity',
+    sources,
+    settings: {
+      remappings,
+      outputSelection: {'*': {'*': ['abi', 'evm.bytecode', 'evm.deployedBytecode']}}
+    }
+  };
+}
+
+export function buildSources(inputs: string[], transform: (input: string) => string) {
+  const sources: Record<string, { urls: string[] }> = {};
+  for (const input of inputs) {
+    sources[input.replace(/\\/g, '/')] = {urls: [transform(input)]};
+  }
+  return sources;
+}

lib/compiler/buildUitls.ts

@@ -0,0 +1,47 @@
+import {join} from 'path';
+import ImportMappingBuilder from './importMappingBuilder';
+import {Config} from '../config/config';
+import {execSync} from 'child_process';
+import {buildInputObject, buildSources} from './buildUitls';
+
+const CONTAINER_PATH = '/home/project';
+const NPM_PATH = '/home/npm';
+
+export function compileDocker(config: Config) {
+  return async function compile(sources: string[]) {
+    const command = createBuildCommand(config);
+    const input = JSON.stringify(buildInputJson(sources, config), null, 2);
+    return JSON.parse(execSync(command, {input}).toString());
+  };
+}
+
+export function createBuildCommand(config: Config) {
+  const configTag = config['docker-tag'];
+  const tag = configTag ? `:${configTag}` : ':stable';
+  const allowedPaths = `"${CONTAINER_PATH},${NPM_PATH}"`;
+  return `docker run ${getVolumes(config)} -i -a stdin -a stdout ` +
+    `ethereum/solc${tag} solc --standard-json --allow-paths ${allowedPaths}`;
+}
+
+export function getVolumes(config: Config) {
+  const hostPath = process.cwd();
+  const hostNpmPath = join(hostPath, config.npmPath);
+  return `-v ${hostPath}:${CONTAINER_PATH} -v ${hostNpmPath}:${NPM_PATH}`;
+}
+
+export function buildInputJson(sources: string[], config: Config) {
+  return buildInputObject(
+    buildSources(sources, (input) => join(CONTAINER_PATH, input)),
+    getMappings(sources, config)
+  );
+}
+
+function getMappings(sources: string[], config: Config) {
+  const mappingBuilder = new ImportMappingBuilder(
+    config.sourcesPath,
+    config.npmPath,
+    CONTAINER_PATH,
+    NPM_PATH
+  );
+  return mappingBuilder.getMappings(sources);
+}

lib/compiler/compileDocker.ts

@@ -0,0 +1,33 @@
+import {join, resolve} from 'path';
+import {execSync} from 'child_process';
+import {Config} from '../config/config';
+import ImportMappingBuilder from './importMappingBuilder';
+import {buildInputObject, buildSources} from './buildUitls';
+
+export function compileNative(config: Config) {
+  return async function compile(sources: string[]) {
+    const command = createBuildCommand(config);
+    const input = JSON.stringify(buildInputJson(sources, config), null, 2);
+    return JSON.parse(execSync(command, {input}).toString());
+  };
+}
+
+export function createBuildCommand(config: Config) {
+  const command = 'solc';
+  const params = '--standard-json';
+  const customAllowedPaths = (config.allowedPaths || []).map((path: string) => resolve(path));
+  const allowedPaths = [resolve(config.sourcesPath), resolve(config.npmPath), ...customAllowedPaths];
+  return `${command} ${params} --allow-paths ${allowedPaths.join(',')}`;
+}
+
+function buildInputJson(sources: string[], config: Config) {
+  return buildInputObject(
+    buildSources(sources, (input) => join(process.cwd(), input)),
+    getMappings(sources, config)
+  );
+}
+
+function getMappings(sources: string[], config: Config) {
+  const mappingBuilder = new ImportMappingBuilder(config.sourcesPath, config.npmPath);
+  return mappingBuilder.getMappings(sources);
+}

lib/compiler/compileNative.ts

@@ -0,0 +1,37 @@
+import solc from 'solc';
+import {promisify} from 'util';
+import {readFileContent} from '../utils';
+import {Config} from '../config/config';
+import {buildInputObject} from './buildUitls';
+
+const loadRemoteVersion = promisify(solc.loadRemoteVersion);
+
+async function loadCompiler(config: Config) {
+  if (config.solcVersion) {
+    return loadRemoteVersion(config.solcVersion);
+  } else {
+    return solc;
+  }
+}
+
+export function compileSolcjs(config: Config) {
+  return async function compile(sources: string[], findImports: (file: string) => any) {
+    const solc = await loadCompiler(config);
+    const inputs = findInputs(sources);
+    const input = buildInputObject(convertInputs(inputs));
+    const output = solc.compile(JSON.stringify(input), findImports);
+    return JSON.parse(output);
+  };
+}
+
+function convertInputs(inputs: Record<string, any>) {
+  const converted: Record<string, { content: string }> = {};
+  Object.keys(inputs).map((key) => converted[key.replace(/\\/g, '/')] = {content: inputs[key]});
+  return converted;
+}
+
+export function findInputs(files: string[]) {
+  return Object.assign({}, ...files.map((file) => ({
+    [file]: readFileContent(file)
+  })));
+}

lib/compiler/compileSolcjs.ts

@@ -0,0 +1,46 @@
+import {Config} from '../config/config';
+import {isWarningMessage} from '../utils';
+import {getCompileFunction} from './getCompileFunction';
+import {findInputs} from './findInputs';
+import {findImports} from './findImports';
+import {loadConfig} from '../config/loadConfig';
+import {saveOutput} from './saveOutput';
+
+export async function compileProject(configPath: string) {
+  await compileAndSave(loadConfig(configPath));
+}
+
+export async function compileAndSave(config: Config) {
+  const output = await compile(config);
+  await processOutput(output, config);
+}
+
+export async function compile(config: Config) {
+  return getCompileFunction(config)(
+    findInputs(config.sourcesPath),
+    findImports(config.npmPath)
+  );
+}
+
+async function processOutput(output: any, config: Config) {
+  if (output.errors) {
+    console.error(formatErrors(output.errors));
+  }
+  if (anyNonWarningErrors(output.errors)) {
+    throw new Error('Compilation failed');
+  } else {
+    await saveOutput(output, config);
+  }
+}
+
+function anyNonWarningErrors(errors?: any[]) {
+  return errors && !errors.every(isWarningMessage);
+}
+
+function formatErrors(errors: any[]) {
+  return errors.map(toFormattedMessage).join('\n');
+}
+
+function toFormattedMessage(error: any) {
+  return typeof error === 'string' ? error : error.formattedMessage;
+}

lib/compiler/compiler.ts

@@ -0,0 +1,20 @@
+import fs from 'fs';
+import path from 'path';
+import {readFileContent} from '../utils';
+
+export function findImports(libraryPath: string) {
+  return (file: string) => {
+    try {
+      const libFile = path.join(libraryPath, file);
+      if (fs.existsSync(file)) {
+        return { contents: readFileContent(file) };
+      } else if (fs.existsSync(libFile)) {
+        return { contents: readFileContent(libFile) };
+      } else {
+        throw new Error(`File not found: ${file}`);
+      }
+    } catch (e) {
+      return { error: e.message };
+    }
+  };
+}

lib/compiler/findImports.ts

@@ -0,0 +1,24 @@
+import fs from 'fs';
+import path from 'path';
+
+export function findInputs(sourcePath: string) {
+  const stack = [sourcePath];
+  const inputFiles: string[] = [];
+  while (stack.length > 0) {
+    const dir = stack.pop();
+    const files = fs.readdirSync(dir);
+    for (const file of files) {
+      const filePath = path.join(dir, file);
+      if (isDirectory(filePath)) {
+        stack.push(filePath);
+      } else if (file.endsWith('.sol')) {
+        inputFiles.push(filePath);
+      }
+    }
+  }
+  return inputFiles;
+}
+
+const isDirectory = (filePath: string) =>
+  fs.existsSync(filePath) &&
+  fs.statSync(filePath).isDirectory();

lib/compiler/findInputs.ts

@@ -0,0 +1,20 @@
+import { Config } from '../config/config';
+import {compileSolcjs} from './compileSolcjs';
+import {compileNative} from './compileNative';
+import {compileDocker} from './compileDocker';
+
+export type CompileFunction = (
+  sources: string[],
+  findImports: (file: string) => any
+) => any;
+
+export function getCompileFunction(config: Config): CompileFunction {
+  if (config.compiler === 'native') {
+    return compileNative(config);
+  } else if (config.compiler === 'dockerized-solc') {
+    return compileDocker(config);
+  } else if (config.compiler === 'solcjs' || !config.compiler) {
+    return compileSolcjs(config);
+  }
+  throw new Error(`Unknown compiler ${config.compiler}`);
+}

lib/compiler/getCompileFunction.ts

@@ -63,10 +63,11 @@ class ImportMappingBuilder {
   }
 
   public getMappings(sources: string[]) {
-    return falttenObjectArray(
+    const mappings = falttenObjectArray(
       sources.map((path) =>
         this.getMappingForUnit(readFileContent(path), path)
       ));
+    return Object.entries(mappings).map(([key, value]) => `${key}=${value}`);
   }
 }