Merge pull request #621 from diffplug/feat/git-attributes-relocatable

Make the the GitAttributesLineEndingsPolicy relocatable across machines

Author: nedtwigg

CHANGES.md

@@ -10,6 +10,8 @@ This document is intended for Spotless developers.
 We adhere to the [keepachangelog](https://keepachangelog.com/en/1.0.0/) format (starting after version `1.27.0`).
 
 ## [Unreleased]
+### Changed
+* `LineEnding.GIT_ATTRIBUTES` now creates a policy whose serialized state can be relocated from one machine to another.  No user-visible change, but paves the way for remote build cache support in Gradle. ([#621](https://github.com/diffplug/spotless/pull/621))
 ### Added
 * `prettier` will now autodetect the parser (and formatter) to use based on the filename, unless you override this using `config` or `configFile` with the option `parser` or `filepath`. ([#620](https://github.com/diffplug/spotless/pull/620))
 

lib-extra/src/main/java/com/diffplug/spotless/extra/GitAttributesLineEndings.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 DiffPlug
+ * Copyright 2016-2020 DiffPlug
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -22,19 +22,13 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.Serializable;
-import java.util.ArrayList;
-import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 import java.util.Objects;
-import java.util.Set;
 import java.util.function.Supplier;
-import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 import javax.annotation.Nullable;
 
@@ -51,11 +45,9 @@
 import org.eclipse.jgit.util.SystemReader;
 
 import com.googlecode.concurrenttrees.radix.ConcurrentRadixTree;
-import com.googlecode.concurrenttrees.radix.node.Node;
 import com.googlecode.concurrenttrees.radix.node.concrete.DefaultCharSequenceNodeFactory;
 
 import com.diffplug.common.base.Errors;
-import com.diffplug.common.tree.TreeStream;
 import com.diffplug.spotless.FileSignature;
 import com.diffplug.spotless.LazyForwardingEquality;
 import com.diffplug.spotless.LineEnding;
@@ -72,62 +64,82 @@ public final class GitAttributesLineEndings {
 	// prevent direct instantiation
 	private GitAttributesLineEndings() {}
 
-	public static Policy create(File projectDir, Supplier<Iterable<File>> toFormat) {
-		return new Policy(projectDir, toFormat);
+	/**
+	 * Creates a line-endings policy whose serialized state is relativized against projectDir,
+	 * at the cost of eagerly evaluating the line-ending state of every target file when the
+	 * policy is checked for equality with another policy.
+	 */
+	public static LineEnding.Policy create(File projectDir, Supplier<Iterable<File>> toFormat) {
+		return new RelocatablePolicy(projectDir, toFormat);
 	}
 
-	static class Policy extends LazyForwardingEquality<FileState> implements LineEnding.Policy {
-		private static final long serialVersionUID = 1L;
+	static class RelocatablePolicy extends LazyForwardingEquality<CachedEndings> implements LineEnding.Policy {
+		private static final long serialVersionUID = 5868522122123693015L;
 
 		final transient File projectDir;
 		final transient Supplier<Iterable<File>> toFormat;
 
-		Policy(File projectDir, Supplier<Iterable<File>> toFormat) {
+		RelocatablePolicy(File projectDir, Supplier<Iterable<File>> toFormat) {
 			this.projectDir = Objects.requireNonNull(projectDir, "projectDir");
 			this.toFormat = Objects.requireNonNull(toFormat, "toFormat");
 		}
 
 		@Override
-		protected FileState calculateState() throws Exception {
-			return new FileState(projectDir, toFormat.get());
+		protected CachedEndings calculateState() throws Exception {
+			Runtime runtime = new RuntimeInit(projectDir, toFormat.get()).atRuntime();
+			return new CachedEndings(projectDir, runtime, toFormat.get());
 		}
 
-		/**
-		 * Initializing the state() for up-to-date checking is faster than the full initialization
-		 * needed to actually do the formatting. We load the Runtime lazily from the state().
-		 */
-		transient Runtime runtime;
-
 		@Override
 		public String getEndingFor(File file) {
-			if (runtime == null) {
-				runtime = state().atRuntime();
-			}
-			return runtime.getEndingFor(file);
+			return state().endingFor(file);
 		}
 	}
 
 	@SuppressFBWarnings("SE_TRANSIENT_FIELD_NOT_RESTORED")
-	static class FileState implements Serializable {
-		private static final long serialVersionUID = 1L;
+	static class CachedEndings implements Serializable {
+		private static final long serialVersionUID = -2534772773057900619L;
+
+		/** this is transient, to simulate PathSensitive.RELATIVE */
+		transient final String rootDir;
+		/** the line ending used for most files */
+		final String defaultEnding;
+		/** any exceptions to that default, in terms of relative path from rootDir */
+		final ConcurrentRadixTree<String> hasNonDefaultEnding = new ConcurrentRadixTree<>(new DefaultCharSequenceNodeFactory());
+
+		CachedEndings(File projectDir, Runtime runtime, Iterable<File> toFormat) {
+			rootDir = FileSignature.pathNativeToUnix(projectDir.getAbsolutePath()) + "/";
+			defaultEnding = runtime.defaultEnding;
+			for (File file : toFormat) {
+				String ending = runtime.getEndingFor(file);
+				if (!ending.equals(defaultEnding)) {
+					String path = FileSignature.pathNativeToUnix(file.getAbsolutePath());
+					hasNonDefaultEnding.put(path, ending);
+				}
+			}
+		}
 
+		/** Returns the line ending appropriate for the given file. */
+		public String endingFor(File file) {
+			String path = FileSignature.pathNativeToUnix(file.getAbsolutePath());
+			String subpath = FileSignature.subpath(rootDir, path);
+			String ending = hasNonDefaultEnding.getValueForExactKey(subpath);
+			return ending == null ? defaultEnding : ending;
+		}
+	}
+
+	static class RuntimeInit {
 		/** /etc/gitconfig (system-global), ~/.gitconfig, project/.git/config (each might-not exist). */
-		transient final FileBasedConfig systemConfig, userConfig, repoConfig;
+		final FileBasedConfig systemConfig, userConfig, repoConfig;
 
 		/** Global .gitattributes file pointed at by systemConfig or userConfig, and the file in the repo. */
-		transient final @Nullable File globalAttributesFile, repoAttributesFile;
+		final @Nullable File globalAttributesFile, repoAttributesFile;
 
 		/** git worktree root, might not exist if we're not in a git repo. */
-		transient final @Nullable File workTree;
-
-		/** All the .gitattributes files in the work tree that we're formatting. */
-		transient final List<File> gitattributes;
-
-		/** The signature of *all* of the files below. */
-		final FileSignature signature;
+		final @Nullable File workTree;
 
 		@SuppressFBWarnings("SIC_INNER_SHOULD_BE_STATIC_ANON")
-		FileState(File projectDir, Iterable<File> toFormat) throws IOException {
+		RuntimeInit(File projectDir, Iterable<File> toFormat) throws IOException {
 			requireElementsNonNull(toFormat);
 			/////////////////////////////////
 			// USER AND SYSTEM-WIDE VALUES //
@@ -178,56 +190,6 @@ public boolean isOutdated() {
 				repoAttributesFile = null;
 			}
 			Errors.log().run(repoConfig::load);
-
-			// The .gitattributes files which apply to the files we are formatting
-			gitattributes = gitAttributes(toFormat);
-
-			// find every actual File which exists above
-			Stream<File> misc = Stream.of(systemConfig.getFile(), userConfig.getFile(), repoConfig.getFile(), globalAttributesFile, repoAttributesFile);
-			List<File> toSign = Stream.concat(gitattributes.stream(), misc)
-					.filter(file -> file != null && file.exists() && file.isFile())
-					.collect(Collectors.toList());
-			// sign it for up-to-date checking
-			signature = FileSignature.signAsSet(toSign);
-		}
-
-		/** Returns all of the .gitattributes files which affect the given files. */
-		static List<File> gitAttributes(Iterable<File> files) {
-			// build a radix tree out of all the parent folders in these files
-			ConcurrentRadixTree<String> tree = new ConcurrentRadixTree<>(new DefaultCharSequenceNodeFactory());
-			for (File file : files) {
-				String parentPath = file.getParent() + File.separator;
-				tree.putIfAbsent(parentPath, parentPath);
-			}
-			// traverse the edge nodes to find the outermost folders
-			List<File> edgeFolders = TreeStream.depthFirst(Node::getOutgoingEdges, tree.getNode())
-					.filter(node -> node.getOutgoingEdges().isEmpty() && node.getValue() != null)
-					.map(node -> new File((String) node.getValue()))
-					.collect(Collectors.toList());
-
-			List<File> gitAttrFiles = new ArrayList<>();
-			Set<File> visitedFolders = new HashSet<>();
-			for (File edgeFolder : edgeFolders) {
-				gitAttrAddWithParents(edgeFolder, visitedFolders, gitAttrFiles);
-			}
-			return gitAttrFiles;
-		}
-
-		/** Searches folder and all its parents for gitattributes files. */
-		private static void gitAttrAddWithParents(File folder, Set<File> visitedFolders, Collection<File> gitAttrFiles) {
-			if (!visitedFolders.add(folder)) {
-				// bail if we already visited this folder
-				return;
-			}
-
-			File gitAttr = new File(folder, Constants.DOT_GIT_ATTRIBUTES);
-			if (gitAttr.exists() && gitAttr.isFile()) {
-				gitAttrFiles.add(gitAttr);
-			}
-			File parentFile = folder.getParentFile();
-			if (parentFile != null) {
-				gitAttrAddWithParents(folder.getParentFile(), visitedFolders, gitAttrFiles);
-			}
 		}
 
 		private Runtime atRuntime() {

lib/src/main/java/com/diffplug/spotless/FileSignature.java

@@ -119,4 +119,13 @@ public static String pathNativeToUnix(String pathNative) {
 	public static String pathUnixToNative(String pathUnix) {
 		return LineEnding.nativeIsWin() ? pathUnix.replace('/', '\\') : pathUnix;
 	}
+
+	/** Asserts that child is a subpath of root. and returns the subpath. */
+	public static String subpath(String root, String child) {
+		if (child.startsWith(root)) {
+			return child.substring(root.length());
+		} else {
+			throw new IllegalArgumentException("Expected '" + child + "' to start with '" + root + "'");
+		}
+	}
 }

plugin-gradle/CHANGES.md

@@ -7,6 +7,8 @@ We adhere to the [keepachangelog](https://keepachangelog.com/en/1.0.0/) format (
 ## [Unreleased]
 ### Added
 * `prettier` will now autodetect the parser (and formatter) to use based on the filename, unless you override this using `config()` or `configFile()` with the option `parser` or `filepath`. ([#620](https://github.com/diffplug/spotless/pull/620))
+### Fixed
+* LineEndings.GIT_ATTRIBUTES is now a bit more efficient, and paves the way for remote build cache support in Gradle. ([#621](https://github.com/diffplug/spotless/pull/621))
 
 ## [4.4.0] - 2020-06-19
 ### Added

plugin-gradle/src/main/java/com/diffplug/gradle/spotless/FormatExtension.java

@@ -669,13 +669,10 @@ public EclipseWtpConfig eclipseWtp(EclipseWtpFormatterStep type, String version)
 	protected void setupTask(SpotlessTask task) {
 		task.setEncoding(getEncoding().name());
 		task.setExceptionPolicy(exceptionPolicy);
-		if (targetExclude == null) {
-			task.setTarget(target);
-		} else {
-			task.setTarget(target.minus(targetExclude));
-		}
+		FileCollection totalTarget = targetExclude == null ? target : target.minus(targetExclude);
+		task.setTarget(totalTarget);
 		task.setSteps(steps);
-		task.setLineEndingsPolicy(getLineEndings().createPolicy(getProject().getProjectDir(), () -> task.target));
+		task.setLineEndingsPolicy(getLineEndings().createPolicy(getProject().getProjectDir(), () -> totalTarget));
 		if (spotless.project != spotless.project.getRootProject()) {
 			spotless.getRegisterDependenciesTask().hookSubprojectTask(task);
 		}

plugin-gradle/src/main/java/com/diffplug/gradle/spotless/SpotlessTaskBase.java

@@ -63,7 +63,7 @@ public void setEncoding(String encoding) {
 		this.encoding = Objects.requireNonNull(encoding);
 	}
 
-	protected LineEnding.Policy lineEndingsPolicy = LineEnding.UNIX.createPolicy();
+	protected LineEnding.Policy lineEndingsPolicy;
 
 	@Input
 	public LineEnding.Policy getLineEndingsPolicy() {

plugin-gradle/src/test/java/com/diffplug/gradle/spotless/FormatTaskTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 DiffPlug
+ * Copyright 2016-2020 DiffPlug
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -37,14 +37,14 @@ public class FormatTaskTest extends ResourceHarness {
 	public void createTask() throws IOException {
 		Project project = TestProvisioner.gradleProject(rootFolder());
 		spotlessTask = project.getTasks().create("spotlessTaskUnderTest", SpotlessTask.class);
+		spotlessTask.setLineEndingsPolicy(LineEnding.UNIX.createPolicy());
 	}
 
 	@Test
 	public void testLineEndings() throws Exception {
 		File testFile = setFile("testFile").toContent("\r\n");
 		File outputFile = new File(spotlessTask.getOutputDirectory(), "testFile");
 
-		spotlessTask.setLineEndingsPolicy(LineEnding.UNIX.createPolicy());
 		spotlessTask.setTarget(Collections.singleton(testFile));
 		execute(spotlessTask);
 

testlib/src/test/java/com/diffplug/spotless/FileSignatureTest.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 DiffPlug
+ * Copyright 2016-2020 DiffPlug
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -56,4 +56,8 @@ private List<File> getTestFiles(final String[] paths) throws IOException {
 		return result;
 	}
 
+	@Test
+	public void testSubpath() {
+		assertThat(FileSignature.subpath("root/", "root/child")).isEqualTo("child");
+	}
 }