Add support for docker hub private registry credentials (#845)

* Fix authentication for use of base64-encoded credentials and credential helpers with private registries and docker hub

* Update following review by @bsideup

* Update to docker-java 3.1.0-rc-4

* fix shading

Author: rnorth

core/build.gradle

@@ -35,6 +35,7 @@ shadowJar {
         'META-INF/services/org.glassfish.hk2.extension.*',
         'META-INF/services/org.jvnet.hk2.external.generator.*',
         'META-INF/services/org.glassfish.jersey.internal.spi.*',
+        'META-INF/services/java.security.Provider',
         'mozilla/public-suffix-list.txt',
     ].each { exclude(it) }
 
@@ -95,7 +96,7 @@ dependencies {
         exclude(group: "net.java.dev.jna")
     }
 
-    shaded ('com.github.docker-java:docker-java:3.1.0-rc-3') {
+    shaded ('com.github.docker-java:docker-java:3.1.0-rc-4') {
         exclude(group: 'org.glassfish.jersey.core')
         exclude(group: 'org.glassfish.jersey.connectors')
         exclude(group: 'log4j')

core/src/jarFileTest/java/org/testcontainers/JarFileShadingTest.java

@@ -42,8 +42,8 @@ public void testMetaInf() throws Exception {
         );
 
         assertThatFileList(root.resolve("META-INF").resolve("native")).containsOnly(
-                "liborg-testcontainers-shaded-netty-transport-native-epoll.so",
-                "liborg-testcontainers-shaded-netty-transport-native-kqueue.jnilib"
+                "liborg-testcontainers-shaded-netty_transport_native_epoll_x86_64.so",
+                "liborg-testcontainers-shaded-netty_transport_native_kqueue_x86_64.jnilib"
         );
     }
 

core/src/main/java/org/testcontainers/dockerclient/DockerClientProviderStrategy.java

@@ -12,6 +12,7 @@
 import org.rnorth.ducttape.unreliables.Unreliables;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
+import org.testcontainers.dockerclient.auth.AuthDelegatingDockerClientConfig;
 import org.testcontainers.dockerclient.transport.TestcontainersDockerCmdExecFactory;
 import org.testcontainers.dockerclient.transport.okhttp.OkHttpDockerCmdExecFactory;
 import org.testcontainers.utility.TestcontainersConfiguration;
@@ -39,6 +40,8 @@ public abstract class DockerClientProviderStrategy {
 
     private static final AtomicBoolean FAIL_FAST_ALWAYS = new AtomicBoolean(false);
 
+    protected static final Logger LOGGER = LoggerFactory.getLogger(DockerClientProviderStrategy.class);
+
     /**
      * @throws InvalidConfigurationException if this strategy fails
      */
@@ -64,8 +67,6 @@ protected int getPriority() {
         return 0;
     }
 
-    protected static final Logger LOGGER = LoggerFactory.getLogger(DockerClientProviderStrategy.class);
-
     /**
      * Determine the right DockerClientConfig to use for building clients by trial-and-error.
      *
@@ -166,7 +167,7 @@ public DockerClient getClient() {
 
     protected DockerClient getClientForConfig(DockerClientConfig config) {
         DockerClientBuilder clientBuilder = DockerClientBuilder
-            .getInstance(config);
+            .getInstance(new AuthDelegatingDockerClientConfig(config));
 
         String transportType = TestcontainersConfiguration.getInstance().getTransportType();
         if ("okhttp".equals(transportType)) {

core/src/main/java/org/testcontainers/dockerclient/auth/AuthDelegatingDockerClientConfig.java

@@ -0,0 +1,52 @@
+package org.testcontainers.dockerclient.auth;
+
+import com.github.dockerjava.api.model.AuthConfig;
+import com.github.dockerjava.core.DockerClientConfig;
+import lombok.experimental.Delegate;
+import lombok.extern.slf4j.Slf4j;
+import org.testcontainers.utility.DockerImageName;
+import org.testcontainers.utility.RegistryAuthLocator;
+
+import static org.testcontainers.utility.AuthConfigUtil.toSafeString;
+
+/**
+ * Facade implementation for {@link DockerClientConfig} which overrides how authentication
+ * configuration is obtained. A delegate {@link DockerClientConfig} will be called first
+ * to try and obtain auth credentials, but after that {@link RegistryAuthLocator} will be
+ * used to try and improve the auth resolution (e.g. using credential helpers).
+ */
+@Slf4j
+public class AuthDelegatingDockerClientConfig implements DockerClientConfig {
+
+    @Delegate(excludes = DelegateExclusions.class)
+    private DockerClientConfig delegate;
+
+    public AuthDelegatingDockerClientConfig(DockerClientConfig delegate) {
+        this.delegate = delegate;
+    }
+
+    public AuthConfig effectiveAuthConfig(String imageName) {
+        // allow docker-java auth config to be used as a fallback
+        AuthConfig fallbackAuthConfig;
+        try {
+            fallbackAuthConfig = delegate.effectiveAuthConfig(imageName);
+        } catch (Exception e) {
+            log.debug("Delegate call to effectiveAuthConfig failed with cause: '{}'. " +
+                "Resolution of auth config will continue using RegistryAuthLocator.",
+                e.getMessage());
+            fallbackAuthConfig = new AuthConfig();
+        }
+
+        // try and obtain more accurate auth config using our resolution
+        final DockerImageName parsed = new DockerImageName(imageName);
+        final AuthConfig effectiveAuthConfig = RegistryAuthLocator.instance()
+            .lookupAuthConfig(parsed, fallbackAuthConfig);
+
+        log.debug("Effective auth config [{}]", toSafeString(effectiveAuthConfig));
+        return effectiveAuthConfig;
+    }
+
+    private interface DelegateExclusions {
+        AuthConfig effectiveAuthConfig(String imageName);
+    }
+}

core/src/main/java/org/testcontainers/dockerclient/transport/TestcontainersDockerCmdExecFactory.java

@@ -80,16 +80,21 @@ public void init(DockerClientConfig dockerClientConfig) {
         bootstrap = new Bootstrap();
 
         String scheme = dockerClientConfig.getDockerHost().getScheme();
+        String host = "";
 
         if ("unix".equals(scheme)) {
             nettyInitializer = new UnixDomainSocketInitializer();
+            host = "DUMMY";
         } else if ("tcp".equals(scheme)) {
             nettyInitializer = new InetSocketInitializer();
+            host = dockerClientConfig.getDockerHost().getHost() + ":"
+                + Integer.toString(dockerClientConfig.getDockerHost().getPort());
         }
 
         eventLoopGroup = nettyInitializer.init(bootstrap, dockerClientConfig);
 
-        baseResource = new NettyWebTarget(this::connect).path(dockerClientConfig.getApiVersion().asWebPathPart());
+        baseResource = new NettyWebTarget(this::connect, host)
+            .path(dockerClientConfig.getApiVersion().asWebPathPart());
     }
 
     private DuplexChannel connect() {

core/src/main/java/org/testcontainers/dockerclient/transport/okhttp/NamedPipeSocketFactory.java

@@ -1,20 +1,18 @@
 package org.testcontainers.dockerclient.transport.okhttp;
 
+import lombok.EqualsAndHashCode;
 import lombok.SneakyThrows;
 import lombok.Value;
 import org.scalasbt.ipcsocket.Win32NamedPipeSocket;
 
 import javax.net.SocketFactory;
-import java.io.FilterInputStream;
-import java.io.FilterOutputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
+import java.io.*;
 import java.net.InetAddress;
 import java.net.Socket;
 import java.net.SocketAddress;
 
 @Value
+@EqualsAndHashCode(callSuper = false)
 public class NamedPipeSocketFactory extends SocketFactory {
 
     String socketPath;

core/src/main/java/org/testcontainers/dockerclient/transport/okhttp/UnixSocketFactory.java

@@ -1,5 +1,6 @@
 package org.testcontainers.dockerclient.transport.okhttp;
 
+import lombok.EqualsAndHashCode;
 import lombok.SneakyThrows;
 import lombok.Value;
 import org.scalasbt.ipcsocket.UnixDomainSocket;
@@ -15,6 +16,7 @@
 import java.net.SocketAddress;
 
 @Value
+@EqualsAndHashCode(callSuper = false)
 public class UnixSocketFactory extends SocketFactory {
 
     String socketPath;

core/src/main/java/org/testcontainers/images/RemoteDockerImage.java

@@ -3,7 +3,6 @@
 import com.github.dockerjava.api.DockerClient;
 import com.github.dockerjava.api.command.ListImagesCmd;
 import com.github.dockerjava.api.exception.DockerClientException;
-import com.github.dockerjava.api.model.AuthConfig;
 import com.github.dockerjava.api.model.Image;
 import com.github.dockerjava.core.command.PullImageResultCallback;
 import lombok.NonNull;
@@ -15,7 +14,6 @@
 import org.testcontainers.utility.DockerImageName;
 import org.testcontainers.utility.DockerLoggerFactory;
 import org.testcontainers.utility.LazyFuture;
-import org.testcontainers.utility.RegistryAuthLocator;
 
 import java.util.HashSet;
 import java.util.List;
@@ -95,14 +93,10 @@ protected final String resolve() {
 
                 // The image is not available locally - pull it
                 try {
-                    final RegistryAuthLocator authLocator = new RegistryAuthLocator(dockerClient.authConfig());
-                    final AuthConfig effectiveAuthConfig = authLocator.lookupAuthConfig(imageName);
-
                     final PullImageResultCallback callback = new PullImageResultCallback();
                     dockerClient
                         .pullImageCmd(imageName.getUnversionedPart())
                         .withTag(imageName.getVersionPart())
-                        .withAuthConfig(effectiveAuthConfig)
                         .exec(callback);
                     callback.awaitCompletion();
                     AVAILABLE_IMAGE_NAME_CACHE.add(imageName);

core/src/main/java/org/testcontainers/utility/AuthConfigUtil.java

@@ -0,0 +1,34 @@
+package org.testcontainers.utility;
+
+import com.github.dockerjava.api.model.AuthConfig;
+import com.google.common.base.MoreObjects;
+import lombok.experimental.UtilityClass;
+import org.jetbrains.annotations.NotNull;
+
+import static com.google.common.base.Strings.isNullOrEmpty;
+
+/**
+ * TODO: Javadocs
+ */
+@UtilityClass
+public class AuthConfigUtil {
+    public static String toSafeString(AuthConfig authConfig) {
+        if (authConfig == null) {
+            return "null";
+        }
+
+        return MoreObjects.toStringHelper(authConfig)
+            .add("username", authConfig.getUsername())
+            .add("password", obfuscated(authConfig.getPassword()))
+            .add("auth", obfuscated(authConfig.getAuth()))
+            .add("email", authConfig.getEmail())
+            .add("registryAddress", authConfig.getRegistryAddress())
+            .add("registryToken", obfuscated(authConfig.getRegistrytoken()))
+            .toString();
+    }
+
+    @NotNull
+    private static String obfuscated(String value) {
+        return isNullOrEmpty(value) ? "blank" : "hidden non-blank value";
+    }
+}

core/src/main/java/org/testcontainers/utility/DockerImageName.java

@@ -7,7 +7,7 @@
 
 import java.util.regex.Pattern;
 
-@EqualsAndHashCode
+@EqualsAndHashCode(exclude = "rawName")
 public final class DockerImageName {
 
     /* Regex patterns used for validation */

core/src/main/java/org/testcontainers/utility/LogUtils.java

@@ -2,12 +2,15 @@
 
 import com.github.dockerjava.api.DockerClient;
 import com.github.dockerjava.api.command.LogContainerCmd;
+import com.github.dockerjava.api.model.AuthConfig;
+import com.google.common.base.MoreObjects;
 import lombok.experimental.UtilityClass;
 import org.testcontainers.containers.output.FrameConsumerResultCallback;
 import org.testcontainers.containers.output.OutputFrame;
 
 import java.util.function.Consumer;
 
+import static com.google.common.base.Strings.isNullOrEmpty;
 import static org.testcontainers.containers.output.OutputFrame.OutputType.STDERR;
 import static org.testcontainers.containers.output.OutputFrame.OutputType.STDOUT;
 
@@ -39,4 +42,5 @@ public void followOutput(DockerClient dockerClient, String containerId,
     public void followOutput(DockerClient dockerClient, String containerId, Consumer<OutputFrame> consumer) {
         followOutput(dockerClient, containerId, consumer, STDOUT, STDERR);
     }
+
 }