Skip to content

Commit 52c1b34

Browse files
lbergelsonlbergelson
authored
make ReadFilter and Annotation packages configurable (#5573)
* changing ReadFilter and Annotations packages be loaded from the config file instead of being hardcoded * closes #2155 * added a new method to base test runToolInNewJvm which lets you run a gatk tool in a clean jvm so you can alter static state like which config file was used. Co-authored-by: Louis Bergelson <[email protected]>
1 parent 688cef5 commit 52c1b34

File tree

13 files changed

+222
-38
lines changed

13 files changed

+222
-38
lines changed

‎src/main/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKAnnotationPluginDescriptor.java

+19-8
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
import org.broadinstitute.hellbender.tools.walkers.annotator.Annotation;
1212
import org.broadinstitute.hellbender.tools.walkers.annotator.PedigreeAnnotation;
1313
import org.broadinstitute.hellbender.utils.Utils;
14+
import org.broadinstitute.hellbender.utils.config.ConfigFactory;
15+
import org.broadinstitute.hellbender.utils.config.GATKConfig;
1416

1517
import java.io.File;
1618
import java.lang.reflect.Modifier;
@@ -32,10 +34,19 @@
3234
* NOTE: this class enforces that annotations with required arguments must see their arguments, yet this is not currently tested
3335
* as no such annotations exist in the GATK.
3436
*/
35-
public class GATKAnnotationPluginDescriptor extends CommandLinePluginDescriptor<Annotation> {
36-
//TODO this should be a configurable option or otherwise exposed to the user when configurations are fully supported.
37-
public static final String pluginPackageName = "org.broadinstitute.hellbender.tools.walkers.annotator";
38-
public static final Class<?> pluginBaseClass = org.broadinstitute.hellbender.tools.walkers.annotator.Annotation.class;
37+
public class GATKAnnotationPluginDescriptor extends CommandLinePluginDescriptor<Annotation> {
38+
/**
39+
* At startup, set the plugin package name to the one(s) in the configuration file.
40+
*/
41+
private static final List<String> PLUGIN_PACKAGE_NAMES;
42+
static {
43+
// Get our configuration:
44+
final GATKConfig config = ConfigFactory.getInstance().getGATKConfig();
45+
// Exclude abstract classes and interfaces from the list of discovered codec classes
46+
PLUGIN_PACKAGE_NAMES = Collections.unmodifiableList(config.annotation_packages());
47+
}
48+
49+
private static final Class<?> PLUGIN_BASE_CLASS = org.broadinstitute.hellbender.tools.walkers.annotator.Annotation.class;
3950

4051
protected transient Logger logger = LogManager.getLogger(this.getClass());
4152

@@ -74,7 +85,7 @@ public class GATKAnnotationPluginDescriptor extends CommandLinePluginDescriptor
7485
*/
7586
@Override
7687
public Class<?> getPluginBaseClass() {
77-
return pluginBaseClass;
88+
return PLUGIN_BASE_CLASS;
7889
}
7990

8091
/**
@@ -84,7 +95,7 @@ public Class<?> getPluginBaseClass() {
8495
*/
8596
@Override
8697
public List<String> getPackageNames() {
87-
return Collections.singletonList(pluginPackageName);
98+
return PLUGIN_PACKAGE_NAMES;
8899
}
89100

90101
/**
@@ -125,7 +136,7 @@ public GATKAnnotationPluginDescriptor(final GATKAnnotationArgumentCollection use
125136
}
126137
/**
127138
* Constructor that allows client tools to specify what annotations (optionally with parameters specified) to use as their defaults
128-
* before discovery of user specified annotations. Defaults to using an empty GATKAnnotationArgumentCollection object.
139+
* before discovery of user specified annotations. Defaults to using an empty GATKAnnotationArgumentCollection object.
129140
*
130141
* @param toolDefaultAnnotations Default annotations that may be supplied with arguments
131142
* on the command line. May be null.
@@ -219,7 +230,7 @@ private void populateAnnotationGroups(final String simpleName, final Annotation
219230
// extend Annotation, groups are discovered by interrogating annotations for their interfaces and
220231
// associating the discovered annotations with their defined groups.
221232
// If a duplicate annotation is added, the group will opt to keep the old instantiation around
222-
if ((inter != pluginBaseClass) && (pluginBaseClass.isAssignableFrom(inter))) {
233+
if ((inter != PLUGIN_BASE_CLASS) && (PLUGIN_BASE_CLASS.isAssignableFrom(inter))) {
223234
Map<String, Annotation> groupIdentity = (discoveredGroups.containsKey(inter.getSimpleName()) ? discoveredGroups.get(inter.getSimpleName()) : new HashMap<>());
224235
groupIdentity.putIfAbsent(simpleName, annot);
225236
discoveredGroups.put(inter.getSimpleName(), groupIdentity);

‎src/main/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKReadFilterPluginDescriptor.java

+14-2
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,8 @@
1111
import org.broadinstitute.hellbender.engine.filters.CountingReadFilter;
1212
import org.broadinstitute.hellbender.engine.filters.ReadFilter;
1313
import org.broadinstitute.hellbender.utils.Utils;
14+
import org.broadinstitute.hellbender.utils.config.ConfigFactory;
15+
import org.broadinstitute.hellbender.utils.config.GATKConfig;
1416

1517
import java.io.IOException;
1618
import java.util.*;
@@ -30,7 +32,17 @@ private void readObject(java.io.ObjectInputStream in)
3032
logger = LogManager.getLogger(this.getClass()); // Logger is not serializable (even by Kryo)
3133
}
3234

33-
private static final String pluginPackageName = "org.broadinstitute.hellbender.engine.filters";
35+
/**
36+
* At startup, set the plugin package name to the one(s) in the configuration file.
37+
*/
38+
private static final List<String> PLUGIN_PACKAGE_NAMES;
39+
static {
40+
// Get our configuration:
41+
final GATKConfig config = ConfigFactory.getInstance().getGATKConfig();
42+
// Exclude abstract classes and interfaces from the list of discovered codec classes
43+
PLUGIN_PACKAGE_NAMES = Collections.unmodifiableList(config.read_filter_packages());
44+
}
45+
3446
private static final Class<ReadFilter> pluginBaseClass = org.broadinstitute.hellbender.engine.filters.ReadFilter.class;
3547
private static final String READ_PLUGIN_DISPLAY_NAME = "readFilter";
3648

@@ -111,7 +123,7 @@ public String getDisplayName() {
111123
* @return
112124
*/
113125
@Override
114-
public List<String> getPackageNames() {return Collections.singletonList(pluginPackageName);}
126+
public List<String> getPackageNames() {return PLUGIN_PACKAGE_NAMES;}
115127

116128
@Override
117129
public boolean includePluginClass(Class<?> c) {

‎src/main/java/org/broadinstitute/hellbender/tools/walkers/annotator/VariantAnnotatorEngine.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,17 @@
11
package org.broadinstitute.hellbender.tools.walkers.annotator;
22

3-
import com.google.common.collect.Sets;
43
import htsjdk.variant.variantcontext.*;
54
import htsjdk.variant.vcf.*;
65
import org.apache.logging.log4j.LogManager;
76
import org.apache.logging.log4j.Logger;
8-
import org.broadinstitute.hellbender.engine.*;
97
import org.broadinstitute.hellbender.engine.FeatureContext;
8+
import org.broadinstitute.hellbender.engine.FeatureDataSource;
109
import org.broadinstitute.hellbender.engine.FeatureInput;
1110
import org.broadinstitute.hellbender.engine.ReferenceContext;
1211
import org.broadinstitute.hellbender.exceptions.GATKException;
1312
import org.broadinstitute.hellbender.exceptions.UserException;
1413
import org.broadinstitute.hellbender.tools.walkers.annotator.allelespecific.ReducibleAnnotation;
1514
import org.broadinstitute.hellbender.tools.walkers.annotator.allelespecific.ReducibleAnnotationData;
16-
import org.broadinstitute.hellbender.utils.ClassUtils;
1715
import org.broadinstitute.hellbender.utils.Utils;
1816
import org.broadinstitute.hellbender.utils.genotyper.ReadLikelihoods;
1917
import org.broadinstitute.hellbender.utils.variant.GATKVariantContextUtils;

‎src/main/java/org/broadinstitute/hellbender/utils/ClassUtils.java

+22-6
Original file line numberDiff line numberDiff line change
@@ -49,17 +49,33 @@ public static <T> List<T> makeInstancesOfSubclasses(final Class<? extends T> cla
4949
final List<T> results = new ArrayList<>(classes.size());
5050

5151
for (final Class<?> found: classes){
52-
if (canMakeInstances(found)){
53-
try {
54-
results.add((T)found.newInstance());
55-
} catch (InstantiationException | IllegalAccessException e) {
56-
throw new GATKException("Problem making an instance of " + found + " Do check that the class has a non-arg constructor", e);
57-
}
52+
final T instance = (T) makeInstanceOf(found);
53+
if (instance != null) {
54+
results.add(instance);
5855
}
5956
}
6057
return results;
6158
}
6259

60+
/**
61+
* Create objects of a concrete class.
62+
*
63+
* The public no-arg constructor is called to create the objects.
64+
*
65+
* @param clazz class to be instantiated
66+
* @return new object or {@code null} if cannot be instantiated.
67+
*/
68+
public static <T> T makeInstanceOf(final Class<T> clazz) {
69+
if (canMakeInstances(clazz)) {
70+
try {
71+
return clazz.newInstance();
72+
} catch (final InstantiationException | IllegalAccessException e) {
73+
throw new GATKException("Problem making an instance of " + clazz + " Do check that the class has a non-arg constructor", e);
74+
}
75+
}
76+
return null;
77+
}
78+
6379
/**
6480
* Finds sub-interfaces of the given interface (in the same package) and returns their simple names.
6581
*/

‎src/main/java/org/broadinstitute/hellbender/utils/config/GATKConfig.java

+11-1
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
package org.broadinstitute.hellbender.utils.config;
22

3+
import com.google.common.annotations.VisibleForTesting;
34
import org.aeonbits.owner.Accessible;
4-
import org.aeonbits.owner.Mutable;
55
import org.aeonbits.owner.Config.LoadPolicy;
66
import org.aeonbits.owner.Config.LoadType;
77
import org.aeonbits.owner.Config.Sources;
8+
import org.aeonbits.owner.Mutable;
89

910
import java.util.List;
1011

@@ -52,6 +53,9 @@ public interface GATKConfig extends Mutable, Accessible {
5253
*/
5354
String CONFIG_FILE_VARIABLE_CLASS_PATH = "GATKConfig.classPathToGatkConfig";
5455

56+
@VisibleForTesting
57+
public String DEFAULT_ANNOTATION_PACKAGES = "org.broadinstitute.hellbender.tools.walkers.annotator";
58+
5559
// =================================================================================================================
5660
// =================================================================================================================
5761
// System Options:
@@ -144,6 +148,12 @@ public interface GATKConfig extends Mutable, Accessible {
144148
@DefaultValue("htsjdk.variant,htsjdk.tribble,org.broadinstitute.hellbender.utils.codecs")
145149
List<String> codec_packages();
146150

151+
@DefaultValue("org.broadinstitute.hellbender.engine.filters")
152+
List<String> read_filter_packages();
153+
154+
@DefaultValue(DEFAULT_ANNOTATION_PACKAGES)
155+
List<String> annotation_packages();
156+
147157
// ----------------------------------------------------------
148158
// GATKTool Options:
149159
// ----------------------------------------------------------

‎src/main/resources/org/broadinstitute/hellbender/utils/config/GATKConfig.properties

+2
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,8 @@ spark.executor.extraJavaOptions =
7373
# ---------------------------------------------------------------------------------
7474

7575
codec_packages = htsjdk.variant, htsjdk.tribble, org.broadinstitute.hellbender.utils.codecs
76+
read_filter_packages = org.broadinstitute.hellbender.engine.filters
77+
annotation_packages = org.broadinstitute.hellbender.tools.walkers.annotator
7678

7779
# ---------------------------------------------------------------------------------
7880
# GATKTool Options:

‎src/test/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKAnnotationPluginDescriptorUnitTest.java

+47-7
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,34 @@
11
package org.broadinstitute.hellbender.cmdline.GATKPlugin;
22

3-
import org.broadinstitute.barclay.argparser.ClassFinder;
43
import htsjdk.variant.variantcontext.*;
54
import org.apache.commons.io.output.NullOutputStream;
6-
import org.broadinstitute.barclay.argparser.CommandLineArgumentParser;
7-
import org.broadinstitute.barclay.argparser.CommandLineException;
8-
import org.broadinstitute.barclay.argparser.CommandLineParser;
5+
import org.broadinstitute.barclay.argparser.*;
96
import org.broadinstitute.hellbender.GATKBaseTest;
7+
import org.broadinstitute.hellbender.cmdline.GATKPlugin.testpluggables.TestAnnotation;
108
import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions;
9+
import org.broadinstitute.hellbender.cmdline.TestProgramGroup;
1110
import org.broadinstitute.hellbender.engine.FeatureContext;
11+
import org.broadinstitute.hellbender.engine.GATKTool;
1212
import org.broadinstitute.hellbender.engine.ReferenceContext;
1313
import org.broadinstitute.hellbender.exceptions.GATKException;
14+
import org.broadinstitute.hellbender.testutils.ArgumentsBuilder;
1415
import org.broadinstitute.hellbender.tools.walkers.annotator.*;
1516
import org.broadinstitute.hellbender.tools.walkers.annotator.allelespecific.AS_RMSMappingQuality;
1617
import org.broadinstitute.hellbender.tools.walkers.annotator.allelespecific.AS_StandardAnnotation;
18+
import org.broadinstitute.hellbender.utils.config.GATKConfig;
1719
import org.broadinstitute.hellbender.utils.genotyper.ReadLikelihoods;
1820
import org.broadinstitute.hellbender.utils.variant.GATKVCFConstants;
1921
import org.mockito.internal.util.collections.Sets;
22+
import org.mockito.internal.util.io.IOUtil;
2023
import org.testng.Assert;
2124
import org.testng.annotations.DataProvider;
2225
import org.testng.annotations.Test;
2326
import org.testng.collections.Lists;
2427

28+
import java.io.File;
29+
import java.io.IOException;
2530
import java.io.PrintStream;
31+
import java.nio.file.Files;
2632
import java.util.*;
2733
import java.util.stream.Collectors;
2834
import java.util.stream.Stream;
@@ -456,7 +462,7 @@ public void testUseAllAnnotations(){
456462
List<Annotation> annots = instantiateAnnotations(clp);
457463

458464
ClassFinder finder = new ClassFinder();
459-
finder.find(GATKAnnotationPluginDescriptor.pluginPackageName, Annotation.class);
465+
finder.find(GATKConfig.DEFAULT_ANNOTATION_PACKAGES, Annotation.class);
460466

461467
Set<Class<?>> classes = finder.getConcreteClasses();
462468
Assert.assertFalse(classes.isEmpty());
@@ -477,7 +483,7 @@ public void testUseAllAnnotationsDisableToolDefaultAnnotaiotns(){
477483
List<Annotation> annots = instantiateAnnotations(clp);
478484

479485
ClassFinder finder = new ClassFinder();
480-
finder.find(GATKAnnotationPluginDescriptor.pluginPackageName, Annotation.class);
486+
finder.find(GATKConfig.DEFAULT_ANNOTATION_PACKAGES, Annotation.class);
481487

482488
Set<Class<?>> classes = finder.getConcreteClasses();
483489
Assert.assertFalse(classes.isEmpty());
@@ -502,7 +508,7 @@ public void testIncludeAllExcludeIndividual(){
502508
Assert.assertFalse(annots.stream().anyMatch(a -> a.getClass()==AS_StandardAnnotation.class));
503509

504510
ClassFinder finder = new ClassFinder();
505-
finder.find(GATKAnnotationPluginDescriptor.pluginPackageName, Annotation.class);
511+
finder.find(GATKConfig.DEFAULT_ANNOTATION_PACKAGES, Annotation.class);
506512

507513
Set<Class<?>> classes = finder.getConcreteClasses();
508514
classes.remove(Coverage.class);
@@ -628,6 +634,7 @@ public List<String> getKeyNames() {
628634
return Collections.singletonList("Test");
629635
}
630636
}
637+
631638
static class testParentAnnotation extends InfoFieldAnnotation implements ParentAnnotationGroup {
632639
boolean dontAnnotate = false;
633640

@@ -651,4 +658,37 @@ public List<String> getKeyNames() {
651658
}
652659
}
653660

661+
@CommandLineProgramProperties(summary="test tool to check annotation loading",
662+
oneLineSummary = "test tool to check annotation loading",
663+
programGroup = TestProgramGroup.class,
664+
omitFromCommandLine = true)
665+
public static class TestAnnotationsTool extends GATKTool {
666+
@Argument(fullName = StandardArgumentDefinitions.OUTPUT_LONG_NAME)
667+
String output;
668+
669+
@Override
670+
public boolean useVariantAnnotations(){
671+
return true;
672+
}
673+
674+
@Override
675+
public void traverse() {
676+
String annotations = makeVariantAnnotations().stream()
677+
.map(a -> a.getClass().getSimpleName())
678+
.collect(Collectors.joining("\n"));
679+
IOUtil.writeText(annotations, new File(output));
680+
}
681+
}
682+
683+
@Test
684+
public void testConfigFileControlsAnnotationPackages() throws IOException {
685+
final File output = createTempFile("annotations", "txt");
686+
final File configFile = new File(packageRootTestDir + "cmdline/GATKPlugin/changePluginPackages.properties");
687+
ArgumentsBuilder args = new ArgumentsBuilder();
688+
args.addArgument(StandardArgumentDefinitions.ANNOTATION_LONG_NAME, TestAnnotation.class.getSimpleName())
689+
.addFileArgument(StandardArgumentDefinitions.GATK_CONFIG_FILE_OPTION, configFile)
690+
.addOutput(output);
691+
runToolInNewJVM("TestAnnotationsTool", args);
692+
Assert.assertEquals(Files.readAllLines(output.toPath()), Collections.singletonList(TestAnnotation.class.getSimpleName()));
693+
}
654694
}

‎src/test/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKReadFilterPluginDescriptorTest.java

+42-3
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,26 @@
44
import htsjdk.samtools.SAMFileHeader;
55
import htsjdk.samtools.TextCigarCodec;
66
import org.apache.commons.io.output.NullOutputStream;
7-
import org.broadinstitute.barclay.argparser.CommandLineArgumentParser;
8-
import org.broadinstitute.barclay.argparser.CommandLineException;
9-
import org.broadinstitute.barclay.argparser.CommandLineParser;
7+
import org.broadinstitute.barclay.argparser.*;
108
import org.broadinstitute.hellbender.GATKBaseTest;
9+
import org.broadinstitute.hellbender.cmdline.GATKPlugin.testpluggables.TestReadFilter;
1110
import org.broadinstitute.hellbender.cmdline.ReadFilterArgumentDefinitions;
11+
import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions;
12+
import org.broadinstitute.hellbender.cmdline.TestProgramGroup;
13+
import org.broadinstitute.hellbender.engine.GATKTool;
1214
import org.broadinstitute.hellbender.engine.filters.*;
15+
import org.broadinstitute.hellbender.testutils.ArgumentsBuilder;
1316
import org.broadinstitute.hellbender.utils.read.ArtificialReadUtils;
1417
import org.broadinstitute.hellbender.utils.read.GATKRead;
18+
import org.mockito.internal.util.io.IOUtil;
1519
import org.testng.Assert;
1620
import org.testng.annotations.DataProvider;
1721
import org.testng.annotations.Test;
1822

23+
import java.io.File;
24+
import java.io.IOException;
1925
import java.io.PrintStream;
26+
import java.nio.file.Files;
2027
import java.util.ArrayList;
2128
import java.util.Arrays;
2229
import java.util.Collections;
@@ -786,5 +793,37 @@ private void setupReadNameTest(SetupTest setup) {
786793
private void setupSampleTest(SetupTest setup) {
787794
setup.hdr.getReadGroup(setup.read.getReadGroup()).setSample(setup.argValue);
788795
}
796+
797+
@CommandLineProgramProperties(summary="test tool to check ReadFilter loading",
798+
oneLineSummary = "test tool to check ReadFilter loading",
799+
programGroup = TestProgramGroup.class,
800+
omitFromCommandLine = true)
801+
public static class TestReadFiltersTool extends GATKTool {
802+
@Argument(fullName = StandardArgumentDefinitions.OUTPUT_LONG_NAME)
803+
String output;
804+
805+
@Override
806+
public void traverse() {
807+
String readFilters = getCommandLineParser()
808+
.getPluginDescriptor(GATKReadFilterPluginDescriptor.class)
809+
.getResolvedInstances()
810+
.stream()
811+
.map(a -> a.getClass().getSimpleName())
812+
.collect(Collectors.joining("\n"));
813+
IOUtil.writeText(readFilters, new File(output));
814+
}
815+
}
816+
817+
@Test
818+
public void testConfigFileControlsReadFiltersPackages() throws IOException {
819+
final File output = createTempFile("annotations", "txt");
820+
final File configFile = new File(packageRootTestDir + "cmdline/GATKPlugin/changePluginPackages.properties");
821+
ArgumentsBuilder args = new ArgumentsBuilder();
822+
args.addArgument(ReadFilterArgumentDefinitions.READ_FILTER_LONG_NAME, TestReadFilter.class.getSimpleName())
823+
.addFileArgument(StandardArgumentDefinitions.GATK_CONFIG_FILE_OPTION, configFile)
824+
.addOutput(output);
825+
runToolInNewJVM("TestReadFiltersTool", args);
826+
Assert.assertEquals(Files.readAllLines(output.toPath()), Arrays.asList(WellformedReadFilter.class.getSimpleName(), TestReadFilter.class.getSimpleName()));
827+
}
789828

790829
}

0 commit comments

Comments
 (0)