Skip to content

Commit 6dbc1c5

Browse files
lbergelsonlbergelson
committed
Rebasing and updating, adding tests.
* added tests to show that the Annotation /ReadFilter loading is controlled by the config file * added a new method to base test runToolInNewJvm which lets you run a gatk tool in a clean jvm so you can alter static state like which config file was used. Co-authored-by: Louis Bergelson <[email protected]>
1 parent f07cef9 commit 6dbc1c5

File tree

11 files changed

+164
-29
lines changed

11 files changed

+164
-29
lines changed

‎src/main/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKAnnotationPluginDescriptor.java

+5-5
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@
3434
* NOTE: this class enforces that annotations with required arguments must see their arguments, yet this is not currently tested
3535
* as no such annotations exist in the GATK.
3636
*/
37-
public class GATKAnnotationPluginDescriptor extends CommandLinePluginDescriptor<Annotation> {
37+
public class GATKAnnotationPluginDescriptor extends CommandLinePluginDescriptor<Annotation> {
3838
/**
3939
* At startup, set the plugin package name to the one(s) in the configuration file.
4040
*/
@@ -43,10 +43,10 @@ public class GATKAnnotationPluginDescriptor extends CommandLinePluginDescriptor
4343
// Get our configuration:
4444
final GATKConfig config = ConfigFactory.getInstance().getGATKConfig();
4545
// Exclude abstract classes and interfaces from the list of discovered codec classes
46-
PLUGIN_PACKAGE_NAMES = Collections.unmodifiableList(config.codec_packages());
46+
PLUGIN_PACKAGE_NAMES = Collections.unmodifiableList(config.annotation_packages());
4747
}
4848

49-
private static final Class<?> pluginBaseClass = org.broadinstitute.hellbender.tools.walkers.annotator.Annotation.class;
49+
private static final Class<?> PLUGIN_BASE_CLASS = org.broadinstitute.hellbender.tools.walkers.annotator.Annotation.class;
5050

5151
protected transient Logger logger = LogManager.getLogger(this.getClass());
5252

@@ -85,7 +85,7 @@ public class GATKAnnotationPluginDescriptor extends CommandLinePluginDescriptor
8585
*/
8686
@Override
8787
public Class<?> getPluginBaseClass() {
88-
return pluginBaseClass;
88+
return PLUGIN_BASE_CLASS;
8989
}
9090

9191
/**
@@ -230,7 +230,7 @@ private void populateAnnotationGroups(final String simpleName, final Annotation
230230
// extend Annotation, groups are discovered by interrogating annotations for their interfaces and
231231
// associating the discovered annotations with their defined groups.
232232
// If a duplicate annotation is added, the group will opt to keep the old instantiation around
233-
if ((inter != pluginBaseClass) && (pluginBaseClass.isAssignableFrom(inter))) {
233+
if ((inter != PLUGIN_BASE_CLASS) && (PLUGIN_BASE_CLASS.isAssignableFrom(inter))) {
234234
Map<String, Annotation> groupIdentity = (discoveredGroups.containsKey(inter.getSimpleName()) ? discoveredGroups.get(inter.getSimpleName()) : new HashMap<>());
235235
groupIdentity.putIfAbsent(simpleName, annot);
236236
discoveredGroups.put(inter.getSimpleName(), groupIdentity);

‎src/main/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKReadFilterPluginDescriptor.java

+1-3
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
import org.apache.logging.log4j.LogManager;
66
import org.apache.logging.log4j.Logger;
77
import org.broadinstitute.barclay.argparser.ArgumentCollection;
8-
import org.broadinstitute.barclay.argparser.ClassFinder;
98
import org.broadinstitute.barclay.argparser.CommandLineException;
109
import org.broadinstitute.barclay.argparser.CommandLinePluginDescriptor;
1110
import org.broadinstitute.hellbender.cmdline.ReadFilterArgumentDefinitions;
@@ -38,11 +37,10 @@ private void readObject(java.io.ObjectInputStream in)
3837
*/
3938
private static final List<String> PLUGIN_PACKAGE_NAMES;
4039
static {
41-
4240
// Get our configuration:
4341
final GATKConfig config = ConfigFactory.getInstance().getGATKConfig();
4442
// Exclude abstract classes and interfaces from the list of discovered codec classes
45-
PLUGIN_PACKAGE_NAMES = Collections.unmodifiableList(config.codec_packages());
43+
PLUGIN_PACKAGE_NAMES = Collections.unmodifiableList(config.read_filter_packages());
4644
}
4745

4846
private static final Class<ReadFilter> pluginBaseClass = org.broadinstitute.hellbender.engine.filters.ReadFilter.class;

‎src/main/java/org/broadinstitute/hellbender/utils/config/GATKConfig.java

+5-1
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
package org.broadinstitute.hellbender.utils.config;
22

3+
import com.netflix.servo.util.VisibleForTesting;
34
import org.aeonbits.owner.Accessible;
45
import org.aeonbits.owner.Mutable;
56
import org.aeonbits.owner.Config.LoadPolicy;
@@ -52,6 +53,9 @@ public interface GATKConfig extends Mutable, Accessible {
5253
*/
5354
String CONFIG_FILE_VARIABLE_CLASS_PATH = "GATKConfig.classPathToGatkConfig";
5455

56+
@VisibleForTesting
57+
public String DEFAULT_ANNOTATION_PACKAGES = "org.broadinstitute.hellbender.tools.walkers.annotator";
58+
5559
// =================================================================================================================
5660
// =================================================================================================================
5761
// System Options:
@@ -147,7 +151,7 @@ public interface GATKConfig extends Mutable, Accessible {
147151
@DefaultValue("org.broadinstitute.hellbender.engine.filters")
148152
List<String> read_filter_packages();
149153

150-
@DefaultValue("org.broadinstitute.hellbender.tools.walkers.annotator")
154+
@DefaultValue(DEFAULT_ANNOTATION_PACKAGES)
151155
List<String> annotation_packages();
152156

153157
// ----------------------------------------------------------

‎src/main/resources/org/broadinstitute/hellbender/utils/config/GATKConfig.properties

+2-2
Original file line numberDiff line numberDiff line change
@@ -73,8 +73,8 @@ spark.executor.extraJavaOptions =
7373
# ---------------------------------------------------------------------------------
7474

7575
codec_packages = htsjdk.variant, htsjdk.tribble, org.broadinstitute.hellbender.utils.codecs
76-
read_filter_packages = "org.broadinstitute.hellbender.engine.filters"
77-
annotation_packages = "org.broadinstitute.hellbender.tools.walkers.annotator"
76+
read_filter_packages = org.broadinstitute.hellbender.engine.filters
77+
annotation_packages = org.broadinstitute.hellbender.tools.walkers.annotator
7878

7979
# ---------------------------------------------------------------------------------
8080
# GATKTool Options:

‎src/test/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKAnnotationPluginDescriptorUnitTest.java

+47-7
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,34 @@
11
package org.broadinstitute.hellbender.cmdline.GATKPlugin;
22

3-
import org.broadinstitute.barclay.argparser.ClassFinder;
43
import htsjdk.variant.variantcontext.*;
54
import org.apache.commons.io.output.NullOutputStream;
6-
import org.broadinstitute.barclay.argparser.CommandLineArgumentParser;
7-
import org.broadinstitute.barclay.argparser.CommandLineException;
8-
import org.broadinstitute.barclay.argparser.CommandLineParser;
5+
import org.broadinstitute.barclay.argparser.*;
96
import org.broadinstitute.hellbender.GATKBaseTest;
7+
import org.broadinstitute.hellbender.cmdline.GATKPlugin.testpluggables.TestAnnotation;
108
import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions;
9+
import org.broadinstitute.hellbender.cmdline.TestProgramGroup;
1110
import org.broadinstitute.hellbender.engine.FeatureContext;
11+
import org.broadinstitute.hellbender.engine.GATKTool;
1212
import org.broadinstitute.hellbender.engine.ReferenceContext;
1313
import org.broadinstitute.hellbender.exceptions.GATKException;
14+
import org.broadinstitute.hellbender.testutils.ArgumentsBuilder;
1415
import org.broadinstitute.hellbender.tools.walkers.annotator.*;
1516
import org.broadinstitute.hellbender.tools.walkers.annotator.allelespecific.AS_RMSMappingQuality;
1617
import org.broadinstitute.hellbender.tools.walkers.annotator.allelespecific.AS_StandardAnnotation;
18+
import org.broadinstitute.hellbender.utils.config.GATKConfig;
1719
import org.broadinstitute.hellbender.utils.genotyper.ReadLikelihoods;
1820
import org.broadinstitute.hellbender.utils.variant.GATKVCFConstants;
1921
import org.mockito.internal.util.collections.Sets;
22+
import org.mockito.internal.util.io.IOUtil;
2023
import org.testng.Assert;
2124
import org.testng.annotations.DataProvider;
2225
import org.testng.annotations.Test;
2326
import org.testng.collections.Lists;
2427

28+
import java.io.File;
29+
import java.io.IOException;
2530
import java.io.PrintStream;
31+
import java.nio.file.Files;
2632
import java.util.*;
2733
import java.util.stream.Collectors;
2834
import java.util.stream.Stream;
@@ -456,7 +462,7 @@ public void testUseAllAnnotations(){
456462
List<Annotation> annots = instantiateAnnotations(clp);
457463

458464
ClassFinder finder = new ClassFinder();
459-
finder.find(GATKAnnotationPluginDescriptor.pluginPackageName, Annotation.class);
465+
finder.find(GATKConfig.DEFAULT_ANNOTATION_PACKAGES, Annotation.class);
460466

461467
Set<Class<?>> classes = finder.getConcreteClasses();
462468
Assert.assertFalse(classes.isEmpty());
@@ -477,7 +483,7 @@ public void testUseAllAnnotationsDisableToolDefaultAnnotaiotns(){
477483
List<Annotation> annots = instantiateAnnotations(clp);
478484

479485
ClassFinder finder = new ClassFinder();
480-
finder.find(GATKAnnotationPluginDescriptor.pluginPackageName, Annotation.class);
486+
finder.find(GATKConfig.DEFAULT_ANNOTATION_PACKAGES, Annotation.class);
481487

482488
Set<Class<?>> classes = finder.getConcreteClasses();
483489
Assert.assertFalse(classes.isEmpty());
@@ -502,7 +508,7 @@ public void testIncludeAllExcludeIndividual(){
502508
Assert.assertFalse(annots.stream().anyMatch(a -> a.getClass()==AS_StandardAnnotation.class));
503509

504510
ClassFinder finder = new ClassFinder();
505-
finder.find(GATKAnnotationPluginDescriptor.pluginPackageName, Annotation.class);
511+
finder.find(GATKConfig.DEFAULT_ANNOTATION_PACKAGES, Annotation.class);
506512

507513
Set<Class<?>> classes = finder.getConcreteClasses();
508514
classes.remove(Coverage.class);
@@ -628,6 +634,7 @@ public List<String> getKeyNames() {
628634
return Collections.singletonList("Test");
629635
}
630636
}
637+
631638
static class testParentAnnotation extends InfoFieldAnnotation implements ParentAnnotationGroup {
632639
boolean dontAnnotate = false;
633640

@@ -651,4 +658,37 @@ public List<String> getKeyNames() {
651658
}
652659
}
653660

661+
@CommandLineProgramProperties(summary="test tool to check annotation loading",
662+
oneLineSummary = "test tool to check annotation loading",
663+
programGroup = TestProgramGroup.class,
664+
omitFromCommandLine = true)
665+
public static class TestAnnotationsTool extends GATKTool {
666+
@Argument(fullName = StandardArgumentDefinitions.OUTPUT_LONG_NAME)
667+
String output;
668+
669+
@Override
670+
public boolean useVariantAnnotations(){
671+
return true;
672+
}
673+
674+
@Override
675+
public void traverse() {
676+
String annotations = makeVariantAnnotations().stream()
677+
.map(a -> a.getClass().getSimpleName())
678+
.collect(Collectors.joining("\n"));
679+
IOUtil.writeText(annotations, new File(output));
680+
}
681+
}
682+
683+
@Test
684+
public void testConfigFileControlsAnnotationPackages() throws IOException {
685+
final File output = createTempFile("annotations", "txt");
686+
final File configFile = new File("src/test/resources/org/broadinstitute/hellbender/cmdline/GATKPlugin/changePluginPackages.properties");
687+
ArgumentsBuilder args = new ArgumentsBuilder();
688+
args.addArgument(StandardArgumentDefinitions.ANNOTATION_LONG_NAME, TestAnnotation.class.getSimpleName())
689+
.addFileArgument(StandardArgumentDefinitions.GATK_CONFIG_FILE_OPTION, configFile)
690+
.addOutput(output);
691+
runToolInNewJVM("TestAnnotationsTool", args);
692+
Assert.assertEquals(Files.readAllLines(output.toPath()), Collections.singletonList(TestAnnotation.class.getSimpleName()));
693+
}
654694
}

‎src/test/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/GATKReadFilterPluginDescriptorTest.java

+42-3
Original file line numberDiff line numberDiff line change
@@ -4,19 +4,26 @@
44
import htsjdk.samtools.SAMFileHeader;
55
import htsjdk.samtools.TextCigarCodec;
66
import org.apache.commons.io.output.NullOutputStream;
7-
import org.broadinstitute.barclay.argparser.CommandLineArgumentParser;
8-
import org.broadinstitute.barclay.argparser.CommandLineException;
9-
import org.broadinstitute.barclay.argparser.CommandLineParser;
7+
import org.broadinstitute.barclay.argparser.*;
108
import org.broadinstitute.hellbender.GATKBaseTest;
9+
import org.broadinstitute.hellbender.cmdline.GATKPlugin.testpluggables.TestReadFilter;
1110
import org.broadinstitute.hellbender.cmdline.ReadFilterArgumentDefinitions;
11+
import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions;
12+
import org.broadinstitute.hellbender.cmdline.TestProgramGroup;
13+
import org.broadinstitute.hellbender.engine.GATKTool;
1214
import org.broadinstitute.hellbender.engine.filters.*;
15+
import org.broadinstitute.hellbender.testutils.ArgumentsBuilder;
1316
import org.broadinstitute.hellbender.utils.read.ArtificialReadUtils;
1417
import org.broadinstitute.hellbender.utils.read.GATKRead;
18+
import org.mockito.internal.util.io.IOUtil;
1519
import org.testng.Assert;
1620
import org.testng.annotations.DataProvider;
1721
import org.testng.annotations.Test;
1822

23+
import java.io.File;
24+
import java.io.IOException;
1925
import java.io.PrintStream;
26+
import java.nio.file.Files;
2027
import java.util.ArrayList;
2128
import java.util.Arrays;
2229
import java.util.Collections;
@@ -786,5 +793,37 @@ private void setupReadNameTest(SetupTest setup) {
786793
private void setupSampleTest(SetupTest setup) {
787794
setup.hdr.getReadGroup(setup.read.getReadGroup()).setSample(setup.argValue);
788795
}
796+
797+
@CommandLineProgramProperties(summary="test tool to check ReadFilter loading",
798+
oneLineSummary = "test tool to check ReadFilter loading",
799+
programGroup = TestProgramGroup.class,
800+
omitFromCommandLine = true)
801+
public static class TestReadFiltersTool extends GATKTool {
802+
@Argument(fullName = StandardArgumentDefinitions.OUTPUT_LONG_NAME)
803+
String output;
804+
805+
@Override
806+
public void traverse() {
807+
String readFilters = getCommandLineParser()
808+
.getPluginDescriptor(GATKReadFilterPluginDescriptor.class)
809+
.getResolvedInstances()
810+
.stream()
811+
.map(a -> a.getClass().getSimpleName())
812+
.collect(Collectors.joining("\n"));
813+
IOUtil.writeText(readFilters, new File(output));
814+
}
815+
}
816+
817+
@Test
818+
public void testConfigFileControlsReadFiltersPackages() throws IOException {
819+
final File output = createTempFile("annotations", "txt");
820+
final File configFile = new File("src/test/resources/org/broadinstitute/hellbender/cmdline/GATKPlugin/changePluginPackages.properties");
821+
ArgumentsBuilder args = new ArgumentsBuilder();
822+
args.addArgument(ReadFilterArgumentDefinitions.READ_FILTER_LONG_NAME, TestReadFilter.class.getSimpleName())
823+
.addFileArgument(StandardArgumentDefinitions.GATK_CONFIG_FILE_OPTION, configFile)
824+
.addOutput(output);
825+
runToolInNewJVM("TestReadFiltersTool", args);
826+
Assert.assertEquals(Files.readAllLines(output.toPath()), Arrays.asList(WellformedReadFilter.class.getSimpleName(), TestReadFilter.class.getSimpleName()));
827+
}
789828

790829
}

‎src/test/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/testpluggables/TestAnnotation.java

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
package org.broadinstitute.hellbender.cmdline.GATKPlugin.testpluggables;
2+
3+
import org.broadinstitute.hellbender.tools.walkers.annotator.Annotation;
4+
5+
public class TestAnnotation implements Annotation {
6+
}

‎src/test/java/org/broadinstitute/hellbender/cmdline/GATKPlugin/testpluggables/TestReadFilter.java

+12
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package org.broadinstitute.hellbender.cmdline.GATKPlugin.testpluggables;
2+
3+
import org.broadinstitute.hellbender.engine.filters.ReadFilter;
4+
import org.broadinstitute.hellbender.utils.read.GATKRead;
5+
6+
public class TestReadFilter extends ReadFilter {
7+
private static final long serialVersionUID = 0L;
8+
@Override
9+
public boolean test(GATKRead read) {
10+
return true;
11+
}
12+
}

‎src/test/java/org/broadinstitute/hellbender/engine/GATKToolUnitTest.java

+19-8
Original file line numberDiff line numberDiff line change
@@ -13,18 +13,19 @@
1313
import htsjdk.variant.vcf.VCFIDHeaderLine;
1414
import org.broadinstitute.barclay.argparser.*;
1515
import org.broadinstitute.hellbender.GATKBaseTest;
16-
import org.broadinstitute.hellbender.cmdline.GATKPlugin.GATKAnnotationPluginDescriptor;
16+
import org.broadinstitute.hellbender.cmdline.GATKPlugin.testpluggables.TestAnnotation;
1717
import org.broadinstitute.hellbender.cmdline.StandardArgumentDefinitions;
1818
import org.broadinstitute.hellbender.cmdline.TestProgramGroup;
1919
import org.broadinstitute.hellbender.exceptions.UserException;
20+
import org.broadinstitute.hellbender.testutils.ArgumentsBuilder;
2021
import org.broadinstitute.hellbender.tools.walkers.annotator.Annotation;
2122
import org.broadinstitute.hellbender.tools.walkers.annotator.ClippingRankSumTest;
2223
import org.broadinstitute.hellbender.tools.walkers.annotator.Coverage;
2324
import org.broadinstitute.hellbender.tools.walkers.annotator.StandardAnnotation;
2425
import org.broadinstitute.hellbender.utils.SimpleInterval;
26+
import org.broadinstitute.hellbender.utils.config.GATKConfig;
2527
import org.broadinstitute.hellbender.utils.read.GATKRead;
2628
import org.broadinstitute.hellbender.utils.reference.ReferenceUtils;
27-
import org.broadinstitute.hellbender.testutils.ArgumentsBuilder;
2829
import org.testng.Assert;
2930
import org.testng.annotations.DataProvider;
3031
import org.testng.annotations.Test;
@@ -631,7 +632,7 @@ public void testGetAllAnnotations() {
631632
Collection<Annotation> annots = tool.makeVariantAnnotations();
632633

633634
ClassFinder finder = new ClassFinder();
634-
finder.find(GATKAnnotationPluginDescriptor.pluginPackageName, Annotation.class);
635+
finder.find(GATKConfig.DEFAULT_ANNOTATION_PACKAGES, Annotation.class);
635636

636637
Set<Class<?>> classes = finder.getConcreteClasses();
637638
Assert.assertFalse(classes.isEmpty());
@@ -641,6 +642,16 @@ public void testGetAllAnnotations() {
641642
}
642643
}
643644

645+
@Test
646+
public void testConfigureAnnotationPackages(){
647+
String[] args = {"--"+StandardArgumentDefinitions.ENABLE_ALL_ANNOTATIONS,
648+
"--"+StandardArgumentDefinitions.GATK_CONFIG_FILE_OPTION, publicTestDir + "org/broadinstitute/hellbender/cmdline/GATKPlugin/changePluginPackages.properties"};
649+
650+
final TestGATKToolWithVariants tool = createTestVariantTool(args);
651+
Collection<Annotation> annots = tool.makeVariantAnnotations();
652+
Assert.assertEquals(annots.iterator().next().getClass(), TestAnnotation.class);
653+
}
654+
644655
@Test
645656
public void testExcludeAnnotation(){
646657
String[] args = {"--"+StandardArgumentDefinitions.ENABLE_ALL_ANNOTATIONS, "-AX", "Coverage"};
@@ -662,7 +673,7 @@ public void testIncludeAnnotationGroups(){
662673

663674
// Asserting that a standard annotation was included but not everything
664675
ClassFinder finder = new ClassFinder();
665-
finder.find(GATKAnnotationPluginDescriptor.pluginPackageName, StandardAnnotation.class);
676+
finder.find(GATKConfig.DEFAULT_ANNOTATION_PACKAGES, StandardAnnotation.class);
666677

667678
Set<Class<?>> classes = finder.getConcreteClasses();
668679
Assert.assertFalse(classes.isEmpty());
@@ -682,7 +693,7 @@ public void testIncludeAnnotation(){
682693

683694
// Asserting coverage was added
684695
Assert.assertTrue(annots.stream().anyMatch(a -> a.getClass()==Coverage.class));
685-
Assert.assertTrue(annots.size() == 1);
696+
Assert.assertEquals(annots.size(), 1);
686697
}
687698

688699
@Test
@@ -694,7 +705,7 @@ public void testMakeDefaultAnnotations() {
694705

695706
// Asserting coverage was added by default
696707
Assert.assertTrue(annots.stream().anyMatch(a -> a.getClass()==Coverage.class));
697-
Assert.assertTrue(annots.size() == 1);
708+
Assert.assertEquals(annots.size(), 1);
698709
}
699710

700711
@Test
@@ -705,7 +716,7 @@ public void testMakeDefaultAnnotationGroups() {
705716
Collection<Annotation> annots = tool.makeVariantAnnotations();
706717

707718
ClassFinder finder = new ClassFinder();
708-
finder.find(GATKAnnotationPluginDescriptor.pluginPackageName, StandardAnnotation.class);
719+
finder.find(GATKConfig.DEFAULT_ANNOTATION_PACKAGES, StandardAnnotation.class);
709720

710721
Set<Class<?>> classes = finder.getConcreteClasses();
711722
Assert.assertFalse(classes.isEmpty());
@@ -752,7 +763,7 @@ public void testHelpWithAllPluginDescriptors() {
752763
new TestGATKToolWithVariants().instanceMain(args);
753764
}
754765

755-
private TestGATKToolWithVariants createTestVariantTool(final String args[]) {
766+
public TestGATKToolWithVariants createTestVariantTool(final String args[]) {
756767
return createTestVariantTool(new TestGATKToolWithVariants(), args);
757768
}
758769

‎src/test/resources/org/broadinstitute/hellbender/cmdline/GATKPlugin/changePluginPackages.properties

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
read_filter_packages = org.broadinstitute.hellbender.cmdline.GATKPlugin.testpluggables
2+
annotation_packages = org.broadinstitute.hellbender.cmdline.GATKPlugin.testpluggables

0 commit comments

Comments
 (0)