diff --git a/build.gradle b/build.gradle index 216423f40..f17c8b146 100644 --- a/build.gradle +++ b/build.gradle @@ -50,11 +50,22 @@ jacoco { final htsjdkVersion = System.getProperty('htsjdk.version', '2.10.1') dependencies { + compile('com.intel.gkl:gkl:0.5.2') { + exclude module: 'htsjdk' + } compile 'com.google.guava:guava:15.0' compile 'com.github.samtools:htsjdk:' + htsjdkVersion //tools dependency for doclet requires sdk devel compile(files(((URLClassLoader) ToolProvider.getSystemToolClassLoader()).getURLs())) testCompile 'org.testng:testng:6.9.10' + testCompile 'org.apache.commons:commons-lang3:3.6' +} + +configurations.all { + resolutionStrategy { + // force the htsjdk version so we don't get a different one transitively + force 'com.github.samtools:htsjdk:' + htsjdkVersion + } } sourceCompatibility = 1.8 diff --git a/src/main/java/picard/cmdline/CommandLineProgram.java b/src/main/java/picard/cmdline/CommandLineProgram.java index 317aa561e..81f2194c8 100644 --- a/src/main/java/picard/cmdline/CommandLineProgram.java +++ b/src/main/java/picard/cmdline/CommandLineProgram.java @@ -23,6 +23,8 @@ */ package picard.cmdline; +import com.intel.gkl.compression.IntelDeflaterFactory; +import com.intel.gkl.compression.IntelInflaterFactory; import htsjdk.samtools.Defaults; import htsjdk.samtools.SAMFileWriterFactory; import htsjdk.samtools.SAMFileWriterImpl; @@ -33,7 +35,7 @@ import htsjdk.samtools.metrics.MetricsFile; import htsjdk.samtools.metrics.StringHeader; import htsjdk.samtools.util.BlockCompressedOutputStream; -import htsjdk.samtools.util.BlockCompressedStreamConstants; +import htsjdk.samtools.util.BlockGunzipper; import htsjdk.samtools.util.IOUtil; import htsjdk.samtools.util.Log; import htsjdk.variant.variantcontext.writer.Options; @@ -86,7 +88,9 @@ public ValidationStringency VALIDATION_STRINGENCY = ValidationStringency.DEFAULT_STRINGENCY; @Option(doc = "Compression level for all compressed files created (e.g. BAM and GELI).", common=true) - public int COMPRESSION_LEVEL = BlockCompressedStreamConstants.DEFAULT_COMPRESSION_LEVEL; + // Reading from a file compressed by the default IntelDeflater at level 1 is ~3x faster than reading from a file + // compressed at level 5 (the htsjdk default), and level 1 files are only slightly larger. + public int COMPRESSION_LEVEL = 1; @Option(doc = "When writing SAM files that need to be sorted, this will specify the number of records stored in RAM before spilling to disk. Increasing this number reduces the number of file handles needed to sort a SAM file, and increases the amount of RAM needed.", optional=true, common=true) public Integer MAX_RECORDS_IN_RAM = SAMFileWriterImpl.getDefaultMaxRecordsInRam(); @@ -102,6 +106,12 @@ @Option(doc="Google Genomics API client_secrets.json file path.", common = true) public String GA4GH_CLIENT_SECRETS="client_secrets.json"; + + @Option(shortName = "use_jdk_deflater", doc = "Use the JDK Deflater instead of the Intel Deflater for writing compressed output", common = true) + public Boolean USE_JDK_DEFLATER = false; + + @Option(shortName = "use_jdk_inflater", doc = "Use the JDK Inflater instead of the Intel Inflater for reading compressed input", common = true) + public Boolean USE_JDK_INFLATER = false; private final String standardUsagePreamble = CommandLineParser.getStandardUsagePreamble(getClass()); @@ -164,18 +174,27 @@ public int instanceMain(final String[] argv) { } SamReaderFactory.setDefaultValidationStringency(VALIDATION_STRINGENCY); BlockCompressedOutputStream.setDefaultCompressionLevel(COMPRESSION_LEVEL); + if (VALIDATION_STRINGENCY != ValidationStringency.STRICT) VariantContextWriterBuilder.setDefaultOption(Options.ALLOW_MISSING_FIELDS_IN_HEADER); if (MAX_RECORDS_IN_RAM != null) { SAMFileWriterImpl.setDefaultMaxRecordsInRam(MAX_RECORDS_IN_RAM); } - if (CREATE_INDEX){ + if (CREATE_INDEX) { SAMFileWriterFactory.setDefaultCreateIndexWhileWriting(true); } SAMFileWriterFactory.setDefaultCreateMd5File(CREATE_MD5_FILE); + if (!USE_JDK_DEFLATER) { + BlockCompressedOutputStream.setDefaultDeflaterFactory(new IntelDeflaterFactory()); + } + + if (!USE_JDK_INFLATER) { + BlockGunzipper.setDefaultInflaterFactory(new IntelInflaterFactory()); + } + for (final File f : TMP_DIR) { // Intentionally not checking the return values, because it may be that the program does not // need a tmp_dir. If this fails, the problem will be discovered downstream. @@ -190,12 +209,18 @@ public int instanceMain(final String[] argv) { // Output a one liner about who/where and what software/os we're running on try { - System.err.println("[" + new Date() + "] Executing as " + - System.getProperty("user.name") + "@" + InetAddress.getLocalHost().getHostName() + - " on " + System.getProperty("os.name") + " " + System.getProperty("os.version") + - " " + System.getProperty("os.arch") + "; " + System.getProperty("java.vm.name") + - " " + System.getProperty("java.runtime.version") + - "; Picard version: " + commandLineParser.getVersion()); + final boolean usingIntelDeflater = (BlockCompressedOutputStream.getDefaultDeflaterFactory() instanceof IntelDeflaterFactory && + ((IntelDeflaterFactory)BlockCompressedOutputStream.getDefaultDeflaterFactory()).usingIntelDeflater()); + final boolean usingIntelInflater = (BlockGunzipper.getDefaultInflaterFactory() instanceof IntelInflaterFactory && + ((IntelInflaterFactory)BlockGunzipper.getDefaultInflaterFactory()).usingIntelInflater()); + final String msg = String.format( + "[%s] Executing as %s@%s on %s %s %s; %s %s; Deflater: %s; Inflater: %s; Picard version: %s", + new Date(), System.getProperty("user.name"), InetAddress.getLocalHost().getHostName(), + System.getProperty("os.name"), System.getProperty("os.version"), System.getProperty("os.arch"), + System.getProperty("java.vm.name"), System.getProperty("java.runtime.version"), + usingIntelDeflater ? "Intel" : "Jdk", usingIntelInflater ? "Intel" : "Jdk", + commandLineParser.getVersion()); + System.err.println(msg); } catch (Exception e) { /* Unpossible! */ } } diff --git a/src/test/java/picard/IntelInflaterDeflaterLoadTest.java b/src/test/java/picard/IntelInflaterDeflaterLoadTest.java new file mode 100644 index 000000000..8875c966c --- /dev/null +++ b/src/test/java/picard/IntelInflaterDeflaterLoadTest.java @@ -0,0 +1,37 @@ +package picard; + +import com.intel.gkl.compression.IntelDeflater; +import com.intel.gkl.compression.IntelInflater; +import org.apache.commons.lang3.SystemUtils; +import org.testng.Assert; +import org.testng.SkipException; +import org.testng.annotations.Test; + +/** + * Test that the Intel Inflater and Deflater can be loaded successfully. + */ +public class IntelInflaterDeflaterLoadTest { + @Test + public void testIntelInflaterIsAvailable() { + checkIntelSupported("IntelInflater"); + Assert.assertTrue(new IntelInflater().load(null), + "Intel shared library was not loaded. This could be due to a configuration error, or your system might not support it."); + } + + @Test + public void testIntelDeflaterIsAvailable() { + checkIntelSupported("IntelDeflater"); + Assert.assertTrue(new IntelDeflater().load(null), + "Intel shared library was not loaded. This could be due to a configuration error, or your system might not support it."); + } + + private void checkIntelSupported(final String componentName) { + if (!SystemUtils.IS_OS_LINUX && !SystemUtils.IS_OS_MAC) { + throw new SkipException(componentName + " is not available on this platform"); + } + + if (SystemUtils.OS_ARCH != null && SystemUtils.OS_ARCH.equals("ppc64le")) { + throw new SkipException(componentName + " is not available for this architecture"); + } + } +} diff --git a/src/test/java/picard/analysis/CollectMultipleMetricsTest.java b/src/test/java/picard/analysis/CollectMultipleMetricsTest.java index d99cf77fa..9406f034a 100644 --- a/src/test/java/picard/analysis/CollectMultipleMetricsTest.java +++ b/src/test/java/picard/analysis/CollectMultipleMetricsTest.java @@ -335,8 +335,10 @@ void setupBuilder() throws IOException { final String flowCellBarcode = "TESTBARCODE"; tempSamFile = File.createTempFile("CollectGcBias", ".bam", TEST_DIR); + final File tempSamIndex = new File(tempSamFile.getAbsolutePath().replace("bam", "bai")); final File tempSamFileUnsorted = File.createTempFile("CollectGcBias", ".bam", TEST_DIR); tempSamFileUnsorted.deleteOnExit(); + tempSamIndex.deleteOnExit(); tempSamFile.deleteOnExit(); BufferedLineReader bufferedLineReader = null; diff --git a/src/test/java/picard/analysis/CollectWgsMetricsTest.java b/src/test/java/picard/analysis/CollectWgsMetricsTest.java index ab5a4304c..6c5259b1d 100644 --- a/src/test/java/picard/analysis/CollectWgsMetricsTest.java +++ b/src/test/java/picard/analysis/CollectWgsMetricsTest.java @@ -136,8 +136,10 @@ void setupBuilder() throws IOException { //Create Sam Files tempSamFile = File.createTempFile("CollectWgsMetrics", ".bam", TEST_DIR); + final File tempSamIndex = new File(tempSamFile.getAbsolutePath().replace("bam", "bai")); final File tempSamFileUnsorted = File.createTempFile("CollectWgsMetrics", ".bam", TEST_DIR); tempSamFileUnsorted.deleteOnExit(); + tempSamIndex.deleteOnExit(); tempSamFile.deleteOnExit(); final File sortedSamIdx = new File(TEST_DIR, tempSamFile.getName() + ".idx"); sortedSamIdx.deleteOnExit(); diff --git a/src/test/java/picard/sam/FilterSamReadsTest.java b/src/test/java/picard/sam/FilterSamReadsTest.java index 35e49b810..3ddf41e3a 100644 --- a/src/test/java/picard/sam/FilterSamReadsTest.java +++ b/src/test/java/picard/sam/FilterSamReadsTest.java @@ -99,7 +99,7 @@ public void testPairedIntervalFilter(final String intervalFilename, final int ex sortedSamIdx.deleteOnExit(); final SAMFileWriter writer = new SAMFileWriterFactory() - .setCreateIndex(true).makeBAMWriter(builder.getHeader(), false, inputSam); + .setCreateIndex(true).makeSAMWriter(builder.getHeader(), false, inputSam); for (final SAMRecord record : builder) { writer.addAlignment(record); @@ -109,6 +109,7 @@ public void testPairedIntervalFilter(final String intervalFilename, final int ex final File intervalFile = new File(intervalFilename); FilterSamReads filterTest = setupProgram(intervalFile, inputSam, FilterSamReads.Filter.includePairedIntervals); + Assert.assertEquals(filterTest.doWork(),0); long count = getReadCount(filterTest);