Updating htsjdk to 2.19.0 (#1297)

* Updating htsjdk from 2.18.2 -> 2.19.0
* Fix SamFormatConverter test
* Replacing deprecated TestUtils method

build.gradle

@@ -64,7 +64,7 @@ def ensureBuildPrerequisites(requiredJavaVersion, buildPrerequisitesMessage) {
 }
 ensureBuildPrerequisites(requiredJavaVersion, buildPrerequisitesMessage)
 
-final htsjdkVersion = System.getProperty('htsjdk.version', '2.18.2')
+final htsjdkVersion = System.getProperty('htsjdk.version', '2.19.0')
 
 // We use a custom shaded build of the NIO library to avoid a regression in the authentication layer.
 // GATK does the same, see https:/broadinstitute/gatk/issues/3591

src/main/java/picard/sam/SamFormatConverter.java

@@ -33,14 +33,13 @@
 import htsjdk.samtools.util.IOUtil;
 import htsjdk.samtools.util.Log;
 import htsjdk.samtools.util.ProgressLogger;
+import org.broadinstitute.barclay.argparser.Argument;
+import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
 import org.broadinstitute.barclay.help.DocumentedFeature;
 import picard.PicardException;
 import picard.cmdline.CommandLineProgram;
-import org.broadinstitute.barclay.argparser.CommandLineProgramProperties;
-import org.broadinstitute.barclay.argparser.Argument;
 import picard.cmdline.StandardOptionDefinitions;
 import picard.cmdline.programgroups.ReadDataManipulationProgramGroup;
-import picard.illumina.parser.ReadData;
 
 import java.io.File;
 
@@ -55,15 +54,12 @@
  oneLineSummary = "Convert a BAM file to a SAM file, or a SAM to a BAM",
  programGroup = ReadDataManipulationProgramGroup.class)
 @DocumentedFeature
-public class
- SamFormatConverter extends CommandLineProgram {
-
- private static final String PROGRAM_VERSION = "1.0";
+public class SamFormatConverter extends CommandLineProgram {
 
  // The following attributes define the command-line arguments
-
  @Argument(doc = "The BAM or SAM file to parse.", shortName = StandardOptionDefinitions.INPUT_SHORT_NAME)
  public File INPUT;
+
  @Argument(doc = "The BAM or SAM output file. ", shortName = StandardOptionDefinitions.OUTPUT_SHORT_NAME)
  public File OUTPUT;
 
@@ -72,12 +68,24 @@ public static void main(final String[] argv) {
  }
 
  protected int doWork() {
- IOUtil.assertFileIsReadable(INPUT);
- IOUtil.assertFileIsWritable(OUTPUT);
- final SamReader reader = SamReaderFactory.makeDefault().referenceSequence(REFERENCE_SEQUENCE).open(INPUT);
- final SAMFileWriter writer = new SAMFileWriterFactory().makeWriter(reader.getFileHeader(), true, OUTPUT, REFERENCE_SEQUENCE);
+ convert(INPUT, OUTPUT, REFERENCE_SEQUENCE, CREATE_INDEX);
+ return 0;
+ }
 
- if (CREATE_INDEX && writer.getFileHeader().getSortOrder() != SAMFileHeader.SortOrder.coordinate) {
+ /**
+ * Convert a file from one of sam/bam/cram format to another based on the extension of output.
+ *
+ * @param input input file in one of sam/bam/cram format
+ * @param output output to write converted file to, the conversion is based on the extension of this filename
+ * @param referenceSequence the reference sequence to use, necessary when reading/writing cram
+ * @param createIndex whether or not an index should be written alongside the output file
+ */
+ public static void convert(final File input, final File output, final File referenceSequence, final Boolean createIndex) {
+ IOUtil.assertFileIsReadable(input);
+ IOUtil.assertFileIsWritable(output);
+ final SamReader reader = SamReaderFactory.makeDefault().referenceSequence(referenceSequence).open(input);
+ final SAMFileWriter writer = new SAMFileWriterFactory().makeWriter(reader.getFileHeader(), true, output, referenceSequence);
+ if (createIndex && writer.getFileHeader().getSortOrder() != SAMFileHeader.SortOrder.coordinate) {
  throw new PicardException("Can't CREATE_INDEX unless sort order is coordinate");
  }
 
@@ -88,6 +96,5 @@ protected int doWork() {
  }
  CloserUtil.close(reader);
  writer.close();
- return 0;
  }
 }

src/main/java/picard/vcf/RenameSampleInVcf.java

@@ -115,7 +115,7 @@ protected int doWork() {
  IOUtil.assertFileIsReadable(INPUT);
  IOUtil.assertFileIsWritable(OUTPUT);
 
- final VCFFileReader in = new VCFFileReader(INPUT);
+ final VCFFileReader in = new VCFFileReader(INPUT, false);
  final VCFHeader header = in.getFileHeader();
 
  if (header.getGenotypeSamples().size() > 1) {

src/test/java/picard/analysis/replicates/CollectIndependentReplicatesMetricTest.java

@@ -60,7 +60,7 @@ public void prepareBams() throws IOException {
 
  @AfterTest
  public void tearDown() {
- TestUtil.recursiveDelete(bamOutDir);
+ IOUtil.recursiveDelete(bamOutDir.toPath());
  }
 
  @DataProvider(name = "simpleTests")

src/test/java/picard/fastq/BamToBfqTest.java

@@ -42,7 +42,7 @@ public void testBamToBfq(final File input, final boolean isPairedRun,
 
  Assert.assertEquals(Files.readAllBytes(output.toPath()), Files.readAllBytes(expectedBFQ.toPath()));
  } finally {
- TestUtil.recursiveDelete(analysisDir);
+ IOUtil.recursiveDelete(analysisDir.toPath());
  }
  }
 }

src/test/java/picard/illumina/IlluminaBasecallsToFastqTest.java

@@ -116,7 +116,7 @@ public void testMultiplexWithIlluminaReadNameHeaders() throws Exception {
  }
 
  } finally {
- TestUtil.recursiveDelete(outputDir);
+ IOUtil.recursiveDelete(outputDir.toPath());
  }
  }
 
@@ -222,7 +222,7 @@ private void runStandardTest(final int lane, final String jobName, final String
  }
  }
  } finally {
- TestUtil.recursiveDelete(outputDir);
+ IOUtil.recursiveDelete(outputDir.toPath());
  }
  }
 

src/test/java/picard/illumina/IlluminaBasecallsToSamTest.java

@@ -260,8 +260,7 @@ private void runStandardTest(final int lane, final String jobName, final String
  IOUtil.assertFilesEqual(outputSam, new File(testDataDir, outputSam.getName()));
  }
  } finally {
- TestUtil.recursiveDelete(outputDir.toFile());
- Files.delete(outputDir);
+ IOUtil.recursiveDelete(outputDir);
  }
  }
 }

src/test/java/picard/sam/SamFileConverterTest.java → src/test/java/picard/sam/SamFormatConverterTest.java

@@ -24,6 +24,8 @@
 
 package picard.sam;
 
+import htsjdk.samtools.Defaults;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 import java.io.File;
@@ -34,63 +36,47 @@
 import static org.testng.Assert.assertEquals;
 import static org.testng.Assert.assertTrue;
 
-public class SamFileConverterTest {
+public class SamFormatConverterTest {
 
- private static final File TEST_DATA_DIR = new File("testdata/picard/sam/SamFileConverterTest");
+ private static final File TEST_DATA_DIR = new File("testdata/picard/sam/SamFormatConverterTest");
  private static final File unmappedSam = new File(TEST_DATA_DIR, "unmapped.sam");
  private static final File unmappedBam = new File(TEST_DATA_DIR, "unmapped.bam");
  private static final File unmappedCram = new File(TEST_DATA_DIR, "unmapped.cram");
 
- @Test
- public void testSAMToBAM() {
- convertFile(unmappedSam, unmappedBam, ".bam");
- }
-
- @Test
- public void testSAMToCRAM() {
- convertFile(unmappedSam, unmappedCram, ".cram");
- }
+ private static final File ESSENTIALLY_EMPTY_REFERENCE_TO_USE_WITH_UNMAPPED_CRAM = new File(TEST_DATA_DIR, "basicallyEmpty.fasta");
 
- @Test
- public void testBAMToCRAM() {
- convertFile(unmappedBam, unmappedCram, ".cram");
- }
-
- @Test
- public void testBAMToSAM() {
- convertFile(unmappedBam, unmappedSam, ".sam");
- }
+ @DataProvider
+ public Object[][] conversionCases() {
+ return new Object[][]{
+  {unmappedSam, unmappedBam, ".bam"},
+ {unmappedSam, unmappedCram, ".cram"},
+  {unmappedBam, unmappedCram, ".cram"},
+  {unmappedBam, unmappedSam, ".sam"},
+  {unmappedCram, unmappedBam, ".bam"},
+  {unmappedCram, unmappedSam, ".sam"},
 
- @Test
- public void testCRAMToBAM() {
- convertFile(unmappedCram, unmappedBam, ".bam");
+ };
  }
 
- @Test
- public void testCRAMToSAM() {
- convertFile(unmappedCram, unmappedSam, ".sam");
+ @Test(dataProvider = "conversionCases")
+ public void testConvert(File input, File expected, String extension) throws IOException {
+ convertFile(input, expected, extension);
  }
 
 
- private void convertFile(final File inputFile, final File fileToCompare, final String extension) {
- final SamFormatConverter samFormatConverter = new SamFormatConverter();
- final List<File> samFiles = new ArrayList<File>();
+ private void convertFile(final File inputFile, final File fileToCompare, final String extension) throws IOException {
+ final List<File> samFiles = new ArrayList<>();
  final ValidateSamFile validateSamFile = new ValidateSamFile();
  final CompareSAMs compareSAMs = new CompareSAMs();
 
- samFormatConverter.INPUT = inputFile;
- try {
- samFormatConverter.OUTPUT = File.createTempFile("SamFileConverterTest." + inputFile.getName(), extension);
- samFormatConverter.OUTPUT.deleteOnExit();
- } catch (final IOException e) {
- e.printStackTrace();
- }
- samFormatConverter.doWork();
+ final File output = File.createTempFile("SamFormatConverterTest." + inputFile.getName(), extension);
+ output.deleteOnExit();
+ SamFormatConverter.convert(inputFile, output, ESSENTIALLY_EMPTY_REFERENCE_TO_USE_WITH_UNMAPPED_CRAM, Defaults.CREATE_INDEX);
 
- validateSamFile.INPUT = samFormatConverter.OUTPUT;
+ validateSamFile.INPUT = output;
  assertEquals(validateSamFile.doWork(), 0);
 
- samFiles.add(samFormatConverter.OUTPUT);
+ samFiles.add(output);
  samFiles.add(fileToCompare);
 
  compareSAMs.samFiles = samFiles;

src/test/java/picard/sam/markduplicates/AbstractMarkDuplicatesCommandLineProgramTester.java

@@ -33,6 +33,7 @@
 import htsjdk.samtools.metrics.MetricsFile;
 import htsjdk.samtools.util.CloseableIterator;
 import htsjdk.samtools.util.FormatUtil;
+import htsjdk.samtools.util.IOUtil;
 import htsjdk.samtools.util.TestUtil;
 import org.testng.Assert;
 import picard.cmdline.CommandLineProgram;
@@ -202,7 +203,7 @@ public MetricsFile<DuplicationMetrics, Double> testMetrics() throws IOException
  }
  return metricsOutput;
  } finally {
- TestUtil.recursiveDelete(getOutputDir());
+ IOUtil.recursiveDelete(getOutputDir().toPath());
  }
  }
 

src/test/java/picard/sam/markduplicates/MarkDuplicatesSetSizeHistogramTester.java

@@ -26,14 +26,17 @@
 
 import htsjdk.samtools.metrics.MetricsFile;
 import htsjdk.samtools.util.Histogram;
-import htsjdk.samtools.util.TestUtil;
+import htsjdk.samtools.util.IOUtil;
 import org.testng.Assert;
 import org.testng.annotations.AfterClass;
 import picard.cmdline.CommandLineProgram;
 import picard.sam.DuplicationMetrics;
 
 import java.io.IOException;
-import java.util.*;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 /**
  * This class is an extension of AbstractMarkDuplicatesCommandLineProgramTester used to test MarkDuplicatesWithMateCigar with SAM files generated on the fly.
@@ -77,7 +80,7 @@ public void test() throws IOException {
 
  @AfterClass
  public void afterTest() {
- TestUtil.recursiveDelete(getOutputDir());
+ IOUtil.recursiveDelete(getOutputDir().toPath());
  }
 
 }

src/test/java/picard/sam/markduplicates/MarkDuplicatesTagRepresentativeReadIndexTester.java

@@ -29,6 +29,7 @@
 import htsjdk.samtools.SamReaderFactory;
 import htsjdk.samtools.metrics.MetricsFile;
 import htsjdk.samtools.util.CloserUtil;
+import htsjdk.samtools.util.IOUtil;
 import htsjdk.samtools.util.TestUtil;
 import org.testng.Assert;
 import picard.PicardException;
@@ -118,7 +119,7 @@ public void test() {
  Assert.assertEquals(observedMetrics.ESTIMATED_LIBRARY_SIZE, expectedMetrics.ESTIMATED_LIBRARY_SIZE, "ESTIMATED_LIBRARY_SIZE does not match expected");
  Assert.assertEquals(observedMetrics.SECONDARY_OR_SUPPLEMENTARY_RDS, expectedMetrics.SECONDARY_OR_SUPPLEMENTARY_RDS, "SECONDARY_OR_SUPPLEMENTARY_RDS does not match expected");
  } finally {
- TestUtil.recursiveDelete(getOutputDir());
+ IOUtil.recursiveDelete(getOutputDir().toPath());
  }
  }
 

src/test/java/picard/sam/markduplicates/MarkDuplicatesTest.java

@@ -139,7 +139,7 @@ public void pgRecordChainingTest(final boolean suppressPg,
  }
 
  } finally {
- TestUtil.recursiveDelete(outputDir);
+ IOUtil.recursiveDelete(outputDir.toPath());
  }
  }
 
@@ -195,7 +195,7 @@ public void testOpticalDuplicateDetection(final File sam, final long expectedNum
  markDuplicates.PROGRAM_RECORD_ID = null;
  Assert.assertEquals(markDuplicates.doWork(), 0);
  Assert.assertEquals(markDuplicates.numOpticalDuplicates(), expectedNumOpticalDuplicates);
- TestUtil.recursiveDelete(outputDir);
+ IOUtil.recursiveDelete(outputDir.toPath());
 
  }
 

src/test/java/picard/sam/testers/CleanSamTester.java

@@ -9,6 +9,7 @@
 import htsjdk.samtools.SamReaderFactory;
 import htsjdk.samtools.ValidationStringency;
 import htsjdk.samtools.util.CloserUtil;
+import htsjdk.samtools.util.IOUtil;
 import htsjdk.samtools.util.TestUtil;
 import org.testng.Assert;
 import picard.sam.CleanSam;
@@ -58,7 +59,7 @@ protected void test() {
 
  Assert.assertTrue(validated, "ValidateSamFile failed");
  } finally {
- TestUtil.recursiveDelete(getOutputDir());
+ IOUtil.recursiveDelete(getOutputDir().toPath());
  }
  }
 }

src/test/java/picard/util/IntervalListToolsTest.java

@@ -262,7 +262,7 @@ public static Iterator<Object[]> testScatterTestcases() {
 
  @AfterTest
  void deleteTempDirs() {
- dirsToDelete.forEach(TestUtil::recursiveDelete);
+ dirsToDelete.forEach(file -> IOUtil.recursiveDelete(file.toPath()));
  }
 
  @Test(dataProvider = "testScatterTestcases")

src/test/java/picard/vcf/MendelianViolations/FindMendelianViolationsTest.java

@@ -128,7 +128,7 @@ public void testFindMedelianViolations() throws IOException {
  Assert.assertEquals(grepMv(violationsFAKE, Haploid_Other.name()), mv.NUM_HAPLOID_OTHER);
  Assert.assertEquals(grepMv(violationsFAKE, Other.name()), mv.NUM_OTHER);
 
- TestUtil.recursiveDelete(directoryForViolations);
+ IOUtil.recursiveDelete(directoryForViolations.toPath());
  }
 
  /** returns the number of lines in the file that contain a regular expression (decorated with "MV=" and

testdata/picard/sam/SamFormatConverterTest/basicallyEmpty.fasta

@@ -0,0 +1,2 @@
+>chrNull
+NNNNNNN