Merge 26.7 to develop

SequenceAnalysis/resources/schemas/dbscripts/postgresql/sequenceanalysis-12.333-12.334.sql

@@ -0,0 +1 @@
+ALTER TABLE sequenceanalysis.barcodes DROP CONSTRAINT IF EXISTS UNIQUE_barcodes;

SequenceAnalysis/resources/schemas/dbscripts/sqlserver/sequenceanalysis-12.333-12.334.sql

@@ -0,0 +1 @@
+ALTER TABLE sequenceanalysis.barcodes DROP CONSTRAINT IF EXISTS UNIQUE_barcodes;

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisController.java

@@ -208,6 +208,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.zip.GZIPInputStream;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
@@ -717,7 +718,7 @@ else if (SequenceAnalysisSchema.TABLE_OUTPUTFILES.equals(_table.getName()))
                                 msg.append("Folder: ").unsafeAppend("<a href='" + url.toString() + "' target='_blank'>" + h(target.getPath()) + "</a><br><input type='hidden' name='jobIds' value='" + h(sf.getRowId()) + "'/>");
                             }
                         }
-                        msg.append("<br>");
+                        msg.unsafeAppend("<br>");
                     }
                 }
             }
@@ -1914,7 +1915,7 @@ else if (!d.getFile().exists())
                     {
                         throw new PipelineValidationException("Missing file for data: " + o.get("dataId"));
                     }
-                    else if (d.getContainer().hasPermission(u, ReadPermission.class))
+                    else if (!d.getContainer().hasPermission(u, ReadPermission.class))
                     {
                         throw new UnauthorizedException("You do not have permission to read data: " + o.get("dataId"));
                     }
@@ -5161,14 +5162,13 @@ public ApiResponse execute(ArchiveReadsetsForm form, BindException errors) throw
 
                 if (!toUpdate.isEmpty())
                 {
-                    List<Map<String, Object>> keys = new ArrayList<>();
-                    toUpdate.forEach(row -> {
-                        keys.add(new CaseInsensitiveHashMap<>(Map.of("rowid", row.get("rowid"))));
-                    });
+                    // Remove duplicates:
+                    List<Map<String, Object>> uniqueToUpdate = toUpdate.stream().distinct().toList();
+                    List<Map<String, Object>> keys = uniqueToUpdate.stream().map(row -> new CaseInsensitiveHashMap<>(Map.of("rowid", row.get("rowid")))).collect(Collectors.toList());
 
                     try
                     {
-                        readData.getUpdateService().updateRows(getUser(), getContainer(), toUpdate, keys, null, null);
+                        readData.getUpdateService().updateRows(getUser(), getContainer(), uniqueToUpdate, keys, null, null);
                     }
                     catch (Exception e)
                     {

SequenceAnalysis/src/org/labkey/sequenceanalysis/SequenceAnalysisModule.java

@@ -213,7 +213,7 @@ public String getName()
     @Override
     public Double getSchemaVersion()
     {
-        return 12.333;
+        return 12.334;
     }
 
     @Override

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/CreateReferenceLibraryTask.java

@@ -68,6 +68,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.Future;
 
 import static org.labkey.api.util.IntegerUtils.asInteger;
 
@@ -467,7 +468,7 @@ public RecordedActionSet run() throws PipelineJobException
                     {
                         getJob().getLogger().info("running genome trigger: " + t.getName());
                         final int libraryId = rowId;
-                        jr.execute(new Job()
+                        Future<?> future = jr.execute(new Job()
                         {
                             @Override
                             public void run()
@@ -481,11 +482,12 @@ public void run()
                                     t.onRecreate(getJob().getContainer(), getJob().getUser(), getJob().getLogger(), libraryId);
                                 }
                             }
-                        });
+                        }, 0);
+
+                        // Wait for this job:
+                        future.get();
                     }
                 }
-
-                jr.waitForCompletion();
             }
         }
         catch (Exception e)

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/ImportGenomeTrackTask.java

@@ -81,6 +81,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.Future;
 
 import static org.labkey.api.util.IntegerUtils.asInteger;
 
@@ -163,18 +164,19 @@ public boolean isJobComplete(PipelineJob job)
                     if (t.isAvailable(genomeContainer))
                     {
                         getJob().getLogger().info("running genome trigger: " + t.getName());
-                        jr.execute(new Job()
+                        Future<?> future = jr.execute(new Job()
                         {
                             @Override
                             public void run()
                             {
                                 t.onTrackAdd(genomeContainer, getJob().getUser(), getJob().getLogger(), libraryId, trackId);
                             }
-                        });
+                        }, 0);
+
+                        // Wait for the job:
+                        future.get();
                     }
                 }
-
-                jr.waitForCompletion();
             }
 
         }

SequenceAnalysis/src/org/labkey/sequenceanalysis/pipeline/SequenceAlignmentTask.java

@@ -71,6 +71,7 @@
 import org.labkey.sequenceanalysis.ReadDataImpl;
 import org.labkey.sequenceanalysis.SequenceReadsetImpl;
 import org.labkey.sequenceanalysis.run.RestoreSraDataHandler;
+import org.labkey.sequenceanalysis.run.SamtoolsMerger;
 import org.labkey.sequenceanalysis.run.bampostprocessing.SortSamStep;
 import org.labkey.sequenceanalysis.run.preprocessing.TrimmomaticWrapper;
 import org.labkey.sequenceanalysis.run.util.AddOrReplaceReadGroupsWrapper;
@@ -1306,7 +1307,6 @@ private File doAlignThenMerge(ReferenceGenome referenceGenome, Readset rs, Map<R
             RecordedAction mergeAction = new RecordedAction(MERGE_ALIGNMENT_ACTIONNAME);
             Date start = new Date();
             mergeAction.setStartTime(start);
-            MergeSamFilesWrapper mergeSamFilesWrapper = new MergeSamFilesWrapper(getJob().getLogger());
             List<File> bams = new ArrayList<>();
             for (File o : alignOutputs)
             {
@@ -1316,20 +1316,45 @@ private File doAlignThenMerge(ReferenceGenome referenceGenome, Readset rs, Map<R
                 getHelper().getFileManager().addIntermediateFile(SequenceAnalysisService.get().getExpectedBamOrCramIndex(o));
             }
 
-            bam = new File(alignOutputs.get(0).getParent(), FileUtil.getBaseName(alignOutputs.get(0).getName()) + ".merged.bam");
+            bam = new File(alignOutputs.getFirst().getParent(), FileUtil.getBaseName(alignOutputs.getFirst().getName()) + ".merged.bam");
             getHelper().getFileManager().addOutput(mergeAction, "Merged BAM", bam);
+            Set<SAMFileHeader.SortOrder> sortOrders = alignOutputs.stream().map(x -> {
+                try
+                {
+                    return SequenceUtil.getBamSortOrder(x);
+                }
+                catch (IOException e)
+                {
+                    throw new RuntimeException(e);
+                }
+            }).collect(Collectors.toSet());
+
             //NOTE: merged BAMs will be deleted as intermediate files, and if we delete too early this breaks job resume
-            mergeSamFilesWrapper.execute(bams, bam, false);
-            getHelper().getFileManager().addCommandsToAction(mergeSamFilesWrapper.getCommandsExecuted(), mergeAction);
+            String toolName;
+            if (sortOrders.size() > 1 || sortOrders.iterator().next() != SAMFileHeader.SortOrder.coordinate)
+            {
+                toolName = "MergeSamFiles";
+                MergeSamFilesWrapper merger = new MergeSamFilesWrapper(getPipelineJob().getLogger());
+                merger.execute(bams, bam, false);
+                getHelper().getFileManager().addCommandsToAction(merger.getCommandsExecuted(), mergeAction);
+            }
+            else
+            {
+                // This will be faster, but requires sorted input:
+                toolName = "Samtools merge";
+                SamtoolsMerger merger = new SamtoolsMerger(getPipelineJob().getLogger());
+                merger.mergeBams(bams, bam);
+                getHelper().getFileManager().addCommandsToAction(merger.getCommandsExecuted(), mergeAction);
+            }
 
             Date end = new Date();
             mergeAction.setEndTime(end);
-            getJob().getLogger().info("MergeSamFiles Duration: " + DurationFormatUtils.formatDurationWords(end.getTime() - start.getTime(), true, true));
+            getJob().getLogger().info(toolName + " Duration: " + DurationFormatUtils.formatDurationWords(end.getTime() - start.getTime(), true, true));
             alignActions.add(mergeAction);
         }
         else
         {
-            bam = alignOutputs.get(0);
+            bam = alignOutputs.getFirst();
         }
 
         return bam;

SequenceAnalysis/src/org/labkey/sequenceanalysis/run/SamtoolsMerger.java

@@ -0,0 +1,51 @@
+package org.labkey.sequenceanalysis.run;
+
+import org.apache.logging.log4j.Logger;
+import org.labkey.api.pipeline.PipelineJobException;
+import org.labkey.api.sequenceanalysis.pipeline.SamtoolsRunner;
+import org.labkey.api.sequenceanalysis.pipeline.SequencePipelineService;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+
+public class SamtoolsMerger extends SamtoolsRunner
+{
+    private static final String COMMAND = "merge";
+
+    public SamtoolsMerger(Logger log)
+    {
+        super(log);
+    }
+
+    public File mergeBams(List<File> inputBams, File outputFile) throws PipelineJobException
+    {
+        getLogger().info("Merging SAM/BAM(s):");
+
+        List<String> params = new ArrayList<>();
+        params.add(getSamtoolsPath().getPath());
+        params.add(COMMAND);
+
+        Integer threads = SequencePipelineService.get().getMaxThreads(getLogger());
+        if (threads != null)
+        {
+            params.add("--threads");
+            params.add(String.valueOf(threads));
+        }
+
+        params.add("-o");
+        params.add(outputFile.getPath());
+
+        inputBams.forEach(f -> params.add(f.getPath()));
+
+        execute(params);
+
+        File idx = SequencePipelineService.get().ensureBamIndex(outputFile, getLogger(), false);
+        if (!idx.exists())
+        {
+            throw new PipelineJobException("Unable to find BAM index: " + idx.getPath());
+        }
+
+        return outputFile;
+    }
+}