OpenMS  2.8.0
OpenSwathBase.h
Go to the documentation of this file.
1 // --------------------------------------------------------------------------
2 // OpenMS -- Open-Source Mass Spectrometry
3 // --------------------------------------------------------------------------
4 // Copyright The OpenMS Team -- Eberhard Karls University Tuebingen,
5 // ETH Zurich, and Freie Universitaet Berlin 2002-2021.
6 //
7 // This software is released under a three-clause BSD license:
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of any author or any participating institution
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 // For a full list of authors, refer to the file AUTHORS.
17 // --------------------------------------------------------------------------
18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 // ARE DISCLAIMED. IN NO EVENT SHALL ANY OF THE AUTHORS OR THE CONTRIBUTING
22 // INSTITUTIONS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
23 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
25 // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
26 // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
27 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
28 // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 //
30 // --------------------------------------------------------------------------
31 // $Maintainer: Hannes Roest$
32 // $Authors: Hannes Roest$
33 // --------------------------------------------------------------------------
34 
35 #pragma once
36 
37 // Consumers
40 
41 // Files
45 #include <OpenMS/FORMAT/MzMLFile.h>
54 
55 // Kernel and implementations
61 
62 // Helpers
66 
67 // Algorithms
73 
75 
76 #include <cassert>
77 #include <limits>
78 
80 
81 namespace OpenMS
82 {
83 
85  public TOPPBase
86 {
87 
88 public:
89 
90  TOPPOpenSwathBase(String name, String description, bool official = true) :
91  TOPPBase(name, description, official)
92  {
93  }
94 
95 private:
96 
97  void loadSwathFiles_(const StringList& file_list,
98  const bool split_file,
99  const String& tmp,
100  const String& readoptions,
101  boost::shared_ptr<ExperimentalSettings > & exp_meta,
102  std::vector< OpenSwath::SwathMap > & swath_maps,
103  Interfaces::IMSDataConsumer* plugin_consumer)
104  {
105  SwathFile swath_file;
106  swath_file.setLogType(log_type_);
107 
108  if (split_file || file_list.size() > 1)
109  {
110  // TODO cannot use data reduction here any more ...
111  swath_maps = swath_file.loadSplit(file_list, tmp, exp_meta, readoptions);
112  }
113  else
114  {
115  FileTypes::Type in_file_type = FileHandler::getTypeByFileName(file_list[0]);
116  if (in_file_type == FileTypes::MZML)
117  {
118  swath_maps = swath_file.loadMzML(file_list[0], tmp, exp_meta, readoptions, plugin_consumer);
119  }
120  else if (in_file_type == FileTypes::MZXML)
121  {
122  swath_maps = swath_file.loadMzXML(file_list[0], tmp, exp_meta, readoptions);
123  }
124  else if (in_file_type == FileTypes::SQMASS)
125  {
126  swath_maps = swath_file.loadSqMass(file_list[0], exp_meta);
127  }
128  else
129  {
130  throw Exception::IllegalArgument(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION,
131  "Input file needs to have ending mzML or mzXML");
132  }
133  }
134  }
135 
136 protected:
137 
162  bool loadSwathFiles(const StringList& file_list,
163  boost::shared_ptr<ExperimentalSettings >& exp_meta,
164  std::vector< OpenSwath::SwathMap >& swath_maps,
165  const bool split_file,
166  const String& tmp,
167  const String& readoptions,
168  const String& swath_windows_file,
169  const double min_upper_edge_dist,
170  const bool force,
171  const bool sort_swath_maps,
172  const bool sonar,
173  const bool prm,
174  Interfaces::IMSDataConsumer* plugin_consumer = nullptr)
175  {
176  // (i) Load files
177  loadSwathFiles_(file_list, split_file, tmp, readoptions, exp_meta, swath_maps, plugin_consumer);
178 
179  // (ii) Allow the user to specify the SWATH windows
180  if (!swath_windows_file.empty())
181  {
182  SwathWindowLoader::annotateSwathMapsFromFile(swath_windows_file, swath_maps, sort_swath_maps, force);
183  }
184 
185  for (Size i = 0; i < swath_maps.size(); i++)
186  {
187  OPENMS_LOG_DEBUG << "Found swath map " << i
188  << " with lower " << swath_maps[i].lower
189  << " and upper " << swath_maps[i].upper
190  << " and " << swath_maps[i].sptr->getNrSpectra()
191  << " spectra." << std::endl;
192  }
193 
194  // (iii) Sanity check: there should be no overlap between the windows:
195  std::vector<std::pair<double, double>> sw_windows;
196  for (Size i = 0; i < swath_maps.size(); i++)
197  {
198  if (!swath_maps[i].ms1)
199  {
200  sw_windows.push_back(std::make_pair(swath_maps[i].lower, swath_maps[i].upper));
201  }
202  }
203  // sort by lower bound (first entry in pair)
204  std::sort(sw_windows.begin(), sw_windows.end());
205 
206  for (Size i = 1; i < sw_windows.size(); i++)
207  {
208  double lower_map_end = sw_windows[i-1].second - min_upper_edge_dist;
209  double upper_map_start = sw_windows[i].first;
210  OPENMS_LOG_DEBUG << "Extraction will go up to " << lower_map_end << " and continue at " << upper_map_start << std::endl;
211 
212  if (prm) {continue;} // skip next step as expect them to overlap and have gaps...
213 
214  if (upper_map_start - lower_map_end > 0.01)
215  {
216  OPENMS_LOG_WARN << "Extraction will have a gap between " << lower_map_end << " and " << upper_map_start << std::endl;
217  if (!force)
218  {
219  OPENMS_LOG_ERROR << "Extraction windows have a gap. Will abort (override with -force)" << std::endl;
220  return false;
221  }
222  }
223 
224  if (sonar) {continue;} // skip next step as expect them to overlap ...
225 
226  if (lower_map_end - upper_map_start > 0.01)
227  {
228  OPENMS_LOG_WARN << "Extraction will overlap between " << lower_map_end << " and " << upper_map_start << "!\n"
229  << "This will lead to multiple extraction of the transitions in the overlapping region "
230  << "which will lead to duplicated output. It is very unlikely that you want this." << "\n"
231  << "Please fix this by providing an appropriate extraction file with -swath_windows_file" << std::endl;
232  if (!force)
233  {
234  OPENMS_LOG_ERROR << "Extraction windows overlap. Will abort (override with -force)" << std::endl;
235  return false;
236  }
237  }
238  }
239  return true;
240  }
241 
255  void prepareChromOutput(Interfaces::IMSDataConsumer ** chromatogramConsumer,
256  const boost::shared_ptr<ExperimentalSettings>& exp_meta,
257  const OpenSwath::LightTargetedExperiment& transition_exp,
258  const String& out_chrom,
259  const UInt64 run_id)
260  {
261  if (!out_chrom.empty())
262  {
263  String tmp = out_chrom;
264  if (tmp.toLower().hasSuffix(".sqmass"))
265  {
266  bool full_meta = false; // can lead to very large files in memory
267  bool lossy_compression = true;
268  *chromatogramConsumer = new MSDataSqlConsumer(out_chrom, run_id, 500, full_meta, lossy_compression);
269  }
270  else
271  {
272  PlainMSDataWritingConsumer * chromConsumer = new PlainMSDataWritingConsumer(out_chrom);
273  int expected_chromatograms = transition_exp.transitions.size();
274  chromConsumer->setExpectedSize(0, expected_chromatograms);
275  chromConsumer->setExperimentalSettings(*exp_meta);
276  chromConsumer->getOptions().setWriteIndex(true); // ensure that we write the index
278 
279  // prepare data structures for lossy compression
281  MSNumpressCoder::NumpressConfig npconfig_int;
282  npconfig_mz.estimate_fixed_point = true; // critical
283  npconfig_int.estimate_fixed_point = true; // critical
284  npconfig_mz.numpressErrorTolerance = -1.0; // skip check, faster
285  npconfig_int.numpressErrorTolerance = -1.0; // skip check, faster
286  npconfig_mz.setCompression("linear");
287  npconfig_int.setCompression("slof");
288  npconfig_mz.linear_fp_mass_acc = 0.05; // set the desired RT accuracy in seconds
289 
290  chromConsumer->getOptions().setNumpressConfigurationMassTime(npconfig_mz);
291  chromConsumer->getOptions().setNumpressConfigurationIntensity(npconfig_int);
292  chromConsumer->getOptions().setCompression(true);
293 
294  *chromatogramConsumer = chromConsumer;
295  }
296  }
297  else
298  {
299  *chromatogramConsumer = new NoopMSDataWritingConsumer("");
300  }
301  }
302 
312  const String& tr_file,
313  const Param& tsv_reader_param)
314  {
315  OpenSwath::LightTargetedExperiment transition_exp;
316  ProgressLogger progresslogger;
317  progresslogger.setLogType(log_type_);
318  if (tr_type == FileTypes::TRAML)
319  {
320  progresslogger.startProgress(0, 1, "Load TraML file");
321  TargetedExperiment targeted_exp;
322  TraMLFile().load(tr_file, targeted_exp);
323  OpenSwathDataAccessHelper::convertTargetedExp(targeted_exp, transition_exp);
324  progresslogger.endProgress();
325  }
326  else if (tr_type == FileTypes::PQP)
327  {
328  progresslogger.startProgress(0, 1, "Load PQP file");
329  TransitionPQPFile().convertPQPToTargetedExperiment(tr_file.c_str(), transition_exp);
330  progresslogger.endProgress();
331  }
332  else if (tr_type == FileTypes::TSV)
333  {
334  progresslogger.startProgress(0, 1, "Load TSV file");
335  TransitionTSVFile tsv_reader;
336  tsv_reader.setParameters(tsv_reader_param);
337  tsv_reader.convertTSVToTargetedExperiment(tr_file.c_str(), tr_type, transition_exp);
338  progresslogger.endProgress();
339  }
340  else
341  {
342  OPENMS_LOG_ERROR << "Provide valid TraML, TSV or PQP transition file." << std::endl;
343  throw Exception::IllegalArgument(__FILE__, __LINE__, OPENMS_PRETTY_FUNCTION, "Need to provide valid input file.");
344  }
345  return transition_exp;
346  }
347 
381  String irt_tr_file,
382  std::vector< OpenSwath::SwathMap > & swath_maps,
383  double min_rsq,
384  double min_coverage,
385  const Param& feature_finder_param,
386  const ChromExtractParams& cp_irt,
387  const Param& irt_detection_param,
388  const Param& calibration_param,
389  Size debug_level,
390  bool sonar,
391  bool load_into_memory,
392  const String& irt_trafo_out,
393  const String& irt_mzml_out)
394  {
395  TransformationDescription trafo_rtnorm;
396 
397  if (!trafo_in.empty())
398  {
399  // get read RT normalization file
400  TransformationXMLFile trafoxml;
401  trafoxml.load(trafo_in, trafo_rtnorm, false);
402  Param model_params = getParam_().copy("model:", true);
403  model_params.setValue("symmetric_regression", "false");
404  model_params.setValue("span", irt_detection_param.getValue("lowess:span"));
405  model_params.setValue("num_nodes", irt_detection_param.getValue("b_spline:num_nodes"));
406  String model_type = irt_detection_param.getValue("alignmentMethod").toString();
407  trafo_rtnorm.fitModel(model_type, model_params);
408  }
409  else if (!irt_tr_file.empty())
410  {
411  // Loading iRT file
412  std::cout << "Will load iRT transitions and try to find iRT peptides" << std::endl;
413  TraMLFile traml;
414  FileTypes::Type tr_type = FileHandler::getType(irt_tr_file);
415  Param tsv_reader_param = TransitionTSVFile().getDefaults();
416  OpenSwath::LightTargetedExperiment irt_transitions = loadTransitionList(tr_type, irt_tr_file, tsv_reader_param);
417 
418  // perform extraction
420  wf.setLogType(log_type_);
421  TransformationDescription im_trafo;
422  trafo_rtnorm = wf.performRTNormalization(irt_transitions, swath_maps, im_trafo,
423  min_rsq, min_coverage,
424  feature_finder_param,
425  cp_irt, irt_detection_param,
426  calibration_param, irt_mzml_out, debug_level, sonar,
427  load_into_memory);
428 
429  if (!irt_trafo_out.empty())
430  {
431  TransformationXMLFile().store(irt_trafo_out, trafo_rtnorm);
432  }
433  }
434  return trafo_rtnorm;
435  }
436 
437 
438 };
439 
440 }
441 
443 
444 
#define OPENMS_LOG_DEBUG
Macro for general debugging information.
Definition: LogStream.h:470
#define OPENMS_LOG_WARN
Macro if a warning, a piece of information which should be read by the user, should be logged.
Definition: LogStream.h:460
#define OPENMS_LOG_ERROR
Macro to be used if non-fatal error are reported (processing continues)
Definition: LogStream.h:455
@ SMOOTHING
Smoothing of the signal to reduce noise.
Definition: DataProcessing.h:63
void setParameters(const Param &param)
Sets the parameters.
const Param & getDefaults() const
Non-mutable access to the default parameters.
A method or algorithm argument contains illegal values.
Definition: Exception.h:650
static FileTypes::Type getType(const String &filename)
Tries to determine the file type (by name or content)
static FileTypes::Type getTypeByFileName(const String &filename)
Determines the file type from a file name.
The interface of a consumer of spectra and chromatograms.
Definition: IMSDataConsumer.h:70
PeakFileOptions & getOptions()
Get the peak file options.
A data consumer that inserts MS data into a SQLite database.
Definition: MSDataSqlConsumer.h:62
void setExpectedSize(Size expectedSpectra, Size expectedChromatograms) override
Set expected size of spectra and chromatograms to be written.
virtual void addDataProcessing(DataProcessing d)
Optionally add a data processing method to each chromatogram and spectrum.
void setExperimentalSettings(const ExperimentalSettings &exp) override
Set experimental settings for the whole file.
Consumer class that perform no operation.
Definition: MSDataWritingConsumer.h:260
Execute all steps for retention time and m/z calibration of SWATH-MS data.
Definition: OpenSwathWorkflow.h:237
TransformationDescription performRTNormalization(const OpenSwath::LightTargetedExperiment &irt_transitions, std::vector< OpenSwath::SwathMap > &swath_maps, TransformationDescription &im_trafo, double min_rsq, double min_coverage, const Param &feature_finder_param, const ChromExtractParams &cp_irt, const Param &irt_detection_param, const Param &calibration_param, const String &irt_mzml_out, Size debug_level, bool sonar=false, bool load_into_memory=false)
Perform RT and m/z correction of the input data using RT-normalization peptides.
static void convertTargetedExp(const OpenMS::TargetedExperiment &transition_exp_, OpenSwath::LightTargetedExperiment &transition_exp)
convert from the OpenMS TargetedExperiment to the LightTargetedExperiment
std::string toString(bool full_precision=true) const
Convert ParamValue to string.
Management and storage of parameters / INI files.
Definition: Param.h:70
Param copy(const std::string &prefix, bool remove_prefix=false) const
Returns a new Param object containing all entries that start with prefix.
const ParamValue & getValue(const std::string &key) const
Returns a value of a parameter.
void setValue(const std::string &key, const ParamValue &value, const std::string &description="", const std::vector< std::string > &tags=std::vector< std::string >())
Sets a value.
void setNumpressConfigurationIntensity(MSNumpressCoder::NumpressConfig config)
Get numpress configuration options for intensity dimension.
void setCompression(bool compress)
void setWriteIndex(bool write_index)
Whether to write an index at the end of the file (e.g. indexedmzML file format)
void setNumpressConfigurationMassTime(MSNumpressCoder::NumpressConfig config)
Get numpress configuration options for m/z or rt dimension.
Consumer class that writes MS data to disk using the mzML format.
Definition: MSDataWritingConsumer.h:242
Base class for all classes that want to report their progress.
Definition: ProgressLogger.h:53
void setLogType(LogType type) const
Sets the progress log that should be used. The default type is NONE!
void startProgress(SignedSize begin, SignedSize end, const String &label) const
Initializes the progress display.
void endProgress() const
Ends the progress display.
A more convenient string class.
Definition: String.h:60
String & toLower()
Converts the string to lowercase.
bool hasSuffix(const String &string) const
true if String ends with string, false otherwise
File adapter for Swath files.
Definition: SwathFile.h:69
std::vector< OpenSwath::SwathMap > loadMzML(const String &file, const String &tmp, boost::shared_ptr< ExperimentalSettings > &exp_meta, const String &readoptions="normal", Interfaces::IMSDataConsumer *plugin_consumer=nullptr)
Loads a Swath run from a single mzML file.
std::vector< OpenSwath::SwathMap > loadSplit(StringList file_list, String tmp, boost::shared_ptr< ExperimentalSettings > &exp_meta, String readoptions="normal")
Loads a Swath run from a list of split mzML files.
std::vector< OpenSwath::SwathMap > loadSqMass(String file, boost::shared_ptr< ExperimentalSettings > &)
Loads a Swath run from a single sqMass file.
std::vector< OpenSwath::SwathMap > loadMzXML(String file, String tmp, boost::shared_ptr< ExperimentalSettings > &exp_meta, String readoptions="normal")
Loads a Swath run from a single mzXML file.
static void annotateSwathMapsFromFile(const std::string &filename, std::vector< OpenSwath::SwathMap > &swath_maps, bool do_sort, bool force)
Annotate a Swath map using a Swath window file specifying the individual windows.
Base class for TOPP applications.
Definition: TOPPBase.h:146
Param const & getParam_() const
Return all parameters relevant to this TOPP tool.
ProgressLogger::LogType log_type_
Type of progress logging.
Definition: TOPPBase.h:932
DataProcessing getProcessingInfo_(DataProcessing::ProcessingAction action) const
Returns the data processing information.
Definition: OpenSwathBase.h:86
bool loadSwathFiles(const StringList &file_list, boost::shared_ptr< ExperimentalSettings > &exp_meta, std::vector< OpenSwath::SwathMap > &swath_maps, const bool split_file, const String &tmp, const String &readoptions, const String &swath_windows_file, const double min_upper_edge_dist, const bool force, const bool sort_swath_maps, const bool sonar, const bool prm, Interfaces::IMSDataConsumer *plugin_consumer=nullptr)
Load the DIA files into internal data structures.
Definition: OpenSwathBase.h:162
void prepareChromOutput(Interfaces::IMSDataConsumer **chromatogramConsumer, const boost::shared_ptr< ExperimentalSettings > &exp_meta, const OpenSwath::LightTargetedExperiment &transition_exp, const String &out_chrom, const UInt64 run_id)
Prepare chromatogram output.
Definition: OpenSwathBase.h:255
TOPPOpenSwathBase(String name, String description, bool official=true)
Definition: OpenSwathBase.h:90
OpenSwath::LightTargetedExperiment loadTransitionList(const FileTypes::Type &tr_type, const String &tr_file, const Param &tsv_reader_param)
Loads transition list from TraML / TSV or PQP.
Definition: OpenSwathBase.h:311
TransformationDescription performCalibration(String trafo_in, String irt_tr_file, std::vector< OpenSwath::SwathMap > &swath_maps, double min_rsq, double min_coverage, const Param &feature_finder_param, const ChromExtractParams &cp_irt, const Param &irt_detection_param, const Param &calibration_param, Size debug_level, bool sonar, bool load_into_memory, const String &irt_trafo_out, const String &irt_mzml_out)
Perform retention time and m/z calibration.
Definition: OpenSwathBase.h:380
void loadSwathFiles_(const StringList &file_list, const bool split_file, const String &tmp, const String &readoptions, boost::shared_ptr< ExperimentalSettings > &exp_meta, std::vector< OpenSwath::SwathMap > &swath_maps, Interfaces::IMSDataConsumer *plugin_consumer)
Definition: OpenSwathBase.h:97
A description of a targeted experiment containing precursor and production ions.
Definition: TargetedExperiment.h:65
File adapter for HUPO PSI TraML files.
Definition: TraMLFile.h:66
void load(const String &filename, TargetedExperiment &id)
Loads a map from a TraML file.
Generic description of a coordinate transformation.
Definition: TransformationDescription.h:63
void fitModel(const String &model_type, const Param &params=Param())
Fits a model to the data.
Used to load and store TransformationXML files.
Definition: TransformationXMLFile.h:59
void load(const String &filename, TransformationDescription &transformation, bool fit_model=true)
Loads the transformation from an TransformationXML file.
void store(String filename, const TransformationDescription &transformation)
Stores the data in an TransformationXML file.
This class supports reading and writing of PQP files.
Definition: TransitionPQPFile.h:217
void convertPQPToTargetedExperiment(const char *filename, OpenMS::TargetedExperiment &targeted_exp, bool legacy_traml_id=false)
Read in a PQP file and construct a targeted experiment (TraML structure)
This class supports reading and writing of OpenSWATH transition lists.
Definition: TransitionTSVFile.h:147
void convertTSVToTargetedExperiment(const char *filename, FileTypes::Type filetype, OpenMS::TargetedExperiment &targeted_exp)
Read in a tsv/mrm file and construct a targeted experiment (TraML structure)
OPENMS_UINT64_TYPE UInt64
Unsigned integer type (64bit)
Definition: Types.h:77
size_t Size
Size type e.g. used as variable which can hold result of size()
Definition: Types.h:127
std::vector< String > StringList
Vector of String.
Definition: ListUtils.h:70
Main OpenMS namespace.
Definition: FeatureDeconvolution.h:47
ChromatogramExtractor parameters.
Definition: OpenSwathWorkflow.h:83
Type
Actual file types enum.
Definition: FileTypes.h:57
@ TRAML
TraML (HUPO PSI format) for transitions (.traML)
Definition: FileTypes.h:80
@ PQP
OpenSWATH Peptide Query Parameter (PQP) SQLite DB, see TransitionPQPFile.
Definition: FileTypes.h:102
@ TSV
any TSV file, for example msInspect file or OpenSWATH transition file (see TransitionTSVFile)
Definition: FileTypes.h:86
@ MZML
MzML file (.mzML)
Definition: FileTypes.h:70
@ SQMASS
SqLite format for mass and chromatograms, see SqMassFile.
Definition: FileTypes.h:101
@ MZXML
MzXML file (.mzXML)
Definition: FileTypes.h:62
Configuration class for MSNumpress.
Definition: MSNumpressCoder.h:89
double numpressErrorTolerance
Check error tolerance after encoding.
Definition: MSNumpressCoder.h:108
void setCompression(const std::string &compression)
Set compression using a string mapping to enum NumpressCompression.
Definition: MSNumpressCoder.h:149
double linear_fp_mass_acc
Desired mass accuracy for *linear* encoding.
Definition: MSNumpressCoder.h:131
bool estimate_fixed_point
Whether to estimate the fixed point used for encoding (highly recommended)
Definition: MSNumpressCoder.h:123
Definition: TransitionExperiment.h:207
std::vector< LightTransition > transitions
Definition: TransitionExperiment.h:215