OpenMS  2.4.0
PeptideIndexing.h
Go to the documentation of this file.
1 // --------------------------------------------------------------------------
2 // OpenMS -- Open-Source Mass Spectrometry
3 // --------------------------------------------------------------------------
4 // Copyright The OpenMS Team -- Eberhard Karls University Tuebingen,
5 // ETH Zurich, and Freie Universitaet Berlin 2002-2018.
6 //
7 // This software is released under a three-clause BSD license:
8 // * Redistributions of source code must retain the above copyright
9 // notice, this list of conditions and the following disclaimer.
10 // * Redistributions in binary form must reproduce the above copyright
11 // notice, this list of conditions and the following disclaimer in the
12 // documentation and/or other materials provided with the distribution.
13 // * Neither the name of any author or any participating institution
14 // may be used to endorse or promote products derived from this software
15 // without specific prior written permission.
16 // For a full list of authors, refer to the file AUTHORS.
17 // --------------------------------------------------------------------------
18 // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
19 // AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 // ARE DISCLAIMED. IN NO EVENT SHALL ANY OF THE AUTHORS OR THE CONTRIBUTING
22 // INSTITUTIONS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
23 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
24 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
25 // OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
26 // WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
27 // OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
28 // ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 //
30 // --------------------------------------------------------------------------
31 // $Maintainer: Chris Bielow $
32 // $Authors: Andreas Bertsch, Chris Bielow $
33 // --------------------------------------------------------------------------
34 
35 #pragma once
36 
37 
53 #include <OpenMS/SYSTEM/SysInfo.h>
54 
55 #include <atomic>
56 #include <algorithm>
57 #include <fstream>
59 
60 namespace OpenMS
61 {
62 
123  class OPENMS_DLLAPI PeptideIndexing :
124  public DefaultParamHandler, public ProgressLogger
125  {
126 public:
127 
130  {
137  };
138 
140  PeptideIndexing();
141 
143  ~PeptideIndexing() override;
144 
145 
147  inline ExitCodes run(std::vector<FASTAFile::FASTAEntry>& proteins, std::vector<ProteinIdentification>& prot_ids, std::vector<PeptideIdentification>& pep_ids)
148  {
149  FASTAContainer<TFI_Vector> protein_container(proteins);
150  return run<TFI_Vector>(protein_container, prot_ids, pep_ids);
151  }
152 
188  template<typename T>
189  ExitCodes run(FASTAContainer<T>& proteins, std::vector<ProteinIdentification>& prot_ids, std::vector<PeptideIdentification>& pep_ids)
190  {
191  // no decoy string provided? try to deduce from data
192  if (decoy_string_.empty())
193  {
194  bool is_decoy_string_auto_successful = findDecoyString_(proteins);
195 
196  if (!is_decoy_string_auto_successful && contains_decoys_)
197  {
198  return DECOYSTRING_EMPTY;
199  }
200  else if (!is_decoy_string_auto_successful && !contains_decoys_)
201  {
202  LOG_WARN << "Unable to determine decoy string automatically, not enough decoys were detected! Using default " << (prefix_ ? "prefix" : "suffix") << " decoy string '" << decoy_string_ << "\n"
203  << "If you think that this is false, please provide a decoy_string and its position manually!" << std::endl;
204  }
205  else
206  {
207  // decoy string and position was extracted successfully
208  LOG_INFO << "Using " << (prefix_ ? "prefix" : "suffix") << " decoy string '" << decoy_string_ << "'" << std::endl;
209  }
210 
211  proteins.reset();
212  }
213 
214  //---------------------------------------------------------------
215  // parsing parameters, correcting xtandem and MSGFPlus parameters
216  //---------------------------------------------------------------
217  ProteaseDigestion enzyme;
218  enzyme.setEnzyme(enzyme_name_);
219  enzyme.setSpecificity(enzyme.getSpecificityByName(enzyme_specificity_));
220 
221  bool xtandem_fix_parameters = true, msgfplus_fix_parameters = true;
222 
223  // specificity is none or semi? don't automate xtandem
226  {
227  xtandem_fix_parameters = false;
228  }
229 
230  // enzyme is already Trypsin/P? don't automate MSGFPlus
231  if (enzyme.getEnzymeName() == "Trypsin/P") { msgfplus_fix_parameters = false; }
232 
233  // determine if search engine is solely xtandem or MSGFPlus
234  for (const auto& prot_id : prot_ids)
235  {
236  if (!msgfplus_fix_parameters && !xtandem_fix_parameters) { break; }
237 
238  const std::string& search_engine = prot_id.getSearchEngine();
239  if (search_engine != "XTANDEM") { xtandem_fix_parameters = false; }
240  if (search_engine != "MSGFPLUS" || "MS-GF+") { msgfplus_fix_parameters = false; }
241  }
242 
243  //solely xtandem -> enzyme specificity to semi
244  if (xtandem_fix_parameters)
245  {
246  LOG_WARN << "XTandem used but specificity set to full. Setting enzyme specificity to semi specific cleavage to cope with special cutting rules in XTandem." << std::endl;
248  }
249  // solely MSGFPlus -> Trypsin P as enzyme
250  else if (msgfplus_fix_parameters && enzyme.getEnzymeName() == "Trypsin")
251  {
252  LOG_WARN << "MSGFPlus detected but enzyme cutting rules were set to Trypsin. Correcting to Trypsin/P to copy with special cutting rule in MSGFPlus." << std::endl;
253  enzyme.setEnzyme("Trypsin/P");
254  }
255 
256  //-------------------------------------------------------------
257  // calculations
258  //-------------------------------------------------------------
259  // cache the first proteins
260  const size_t PROTEIN_CACHE_SIZE = 4e5; // 400k should be enough for most DB's and is not too hard on memory either (~200 MB FASTA)
261 
262  proteins.cacheChunk(PROTEIN_CACHE_SIZE);
263 
264  if (proteins.empty()) // we do not allow an empty database
265  {
266  LOG_ERROR << "Error: An empty database was provided. Mapping makes no sense. Aborting..." << std::endl;
267  return DATABASE_EMPTY;
268  }
269 
270  if (pep_ids.empty()) // Aho-Corasick requires non-empty input; but we allow this case, since the TOPP tool should not crash when encountering a bad raw file (with no PSMs)
271  {
272  LOG_WARN << "Warning: An empty set of peptide identifications was provided. Output will be empty as well." << std::endl;
273  if (!keep_unreferenced_proteins_)
274  {
275  // delete only protein hits, not whole ID runs incl. meta data:
276  for (std::vector<ProteinIdentification>::iterator it = prot_ids.begin();
277  it != prot_ids.end(); ++it)
278  {
279  it->getHits().clear();
280  }
281  }
282  return PEPTIDE_IDS_EMPTY;
283  }
284 
285  FoundProteinFunctor func(enzyme); // store the matches
286  Map<String, Size> acc_to_prot; // map: accessions --> FASTA protein index
287  std::vector<bool> protein_is_decoy; // protein index -> is decoy?
288  std::vector<std::string> protein_accessions; // protein index -> accession
289 
290  bool invalid_protein_sequence = false; // check for proteins with modifications, i.e. '[' or '(', and throw an exception
291 
292  { // new scope - forget data after search
293 
294  /*
295  BUILD Peptide DB
296  */
297  bool has_illegal_AAs(false);
299  for (std::vector<PeptideIdentification>::const_iterator it1 = pep_ids.begin(); it1 != pep_ids.end(); ++it1)
300  {
301  //String run_id = it1->getIdentifier();
302  const std::vector<PeptideHit>& hits = it1->getHits();
303  for (std::vector<PeptideHit>::const_iterator it2 = hits.begin(); it2 != hits.end(); ++it2)
304  {
305  //
306  // Warning:
307  // do not skip over peptides here, since the results are iterated in the same way
308  //
309  String seq = it2->getSequence().toUnmodifiedString().remove('*'); // make a copy, i.e. do NOT change the peptide sequence!
310  if (seqan::isAmbiguous(seqan::AAString(seq.c_str())))
311  { // do not quit here, to show the user all sequences .. only quit after loop
312  LOG_ERROR << "Peptide sequence '" << it2->getSequence() << "' contains one or more ambiguous amino acids (B|J|Z|X).\n";
313  has_illegal_AAs = true;
314  }
315  if (IL_equivalent_) // convert L to I;
316  {
317  seq.substitute('L', 'I');
318  }
319  appendValue(pep_DB, seq.c_str());
320  }
321  }
322  if (has_illegal_AAs)
323  {
324  LOG_ERROR << "One or more peptides contained illegal amino acids. This is not allowed!"
325  << "\nPlease either remove the peptide or replace it with one of the unambiguous ones (while allowing for ambiguous AA's to match the protein)." << std::endl;;
326  }
327 
328  LOG_INFO << "Mapping " << length(pep_DB) << " peptides to " << (proteins.size() == PROTEIN_CACHE_SIZE ? "? (unknown number of)" : String(proteins.size())) << " proteins." << std::endl;
329 
330  if (length(pep_DB) == 0)
331  { // Aho-Corasick will crash if given empty needles as input
332  LOG_WARN << "Warning: Peptide identifications have no hits inside! Output will be empty as well." << std::endl;
333  return PEPTIDE_IDS_EMPTY;
334  }
335 
336  /*
337  Aho Corasick (fast)
338  */
339  LOG_INFO << "Searching with up to " << aaa_max_ << " ambiguous amino acid(s) and " << mm_max_ << " mismatch(es)!" << std::endl;
341  LOG_INFO << "Building trie ...";
342  StopWatch s;
343  s.start();
345  AhoCorasickAmbiguous::initPattern(pep_DB, aaa_max_, mm_max_, pattern);
346  s.stop();
347  LOG_INFO << " done (" << int(s.getClockTime()) << "s)" << std::endl;
348  s.reset();
349 
350  uint16_t count_j_proteins(0);
351  bool has_active_data = true; // becomes false if end of FASTA file is reached
352  const std::string jumpX(aaa_max_ + mm_max_ + 1, 'X'); // jump over stretches of 'X' which cost a lot of time; +1 because AXXA is a valid hit for aaa_max == 2 (cannot split it)
353  this->startProgress(0, proteins.size(), "Aho-Corasick");
354  std::atomic<int> progress_prots(0);
355 #ifdef _OPENMP
356 #pragma omp parallel
357 #endif
358  {
359  FoundProteinFunctor func_threads(enzyme);
360  Map<String, Size> acc_to_prot_thread; // map: accessions --> FASTA protein index
361  AhoCorasickAmbiguous fuzzyAC;
362  String prot;
363 
364  while (true)
365  {
366  #pragma omp barrier // all threads need to be here, since we are about to swap protein data
367  #pragma omp single
368  {
369  DEBUG_ONLY std::cerr << " activating cache ...\n";
370  has_active_data = proteins.activateCache(); // swap in last cache
371  protein_accessions.resize(proteins.getChunkOffset() + proteins.chunkSize());
372  } // implicit barrier here
373 
374  if (!has_active_data) break; // leave while-loop
375  SignedSize prot_count = (SignedSize)proteins.chunkSize();
376 
377  #pragma omp master
378  {
379  DEBUG_ONLY std::cerr << "Filling Protein Cache ...";
380  proteins.cacheChunk(PROTEIN_CACHE_SIZE);
381  protein_is_decoy.resize(proteins.getChunkOffset() + prot_count);
382  for (SignedSize i = 0; i < prot_count; ++i)
383  { // do this in master only, to avoid false sharing
384  const String& seq = proteins.chunkAt(i).identifier;
385  protein_is_decoy[i + proteins.getChunkOffset()] = (prefix_ ? seq.hasPrefix(decoy_string_) : seq.hasSuffix(decoy_string_));
386  }
387  DEBUG_ONLY std::cerr << " done" << std::endl;
388  }
389  DEBUG_ONLY std::cerr << " starting for loop \n";
390  // search all peptides in each protein
391  #pragma omp for schedule(dynamic, 100) nowait
392  for (SignedSize i = 0; i < prot_count; ++i)
393  {
394  ++progress_prots; // atomic
395  if (omp_get_thread_num() == 0)
396  {
397  this->setProgress(progress_prots);
398  }
399 
400  prot = proteins.chunkAt(i).sequence;
401  prot.remove('*');
402 
403  // check for invalid sequences with modifications
404  if (prot.has('[') || prot.has('('))
405  {
406  invalid_protein_sequence = true; // not omp-critical because its write-only
407  // we cannot throw an exception here, since we'd need to catch it within the parallel region
408  }
409 
410  // convert L/J to I; also replace 'J' in proteins
411  if (IL_equivalent_)
412  {
413  prot.substitute('L', 'I');
414  prot.substitute('J', 'I');
415  }
416  else
417  { // warn if 'J' is found (it eats into aaa_max)
418  if (prot.has('J'))
419  {
420  #pragma omp atomic
421  ++count_j_proteins;
422  }
423  }
424 
425  Size prot_idx = i + proteins.getChunkOffset();
426 
427  // test if protein was a hit
428  Size hits_total = func_threads.filter_passed + func_threads.filter_rejected;
429 
430  // check if there are stretches of 'X'
431  if (prot.has('X'))
432  {
433  // create chunks of the protein (splitting it at stretches of 'X..X') and feed them to AC one by one
434  size_t offset = -1, start = 0;
435  while ((offset = prot.find(jumpX, offset + 1)) != std::string::npos)
436  {
437  //std::cout << "found X..X at " << offset << " in protein " << proteins[i].identifier << "\n";
438  addHits_(fuzzyAC, pattern, pep_DB, prot.substr(start, offset + jumpX.size() - start), prot, prot_idx, (int)start, func_threads);
439  // skip ahead while we encounter more X...
440  while (offset + jumpX.size() < prot.size() && prot[offset + jumpX.size()] == 'X') ++offset;
441  start = offset;
442  //std::cout << " new start: " << start << "\n";
443  }
444  // last chunk
445  if (start < prot.size())
446  {
447  addHits_(fuzzyAC, pattern, pep_DB, prot.substr(start), prot, prot_idx, (int)start, func_threads);
448  }
449  }
450  else
451  {
452  addHits_(fuzzyAC, pattern, pep_DB, prot, prot, prot_idx, 0, func_threads);
453  }
454  // was protein found?
455  if (hits_total < func_threads.filter_passed + func_threads.filter_rejected)
456  {
457  protein_accessions[prot_idx] = proteins.chunkAt(i).identifier;
458  acc_to_prot_thread[protein_accessions[prot_idx]] = prot_idx;
459  }
460  } // end parallel FOR
461 
462  // join results again
463  DEBUG_ONLY std::cerr << " critical now \n";
464  #ifdef _OPENMP
465  #pragma omp critical(PeptideIndexer_joinAC)
466  #endif
467  {
468  s.start();
469  // hits
470  func.merge(func_threads);
471  // accession -> index
472  acc_to_prot.insert(acc_to_prot_thread.begin(), acc_to_prot_thread.end());
473  acc_to_prot_thread.clear();
474  s.stop();
475  } // OMP end critical
476  } // end readChunk
477  } // OMP end parallel
478  this->endProgress();
479  std::cout << "Merge took: " << s.toString() << "\n";
480  mu.after();
481  std::cout << mu.delta("Aho-Corasick") << "\n\n";
482 
483  LOG_INFO << "\nAho-Corasick done:\n found " << func.filter_passed << " hits for " << func.pep_to_prot.size() << " of " << length(pep_DB) << " peptides.\n";
484 
485  // write some stats
486  LOG_INFO << "Peptide hits passing enzyme filter: " << func.filter_passed << "\n"
487  << " ... rejected by enzyme filter: " << func.filter_rejected << std::endl;
488 
489  if (count_j_proteins)
490  {
491  LOG_WARN << "PeptideIndexer found " << count_j_proteins << " protein sequences in your database containing the amino acid 'J'."
492  << "To match 'J' in a protein, an ambiguous amino acid placeholder for I/L will be used.\n"
493  << "This costs runtime and eats into the 'aaa_max' limit, leaving less opportunity for B/Z/X matches.\n"
494  << "If you want 'J' to be treated as unambiguous, enable '-IL_equivalent'!" << std::endl;
495  }
496 
497  } // end local scope
498 
499  //
500  // do mapping
501  //
502  // index existing proteins
503  Map<String, Size> runid_to_runidx; // identifier to index
504  for (Size run_idx = 0; run_idx < prot_ids.size(); ++run_idx)
505  {
506  runid_to_runidx[prot_ids[run_idx].getIdentifier()] = run_idx;
507  }
508 
509  // for peptides --> proteins
510  Size stats_matched_unique(0);
511  Size stats_matched_multi(0);
512  Size stats_unmatched(0); // no match to DB
513  Size stats_count_m_t(0); // match to Target DB
514  Size stats_count_m_d(0); // match to Decoy DB
515  Size stats_count_m_td(0); // match to T+D DB
516 
517  Map<Size, std::set<Size> > runidx_to_protidx; // in which protID do appear which proteins (according to mapped peptides)
518 
519  Size pep_idx(0);
520  for (std::vector<PeptideIdentification>::iterator it1 = pep_ids.begin(); it1 != pep_ids.end(); ++it1)
521  {
522  // which ProteinIdentification does the peptide belong to?
523  Size run_idx = runid_to_runidx[it1->getIdentifier()];
524 
525  std::vector<PeptideHit>& hits = it1->getHits();
526 
527  for (std::vector<PeptideHit>::iterator it2 = hits.begin(); it2 != hits.end(); ++it2)
528  {
529  // clear protein accessions
530  it2->setPeptideEvidences(std::vector<PeptideEvidence>());
531 
532  //
533  // is this a decoy hit?
534  //
535  bool matches_target(false);
536  bool matches_decoy(false);
537 
538  std::set<Size> prot_indices;
539  // add new protein references
540  for (std::set<PeptideProteinMatchInformation>::const_iterator it_i = func.pep_to_prot[pep_idx].begin();
541  it_i != func.pep_to_prot[pep_idx].end(); ++it_i)
542  {
543  prot_indices.insert(it_i->protein_index);
544  const String& accession = protein_accessions[it_i->protein_index];
545  PeptideEvidence pe(accession, it_i->position, it_i->position + (int)it2->getSequence().size() - 1, it_i->AABefore, it_i->AAAfter);
546  it2->addPeptideEvidence(pe);
547 
548  runidx_to_protidx[run_idx].insert(it_i->protein_index); // fill protein hits
549 
550  if (protein_is_decoy[it_i->protein_index])
551  {
552  matches_decoy = true;
553  }
554  else
555  {
556  matches_target = true;
557  }
558  }
559 
560  if (matches_decoy && matches_target)
561  {
562  it2->setMetaValue("target_decoy", "target+decoy");
563  ++stats_count_m_td;
564  }
565  else if (matches_target)
566  {
567  it2->setMetaValue("target_decoy", "target");
568  ++stats_count_m_t;
569  }
570  else if (matches_decoy)
571  {
572  it2->setMetaValue("target_decoy", "decoy");
573  ++stats_count_m_d;
574  } // else: could match to no protein (i.e. both are false)
575  //else ... // not required (handled below; see stats_unmatched);
576 
577  if (prot_indices.size() == 1)
578  {
579  it2->setMetaValue("protein_references", "unique");
580  ++stats_matched_unique;
581  }
582  else if (prot_indices.size() > 1)
583  {
584  it2->setMetaValue("protein_references", "non-unique");
585  ++stats_matched_multi;
586  }
587  else
588  {
589  it2->setMetaValue("protein_references", "unmatched");
590  ++stats_unmatched;
591  if (stats_unmatched < 15) LOG_INFO << "Unmatched peptide: " << it2->getSequence() << "\n";
592  else if (stats_unmatched == 15) LOG_INFO << "Unmatched peptide: ...\n";
593  }
594 
595  ++pep_idx; // next hit
596  }
597 
598  }
599 
600  Size total_peptides = stats_count_m_t + stats_count_m_d + stats_count_m_td + stats_unmatched;
601  LOG_INFO << "-----------------------------------\n";
602  LOG_INFO << "Peptide statistics\n";
603  LOG_INFO << "\n";
604  LOG_INFO << " unmatched : " << stats_unmatched << " (" << stats_unmatched * 100 / total_peptides << " %)\n";
605  LOG_INFO << " target/decoy:\n";
606  LOG_INFO << " match to target DB only: " << stats_count_m_t << " (" << stats_count_m_t * 100 / total_peptides << " %)\n";
607  LOG_INFO << " match to decoy DB only : " << stats_count_m_d << " (" << stats_count_m_d * 100 / total_peptides << " %)\n";
608  LOG_INFO << " match to both : " << stats_count_m_td << " (" << stats_count_m_td * 100 / total_peptides << " %)\n";
609  LOG_INFO << "\n";
610  LOG_INFO << " mapping to proteins:\n";
611  LOG_INFO << " no match (to 0 protein) : " << stats_unmatched << "\n";
612  LOG_INFO << " unique match (to 1 protein) : " << stats_matched_unique << "\n";
613  LOG_INFO << " non-unique match (to >1 protein): " << stats_matched_multi << std::endl;
614 
616  Size stats_matched_proteins(0), stats_matched_new_proteins(0), stats_orphaned_proteins(0), stats_proteins_target(0), stats_proteins_decoy(0);
617 
618  // all peptides contain the correct protein hit references, now update the protein hits
619  for (Size run_idx = 0; run_idx < prot_ids.size(); ++run_idx)
620  {
621  std::set<Size> masterset = runidx_to_protidx[run_idx]; // all protein matches from above
622 
623  std::vector<ProteinHit>& phits = prot_ids[run_idx].getHits();
624  {
625  // go through existing protein hits and count orphaned proteins (with no peptide hits)
626  std::vector<ProteinHit> orphaned_hits;
627  for (std::vector<ProteinHit>::iterator p_hit = phits.begin(); p_hit != phits.end(); ++p_hit)
628  {
629  const String& acc = p_hit->getAccession();
630  if (!acc_to_prot.has(acc)) // acc_to_prot only contains found proteins from current run
631  { // old hit is orphaned
632  ++stats_orphaned_proteins;
633  if (keep_unreferenced_proteins_)
634  {
635  p_hit->setMetaValue("target_decoy", "");
636  orphaned_hits.push_back(*p_hit);
637  }
638  }
639  }
640  // only keep orphaned hits (if any)
641  phits = orphaned_hits;
642  }
643 
644  // add new protein hits
646  phits.reserve(phits.size() + masterset.size());
647  for (std::set<Size>::const_iterator it = masterset.begin(); it != masterset.end(); ++it)
648  {
649  ProteinHit hit;
650  hit.setAccession(protein_accessions[*it]);
651 
652  if (write_protein_sequence_ || write_protein_description_)
653  {
654  proteins.readAt(fe, *it);
655  if (write_protein_sequence_)
656  {
657  hit.setSequence(fe.sequence);
658  } // no else, since sequence is empty by default
659  if (write_protein_description_)
660  {
661  hit.setDescription(fe.description);
662  } // no else, since description is empty by default
663  }
664  if (protein_is_decoy[*it])
665  {
666  hit.setMetaValue("target_decoy", "decoy");
667  ++stats_proteins_decoy;
668  }
669  else
670  {
671  hit.setMetaValue("target_decoy", "target");
672  ++stats_proteins_target;
673  }
674  phits.push_back(hit);
675  ++stats_matched_new_proteins;
676  }
677  stats_matched_proteins += phits.size();
678  }
679 
680 
681  LOG_INFO << "-----------------------------------\n";
682  LOG_INFO << "Protein statistics\n";
683  LOG_INFO << "\n";
684  LOG_INFO << " total proteins searched: " << proteins.size() << "\n";
685  LOG_INFO << " matched proteins : " << stats_matched_proteins << " (" << stats_matched_new_proteins << " new)\n";
686  if (stats_matched_proteins)
687  { // prevent Division-by-0 Exception
688  LOG_INFO << " matched target proteins: " << stats_proteins_target << " (" << stats_proteins_target * 100 / stats_matched_proteins << " %)\n";
689  LOG_INFO << " matched decoy proteins : " << stats_proteins_decoy << " (" << stats_proteins_decoy * 100 / stats_matched_proteins << " %)\n";
690  }
691  LOG_INFO << " orphaned proteins : " << stats_orphaned_proteins << (keep_unreferenced_proteins_ ? " (all kept)" : " (all removed)\n");
692  LOG_INFO << "-----------------------------------" << std::endl;
693 
694 
696  bool has_error = false;
697 
698  if (invalid_protein_sequence)
699  {
700  LOG_ERROR << "Error: One or more protein sequences contained the characters '[' or '(', which are illegal in protein sequences."
701  << "\nPeptide hits might be masked by these characters (which usually indicate presence of modifications).\n";
702  has_error = true;
703  }
704 
705  if ((stats_count_m_d + stats_count_m_td) == 0)
706  {
707  String msg("No peptides were matched to the decoy portion of the database! Did you provide the correct concatenated database? Are your 'decoy_string' (=" + String(decoy_string_) + ") and 'decoy_string_position' (=" + String(param_.getValue("decoy_string_position")) + ") settings correct?");
708  if (missing_decoy_action_ == "error")
709  {
710  LOG_ERROR << "Error: " << msg << "\nSet 'missing_decoy_action' to 'warn' if you are sure this is ok!\nAborting ..." << std::endl;
711  has_error = true;
712  }
713  else if (missing_decoy_action_ == "warn")
714  {
715  LOG_WARN << "Warn: " << msg << "\nSet 'missing_decoy_action' to 'error' if you want to elevate this to an error!" << std::endl;
716  }
717  else // silent
718  {
719  }
720  }
721 
722  if ((!allow_unmatched_) && (stats_unmatched > 0))
723  {
724  LOG_ERROR << "PeptideIndexer found unmatched peptides, which could not be associated to a protein.\n"
725  << "Potential solutions:\n"
726  << " - check your FASTA database for completeness\n"
727  << " - set 'enzyme:specificity' to match the identification parameters of the search engine\n"
728  << " - some engines (e.g. X! Tandem) employ loose cutting rules generating non-tryptic peptides;\n"
729  << " if you trust them, disable enzyme specificity\n"
730  << " - increase 'aaa_max' to allow more ambiguous amino acids\n"
731  << " - as a last resort: use the 'allow_unmatched' option to accept unmatched peptides\n"
732  << " (note that unmatched peptides cannot be used for FDR calculation or quantification)\n";
733  has_error = true;
734  }
735 
736  if (has_error)
737  {
738  LOG_ERROR << "Result files will be written, but PeptideIndexer will exit with an error code." << std::endl;
739  return UNEXPECTED_RESULT;
740  }
741  return EXECUTION_OK;
742  }
743 
744  const String& getDecoyString() const;
745 
746  bool isPrefix() const;
747 
748  protected:
749  using DecoyStringToAffixCount = std::map<std::string, std::pair<int, int>>;
750  using CaseInsensitiveToCaseSensitiveDecoy = std::map<std::string, std::string>;
752 
753  template<typename T>
755  {
756  // common decoy strings in FASTA files
757  // note: decoy prefixes/suffices must be provided in lower case
758  std::vector<std::string> affixes = {"decoy", "dec", "reverse", "rev", "__id_decoy", "xxx", "shuffled", "shuffle", "pseudo", "random"};
759 
760  // map decoys to counts of occurrences as prefix/suffix
761  DecoyStringToAffixCount decoy_count;
762  // map case insensitive strings back to original case (as used in fasta)
763  CaseInsensitiveToCaseSensitiveDecoy decoy_case_sensitive;
764 
765  // assume that it contains decoys for now
766  contains_decoys_ = true;
767 
768  // setup prefix- and suffix regex strings
769  const std::string regexstr_prefix = std::string("^(") + ListUtils::concatenate<std::string>(affixes, "_*|") + "_*)";
770  const std::string regexstr_suffix = std::string("(") + ListUtils::concatenate<std::string>(affixes, "_*|") + "_*)$";
771 
772  // setup regexes
773  const boost::regex pattern_prefix(regexstr_prefix);
774  const boost::regex pattern_suffix(regexstr_suffix);
775 
776  int all_prefix_occur(0), all_suffix_occur(0), all_proteins_count(0);
777 
778  const size_t PROTEIN_CACHE_SIZE = 4e5;
779 
780  while (true)
781  {
782  proteins.cacheChunk(PROTEIN_CACHE_SIZE);
783  if (!proteins.activateCache()) break;
784 
785  auto prot_count = (SignedSize) proteins.chunkSize();
786  all_proteins_count += prot_count;
787 
788  {
789  for (SignedSize i = 0; i < prot_count; ++i)
790  {
791  String seq = proteins.chunkAt(i).identifier;
792 
793  String seq_lower = seq;
794  seq_lower.toLower();
795 
796  boost::smatch sm;
797  // search for prefix
798  bool found_prefix = boost::regex_search(seq_lower, sm, pattern_prefix);
799  if (found_prefix)
800  {
801  std::string match = sm[0];
802  all_prefix_occur++;
803 
804  // increase count of observed prefix
805  decoy_count[match].first++;
806 
807  // store observed (case sensitive and with special characters)
808  std::string seq_decoy = StringUtils::prefix(seq, match.length());
809  decoy_case_sensitive[match] = seq_decoy;
810  }
811 
812  // search for suffix
813  bool found_suffix = boost::regex_search(seq_lower, sm, pattern_suffix);
814  if (found_suffix)
815  {
816  std::string match = sm[0];
817  all_suffix_occur++;
818 
819  // increase count of observed suffix
820  decoy_count[match].second++;
821 
822  // store observed (case sensitive and with special characters)
823  std::string seq_decoy = StringUtils::suffix(seq, match.length());
824  decoy_case_sensitive[match] = seq_decoy;
825  }
826  }
827  }
828  }
829 
830  // DEBUG ONLY: print counts of found decoys
831  for (auto &a : decoy_count) LOG_DEBUG << a.first << "\t" << a.second.first << "\t" << a.second.second << std::endl;
832 
833  // less than 40% of proteins are decoys -> won't be able to determine a decoy string and its position
834  // return default values
835  if (all_prefix_occur + all_suffix_occur < 0.4 * all_proteins_count) {
836  decoy_string_ = "DECOY_";
837  prefix_ = true;
838 
839  contains_decoys_ = false;
840  return false;
841  }
842 
843  if (all_prefix_occur == all_suffix_occur)
844  {
845  LOG_ERROR << "Unable to determine decoy string!" << std::endl;
846  return false;
847  }
848 
849  // Decoy prefix occurred at least 80% of all prefixes + observed in at least 40% of all proteins -> set it as prefix decoy
850  for (const auto& pair : decoy_count)
851  {
852  const std::string & case_insensitive_decoy_string = pair.first;
853  const std::pair<int, int>& prefix_suffix_counts = pair.second;
854  double freq_prefix = static_cast<double>(prefix_suffix_counts.first) / static_cast<double>(all_prefix_occur);
855  double freq_prefix_in_proteins = static_cast<double>(prefix_suffix_counts.first) / static_cast<double>(all_proteins_count);
856 
857  if (freq_prefix >= 0.8 && freq_prefix_in_proteins >= 0.4)
858  {
859  prefix_ = true;
860  decoy_string_ = decoy_case_sensitive[case_insensitive_decoy_string];
861 
862  if (prefix_suffix_counts.first != all_prefix_occur)
863  {
864  LOG_WARN << "More than one decoy prefix observed!" << std::endl;
865  LOG_WARN << "Using most frequent decoy prefix (" << (int) (freq_prefix * 100) <<"%)" << std::endl;
866  }
867 
868  return true;
869  }
870  }
871 
872  // Decoy suffix occurred at least 80% of all suffixes + observed in at least 40% of all proteins -> set it as suffix decoy
873  for (const auto& pair : decoy_count)
874  {
875  const std::string& case_insensitive_decoy_string = pair.first;
876  const std::pair<int, int>& prefix_suffix_counts = pair.second;
877  double freq_suffix = static_cast<double>(prefix_suffix_counts.second) / static_cast<double>(all_suffix_occur);
878  double freq_suffix_in_proteins = static_cast<double>(prefix_suffix_counts.second) / static_cast<double>(all_proteins_count);
879 
880  if (freq_suffix >= 0.8 && freq_suffix_in_proteins >= 0.4)
881  {
882  prefix_ = false;
883  decoy_string_ = decoy_case_sensitive[case_insensitive_decoy_string];
884 
885  if (prefix_suffix_counts.second != all_suffix_occur)
886  {
887  LOG_WARN << "More than one decoy suffix observed!" << std::endl;
888  LOG_WARN << "Using most frequent decoy suffix (" << (int) (freq_suffix * 100) <<"%)" << std::endl;
889  }
890 
891  return true;
892  }
893  }
894 
895  LOG_ERROR << "Unable to determine decoy string and its position. Please provide a decoy string and its position as parameters." << std::endl;
896  return false;
897  }
898 
899 
901  {
904 
907 
909  char AABefore;
910 
912  char AAAfter;
913 
914  bool operator<(const PeptideProteinMatchInformation& other) const
915  {
916  if (protein_index != other.protein_index)
917  {
918  return protein_index < other.protein_index;
919  }
920  else if (position != other.position)
921  {
922  return position < other.position;
923  }
924  else if (AABefore != other.AABefore)
925  {
926  return AABefore < other.AABefore;
927  }
928  else if (AAAfter != other.AAAfter)
929  {
930  return AAAfter < other.AAAfter;
931  }
932  return false;
933  }
934 
936  {
937  return protein_index == other.protein_index &&
938  position == other.position &&
939  AABefore == other.AABefore &&
940  AAAfter == other.AAAfter;
941  }
942 
943  };
945  {
946  public:
947  typedef std::map<OpenMS::Size, std::set<PeptideProteinMatchInformation> > MapType;
948 
951 
954 
957 
958  private:
960 
961  public:
962  explicit FoundProteinFunctor(const ProteaseDigestion& enzyme) :
963  pep_to_prot(), filter_passed(0), filter_rejected(0), enzyme_(enzyme)
964  {
965  }
966 
968  {
969  if (pep_to_prot.empty())
970  { // first merge is easy
971  pep_to_prot.swap(other.pep_to_prot);
972  }
973  else
974  {
975  for (FoundProteinFunctor::MapType::const_iterator it = other.pep_to_prot.begin(); it != other.pep_to_prot.end(); ++it)
976  { // augment set
977  this->pep_to_prot[it->first].insert(other.pep_to_prot[it->first].begin(), other.pep_to_prot[it->first].end());
978  }
979  other.pep_to_prot.clear();
980  }
981  // cheap members
982  this->filter_passed += other.filter_passed;
983  other.filter_passed = 0;
984  this->filter_rejected += other.filter_rejected;
985  other.filter_rejected = 0;
986  }
987 
988  void addHit(const OpenMS::Size idx_pep,
989  const OpenMS::Size idx_prot,
990  const OpenMS::Size len_pep,
991  const OpenMS::String& seq_prot,
993  {
994  if (enzyme_.isValidProduct(seq_prot, position, len_pep, true, true))
995  {
997  match.protein_index = idx_prot;
998  match.position = position;
999  match.AABefore = (position == 0) ? PeptideEvidence::N_TERMINAL_AA : seq_prot[position - 1];
1000  match.AAAfter = (position + len_pep >= seq_prot.size()) ? PeptideEvidence::C_TERMINAL_AA : seq_prot[position + len_pep];
1001  pep_to_prot[idx_pep].insert(match);
1002  ++filter_passed;
1003  }
1004  else
1005  {
1006  //std::cerr << "REJECTED Peptide " << seq_pep << " with hit to protein "
1007  // << seq_prot << " at position " << position << std::endl;
1008  ++filter_rejected;
1009  }
1010  }
1011 
1012  };
1013 
1014  inline void addHits_(AhoCorasickAmbiguous& fuzzyAC, const AhoCorasickAmbiguous::FuzzyACPattern& pattern, const AhoCorasickAmbiguous::PeptideDB& pep_DB, const String& prot, const String& full_prot, SignedSize idx_prot, Int offset, FoundProteinFunctor& func_threads) const
1015  {
1016  fuzzyAC.setProtein(prot);
1017  while (fuzzyAC.findNext(pattern))
1018  {
1019  const seqan::Peptide& tmp_pep = pep_DB[fuzzyAC.getHitDBIndex()];
1020  func_threads.addHit(fuzzyAC.getHitDBIndex(), idx_prot, length(tmp_pep), full_prot, fuzzyAC.getHitProteinPosition() + offset);
1021  }
1022 
1023  }
1024 
1025  void updateMembers_() override;
1026 
1028  bool prefix_;
1032 
1038 
1041 
1042  };
1043 }
1044 
void setMetaValue(const String &name, const DataValue &value)
Sets the DataValue corresponding to a name.
FoundProteinFunctor(const ProteaseDigestion &enzyme)
Definition: PeptideIndexing.h:962
std::map< std::string, std::string > CaseInsensitiveToCaseSensitiveDecoy
Definition: PeptideIndexing.h:750
bool has(Byte byte) const
true if String contains the byte, false otherwise
char AABefore
the amino acid after the peptide in the protein
Definition: PeptideIndexing.h:909
A more convenient string class.
Definition: String.h:57
void after()
record data for the second timepoint
ExitCodes run(FASTAContainer< T > &proteins, std::vector< ProteinIdentification > &prot_ids, std::vector< PeptideIdentification > &pep_ids)
Re-index peptide identifications honoring enzyme cutting rules, ambiguous amino acids and target/deco...
Definition: PeptideIndexing.h:189
bool contains_decoys_
Definition: PeptideIndexing.h:751
#define LOG_INFO
Macro if a information, e.g. a status should be reported.
Definition: LogStream.h:454
OpenMS::Int position
the position of the peptide in the protein
Definition: PeptideIndexing.h:906
void setSequence(const String &sequence)
sets the protein sequence
Size< TNeedle >::Type position(const PatternAuxData< TNeedle > &dh)
Definition: AhoCorasickAmbiguous.h:561
bool operator==(const PeptideProteinMatchInformation &other) const
Definition: PeptideIndexing.h:935
StopWatch Class.
Definition: StopWatch.h:59
bool operator<(const PeptideProteinMatchInformation &other) const
Definition: PeptideIndexing.h:914
OpenMS::Size protein_index
index of the protein the peptide is contained in
Definition: PeptideIndexing.h:903
void setProtein(const String &protein_sequence)
Reset to new protein sequence. All previous data is forgotten.
Definition: AhoCorasickAmbiguous.h:1024
bool findDecoyString_(FASTAContainer< T > &proteins)
Definition: PeptideIndexing.h:754
A convenience class to report either absolute or delta (between two timepoints) RAM usage...
Definition: SysInfo.h:83
::seqan::Pattern< PeptideDB, ::seqan::FuzzyAC > FuzzyACPattern
Definition: AhoCorasickAmbiguous.h:974
Definition: PeptideIndexing.h:944
ptrdiff_t SignedSize
Signed Size type e.g. used as pointer difference.
Definition: Types.h:134
Refreshes the protein references for all peptide hits in a vector of PeptideIdentifications and adds ...
Definition: PeptideIndexing.h:123
bool keep_unreferenced_proteins_
Definition: PeptideIndexing.h:1035
no requirements on start / end
Definition: EnzymaticDigestion.h:70
String decoy_string_
Definition: PeptideIndexing.h:1027
String getEnzymeName() const
Returns the enzyme for the digestion.
Main OpenMS namespace.
Definition: FeatureDeconvolution.h:46
#define DEBUG_ONLY
Definition: AhoCorasickAmbiguous.h:46
std::map< std::string, std::pair< int, int > > DecoyStringToAffixCount
Definition: PeptideIndexing.h:749
Definition: PeptideIndexing.h:132
String missing_decoy_action_
Definition: PeptideIndexing.h:1029
void setSpecificity(Specificity spec)
Sets the specificity for the digestion (default is SPEC_FULL).
#define LOG_DEBUG
Macro for general debugging information.
Definition: LogStream.h:458
String toString() const
get a compact representation of the current time status.
String & remove(char what)
Remove all occurrences of the character what.
ExitCodes
Exit codes.
Definition: PeptideIndexing.h:129
bool prefix_
Definition: PeptideIndexing.h:1028
String enzyme_name_
Definition: PeptideIndexing.h:1030
#define LOG_ERROR
Macro to be used if non-fatal error are reported (processing continues)
Definition: LogStream.h:446
Size getHitDBIndex()
Get index of hit into peptide database of the pattern.
Definition: AhoCorasickAmbiguous.h:1047
String substr(size_t pos=0, size_t n=npos) const
Wrapper for the STL substr() method. Returns a String object with its contents initialized to a subst...
#define LOG_WARN
Macro if a warning, a piece of information which should be read by the user, should be logged...
Definition: LogStream.h:450
std::map< OpenMS::Size, std::set< PeptideProteinMatchInformation > > MapType
Definition: PeptideIndexing.h:947
bool findNext(const FuzzyACPattern &pattern)
Enumerate hits.
Definition: AhoCorasickAmbiguous.h:1037
Int aaa_max_
Definition: PeptideIndexing.h:1039
Int getHitProteinPosition()
Offset into protein sequence where hit was found.
Definition: AhoCorasickAmbiguous.h:1057
template parameter for vector-based FASTA access
Definition: FASTAContainer.h:76
Extended Aho-Corasick algorithm capable of matching ambiguous amino acids in the pattern (i...
Definition: AhoCorasickAmbiguous.h:970
Specificity getSpecificity() const
Returns the specificity for the digestion.
void merge(FoundProteinFunctor &other)
Definition: PeptideIndexing.h:967
static const char N_TERMINAL_AA
Definition: PeptideEvidence.h:60
bool write_protein_sequence_
Definition: PeptideIndexing.h:1033
Class for the enzymatic digestion of proteins.
Definition: ProteaseDigestion.h:60
Definition: PeptideIndexing.h:133
String & toLower()
Converts the string to lowercase.
bool isAmbiguous(AAcid c)
Definition: AhoCorasickAmbiguous.h:578
semi specific, i.e., one of the two cleavage sites must fulfill requirements
Definition: EnzymaticDigestion.h:69
Definition: PeptideIndexing.h:131
static String suffix(const String &this_s, size_t length)
Definition: StringUtils.h:269
MapType pep_to_prot
peptide index –> protein indices
Definition: PeptideIndexing.h:950
Definition: PeptideIndexing.h:134
void setDescription(const String &description)
sets the description of the protein
bool IL_equivalent_
Definition: PeptideIndexing.h:1037
Representation of a peptide evidence.
Definition: PeptideEvidence.h:50
bool has(const Key &key) const
Test whether the map contains the given key.
Definition: Map.h:108
static void initPattern(const PeptideDB &pep_db, const int aaa_max, const int mm_max, FuzzyACPattern &pattern)
Construct a trie from a set of peptide sequences (which are to be found in a protein).
Definition: AhoCorasickAmbiguous.h:991
void setEnzyme(const String &name)
Sets the enzyme for the digestion (by name)
Definition: PeptideIndexing.h:136
void setAccession(const String &accession)
sets the accession of the protein
Representation of a protein hit.
Definition: ProteinHit.h:53
String enzyme_specificity_
Definition: PeptideIndexing.h:1031
static Specificity getSpecificityByName(const String &name)
String delta(const String &event="delta")
bool hasPrefix(const String &string) const
true if String begins with string, false otherwise
size_t Size
Size type e.g. used as variable which can hold result of size()
Definition: Types.h:127
String & substitute(char from, char to)
Replaces all occurrences of the character from by the character to.
OpenMS::Size filter_rejected
number of rejected hits (not passing addHit())
Definition: PeptideIndexing.h:956
bool write_protein_description_
Definition: PeptideIndexing.h:1034
Base class for all classes that want to report their progress.
Definition: ProgressLogger.h:54
Int mm_max_
Definition: PeptideIndexing.h:1040
String sequence
Definition: FASTAFile.h:80
static String prefix(const String &this_s, size_t length)
Definition: StringUtils.h:260
FASTA entry type (identifier, description and sequence)
Definition: FASTAFile.h:76
A base class for all classes handling default parameters.
Definition: DefaultParamHandler.h:91
Definition: PeptideIndexing.h:135
bool allow_unmatched_
Definition: PeptideIndexing.h:1036
static const char C_TERMINAL_AA
Definition: PeptideEvidence.h:61
String< AAcid, Alloc< void > > AAString
Definition: AhoCorasickAmbiguous.h:206
int Int
Signed integer type.
Definition: Types.h:102
bool isValidProduct(const String &protein, int pep_pos, int pep_length, bool ignore_missed_cleavages=true, bool allow_nterm_protein_cleavage=false, bool allow_random_asp_pro_cleavage=false) const
Variant of EnzymaticDigestion::isValidProduct() with support for n-term protein cleavage and random D...
char AAAfter
the amino acid before the peptide in the protein
Definition: PeptideIndexing.h:912
Map class based on the STL map (containing several convenience functions)
Definition: Map.h:50
FASTAContainer<TFI_Vector> simply takes an existing vector of FASTAEntries and provides the same inte...
Definition: FASTAContainer.h:237
::seqan::StringSet<::seqan::AAString > PeptideDB
Definition: AhoCorasickAmbiguous.h:973
String description
Definition: FASTAFile.h:79
bool hasSuffix(const String &string) const
true if String ends with string, false otherwise
OpenMS::Size filter_passed
number of accepted hits (passing addHit() constraints)
Definition: PeptideIndexing.h:953
ExitCodes run(std::vector< FASTAFile::FASTAEntry > &proteins, std::vector< ProteinIdentification > &prot_ids, std::vector< PeptideIdentification > &pep_ids)
forward for old interface and pyOpenMS; use run<T>() for more control
Definition: PeptideIndexing.h:147
void addHit(const OpenMS::Size idx_pep, const OpenMS::Size idx_prot, const OpenMS::Size len_pep, const OpenMS::String &seq_prot, OpenMS::Int position)
Definition: PeptideIndexing.h:988
double getClockTime() const
ProteaseDigestion enzyme_
Definition: PeptideIndexing.h:959
void addHits_(AhoCorasickAmbiguous &fuzzyAC, const AhoCorasickAmbiguous::FuzzyACPattern &pattern, const AhoCorasickAmbiguous::PeptideDB &pep_DB, const String &prot, const String &full_prot, SignedSize idx_prot, Int offset, FoundProteinFunctor &func_threads) const
Definition: PeptideIndexing.h:1014