ArrowExport.h

Go to the documentation of this file.

// Copyright (c) 2002-present, OpenMS Inc. -- EKU Tuebingen, ETH Zurich, and FU Berlin
// SPDX-License-Identifier: BSD-3-Clause
//
// --------------------------------------------------------------------------
// $Maintainer: Timo Sachsenberg $
// $Authors: Timo Sachsenberg $
// --------------------------------------------------------------------------
 
#pragma once
 
#include <OpenMS/config.h>
 
#ifdef WITH_PARQUET
 
#include <OpenMS/CONCEPT/Types.h>
#include <OpenMS/KERNEL/MSExperiment.h>
 
#include <cstdint>
#include <memory>
#include <vector>
#include <string>
 
// Forward declarations for Arrow C Data Interface structs (opaque pointers only)
// Full definitions are in <arrow/c/abi.h>, included only in ArrowExport.cpp
struct ArrowSchema;
struct ArrowArray;
 
// Forward declarations - avoid exposing Arrow types in header
namespace arrow
{
  class Table;
}
 
namespace OpenMS
{
 
enum class ArrowExportFormat
{
  Long,      
  SemiWide   
};
 
struct OPENMS_DLLAPI ArrowSpectraExportConfig
{
  ArrowExportFormat format = ArrowExportFormat::Long;
 
  std::vector<UInt> ms_levels;
 
  double min_rt = 0;
 
  double max_rt = 0;
 
  double min_mz = 0;
 
  double max_mz = 0;
 
  std::vector<std::string> columns;
 
  bool include_precursor_info = true;
 
  bool include_ion_mobility = true;
};
 
 
struct OPENMS_DLLAPI ArrowChromatogramExportConfig
{
  ArrowExportFormat format = ArrowExportFormat::Long;
 
  double min_rt = 0;
 
  double max_rt = 0;
 
  std::vector<std::string> columns;
};
 
 
struct OPENMS_DLLAPI ParquetWriteConfig
{
  enum class Compression
  {
    NONE,       
    SNAPPY,     
    GZIP,       
    LZ4,        
    ZSTD        
  };
 
  Compression compression = Compression::ZSTD;
 
  int compression_level = 3;
 
  int64_t row_group_size = 128 * 1024 * 1024;
 
  bool write_statistics = true;
 
  int64_t data_page_size = 1024 * 1024;
};
 
 
class OPENMS_DLLAPI ArrowExport
{
public:
  static std::shared_ptr<arrow::Table> exportSpectraToArrow(
    const MSExperiment& exp,
    const ArrowSpectraExportConfig& config = ArrowSpectraExportConfig{});
 
 
  static std::vector<std::string> getSpectraArrowColumnNames(
    const MSExperiment& exp,
    const ArrowSpectraExportConfig& config = ArrowSpectraExportConfig{});
 
 
  static std::shared_ptr<arrow::Table> exportChromatogramsToArrow(
    const MSExperiment& exp,
    const ArrowChromatogramExportConfig& config = ArrowChromatogramExportConfig{});
 
 
  static std::vector<std::string> getChromatogramArrowColumnNames(
    const MSExperiment& exp,
    const ArrowChromatogramExportConfig& config = ArrowChromatogramExportConfig{});
 
 
  static bool exportSpectraToArrowCDataInterface(
    const MSExperiment& exp,
    const ArrowSpectraExportConfig& config,
    ::ArrowSchema* out_schema,
    ::ArrowArray* out_array);
 
 
  static bool exportChromatogramsToArrowCDataInterface(
    const MSExperiment& exp,
    const ArrowChromatogramExportConfig& config,
    ::ArrowSchema* out_schema,
    ::ArrowArray* out_array);
 
 
  static bool exportSpectraToParquet(
    const MSExperiment& exp,
    const String& filename,
    const ArrowSpectraExportConfig& config = ArrowSpectraExportConfig{},
    const ParquetWriteConfig& parquet_config = ParquetWriteConfig{});
 
 
  static bool exportChromatogramsToParquet(
    const MSExperiment& exp,
    const String& filename,
    const ArrowChromatogramExportConfig& config = ArrowChromatogramExportConfig{},
    const ParquetWriteConfig& parquet_config = ParquetWriteConfig{});
}; // class ArrowExport
 
} // namespace OpenMS
 
#endif // WITH_PARQUET