corems.mass_spectra.output.export
1__author__ = "Yuri E. Corilo" 2__date__ = "Dec 14, 2010" 3 4 5import csv 6import json 7import re 8import uuid 9import warnings 10from datetime import datetime, timezone 11from pathlib import Path 12 13import h5py 14import numpy as np 15import pandas as pd 16from openpyxl import load_workbook 17from pandas import DataFrame, ExcelWriter, read_excel 18 19from corems import __version__, corems_md5 20from corems.encapsulation.output import parameter_to_dict 21from corems.encapsulation.output.parameter_to_json import ( 22 dump_lcms_settings_json, 23 dump_lcms_settings_toml, 24 dump_lcms_collection_settings_json, 25 dump_lcms_collection_settings_toml, 26) 27from corems.encapsulation.constant import DEFAULT_HDF5_COMPRESSION 28from corems.mass_spectrum.output.export import ( 29 HighResMassSpecExport, 30 hdf5_dataset_kwargs, 31) 32from corems.molecular_formula.calc.MolecularFormulaCalc import MolecularFormulaCalc 33from corems.molecular_formula.factory.MolecularFormulaFactory import MolecularFormula 34from corems.molecular_id.calc.SpectralSimilarity import methods_name 35 36# Canonical location is encapsulation.constant.ION_TYPE_DICT; re-export. 37from corems.encapsulation.constant import ION_TYPE_DICT 38 39ion_type_dict = ION_TYPE_DICT 40 41 42class LowResGCMSExport: 43 """A class to export low resolution GC-MS data. 44 45 This class provides methods to export low resolution GC-MS data to various formats such as Excel, CSV, Parquet, HDF5, and Pandas DataFrame. 46 47 Parameters: 48 ---------- 49 out_file_path : str 50 The output file path. 51 gcms : object 52 The low resolution GCMS object. 53 54 Attributes: 55 ---------- 56 output_file : Path 57 The output file path as a Path object. 58 gcms : object 59 The low resolution GCMS object. 60 61 Methods: 62 ------- 63 * get_pandas_df(id_label="corems:"). Get the exported data as a Pandas DataFrame. 64 * get_json(nan=False, id_label="corems:"). Get the exported data as a JSON string. 65 * to_pandas(write_metadata=True, id_label="corems:"). Export the data to a Pandas DataFrame and save it as a pickle file. 66 * to_excel(write_mode='a', write_metadata=True, id_label="corems:"), 67 Export the data to an Excel file. 68 * to_csv(separate_output=False, write_mode="w", write_metadata=True, id_label="corems:"). 69 Export the data to a CSV file. 70 * to_parquet(write_metadata=True, id_label="corems:"). 71 Export the data to a Parquet file. 72 * to_hdf(id_label="corems:"). 73 Export the data to an HDF5 file. 74 * get_data_stats(gcms). 75 Get statistics about the GCMS data. 76 77 """ 78 79 def __init__(self, out_file_path, gcms): 80 self.output_file = Path(out_file_path) 81 82 self.gcms = gcms 83 84 self._init_columns() 85 86 def _init_columns(self): 87 """Initialize the column names for the exported data. 88 89 Returns: 90 ------- 91 list 92 The list of column names. 93 """ 94 95 columns = [ 96 "Sample name", 97 "Peak Index", 98 "Retention Time", 99 "Retention Time Ref", 100 "Peak Height", 101 "Peak Area", 102 "Retention index", 103 "Retention index Ref", 104 "Retention Index Score", 105 "Similarity Score", 106 "Spectral Similarity Score", 107 "Compound Name", 108 "Chebi ID", 109 "Kegg Compound ID", 110 "Inchi", 111 "Inchi Key", 112 "Smiles", 113 "Molecular Formula", 114 "IUPAC Name", 115 "Traditional Name", 116 "Common Name", 117 "Derivatization", 118 ] 119 120 if self.gcms.molecular_search_settings.exploratory_mode: 121 columns.extend( 122 [ 123 "Weighted Cosine Correlation", 124 "Cosine Correlation", 125 "Stein Scott Similarity", 126 "Pearson Correlation", 127 "Spearman Correlation", 128 "Kendall Tau Correlation", 129 "Euclidean Distance", 130 "Manhattan Distance", 131 "Jaccard Distance", 132 "DWT Correlation", 133 "DFT Correlation", 134 ] 135 ) 136 137 columns.extend(list(methods_name.values())) 138 139 return columns 140 141 def get_pandas_df(self, id_label="corems:"): 142 """Get the exported data as a Pandas DataFrame. 143 144 Parameters: 145 ---------- 146 id_label : str, optional 147 The ID label for the data. Default is "corems:". 148 149 Returns: 150 ------- 151 DataFrame 152 The exported data as a Pandas DataFrame. 153 """ 154 155 columns = self._init_columns() 156 157 dict_data_list = self.get_list_dict_data(self.gcms) 158 159 df = DataFrame(dict_data_list, columns=columns) 160 161 df.attrs["name"] = self.gcms.sample_name 162 163 return df 164 165 def get_json(self, nan=False, id_label="corems:"): 166 """Get the exported data as a JSON string. 167 168 Parameters: 169 ---------- 170 nan : bool, optional 171 Whether to include NaN values in the JSON string. Default is False. 172 id_label : str, optional 173 The ID label for the data. Default is "corems:". 174 175 """ 176 177 import json 178 179 dict_data_list = self.get_list_dict_data(self.gcms) 180 181 return json.dumps( 182 dict_data_list, sort_keys=False, indent=4, separators=(",", ": ") 183 ) 184 185 def to_pandas(self, write_metadata=True, id_label="corems:"): 186 """Export the data to a Pandas DataFrame and save it as a pickle file. 187 188 Parameters: 189 ---------- 190 write_metadata : bool, optional 191 Whether to write metadata to the output file. 192 id_label : str, optional 193 The ID label for the data. 194 """ 195 196 columns = self._init_columns() 197 198 dict_data_list = self.get_list_dict_data(self.gcms) 199 200 df = DataFrame(dict_data_list, columns=columns) 201 202 df.to_pickle(self.output_file.with_suffix(".pkl")) 203 204 if write_metadata: 205 self.write_settings( 206 self.output_file.with_suffix(".pkl"), self.gcms, id_label="corems:" 207 ) 208 209 def to_excel(self, write_mode="a", write_metadata=True, id_label="corems:"): 210 """Export the data to an Excel file. 211 212 Parameters: 213 ---------- 214 write_mode : str, optional 215 The write mode for the Excel file. Default is 'a' (append). 216 write_metadata : bool, optional 217 Whether to write metadata to the output file. Default is True. 218 id_label : str, optional 219 The ID label for the data. Default is "corems:". 220 """ 221 222 out_put_path = self.output_file.with_suffix(".xlsx") 223 224 columns = self._init_columns() 225 226 dict_data_list = self.get_list_dict_data(self.gcms) 227 228 df = DataFrame(dict_data_list, columns=columns) 229 230 if write_mode == "a" and out_put_path.exists(): 231 writer = ExcelWriter(out_put_path, engine="openpyxl") 232 # try to open an existing workbook 233 writer.book = load_workbook(out_put_path) 234 # copy existing sheets 235 writer.sheets = dict((ws.title, ws) for ws in writer.book.worksheets) 236 # read existing file 237 reader = read_excel(out_put_path) 238 # write out the new sheet 239 df.to_excel(writer, index=False, header=False, startrow=len(reader) + 1) 240 241 writer.close() 242 else: 243 df.to_excel( 244 self.output_file.with_suffix(".xlsx"), index=False, engine="openpyxl" 245 ) 246 247 if write_metadata: 248 self.write_settings(out_put_path, self.gcms, id_label=id_label) 249 250 def to_csv( 251 self, 252 separate_output=False, 253 write_mode="w", 254 write_metadata=True, 255 id_label="corems:", 256 ): 257 """Export the data to a CSV file. 258 259 Parameters: 260 ---------- 261 separate_output : bool, optional 262 Whether to separate the output into multiple files. Default is False. 263 write_mode : str, optional 264 The write mode for the CSV file. Default is 'w' (write). 265 write_metadata : bool, optional 266 Whether to write metadata to the output file. Default is True. 267 id_label : str, optional 268 The ID label for the data. Default is "corems:". 269 """ 270 271 if separate_output: 272 # set write mode to write 273 # this mode will overwrite the file without warning 274 write_mode = "w" 275 else: 276 # set write mode to append 277 write_mode = "a" 278 279 columns = self._init_columns() 280 281 dict_data_list = self.get_list_dict_data(self.gcms) 282 283 out_put_path = self.output_file.with_suffix(".csv") 284 285 write_header = not out_put_path.exists() 286 287 try: 288 with open(out_put_path, write_mode, newline="") as csvfile: 289 writer = csv.DictWriter(csvfile, fieldnames=columns) 290 if write_header: 291 writer.writeheader() 292 for data in dict_data_list: 293 writer.writerow(data) 294 295 if write_metadata: 296 self.write_settings(out_put_path, self.gcms, id_label=id_label) 297 298 except IOError as ioerror: 299 print(ioerror) 300 301 def to_parquet(self, write_metadata=True, id_label="corems:"): 302 """Export the data to a Parquet file. 303 304 Parameters: 305 ---------- 306 write_metadata : bool, optional 307 Whether to write metadata to the output file. Default is True. 308 id_label : str, optional 309 The ID label for the data. Default is "corems:". 310 311 Notes 312 ----- 313 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 314 """ 315 316 columns = self._init_columns() 317 318 dict_data_list = self.get_list_dict_data(self.gcms) 319 320 df = DataFrame(dict_data_list, columns=columns) 321 322 out_put_path = self.output_file.with_suffix(".parquet") 323 df.to_parquet(out_put_path, index=False) 324 325 if write_metadata: 326 self.write_settings(out_put_path, self.gcms, id_label=id_label) 327 328 def to_hdf(self, id_label="corems:"): 329 """Export the data to an HDF5 file. 330 331 Parameters: 332 ---------- 333 id_label : str, optional 334 The ID label for the data. Default is "corems:". 335 """ 336 337 # save sample at a time 338 def add_compound(gc_peak, compound_obj): 339 modifier = compound_obj.classify if compound_obj.classify else "" 340 compound_group = compound_obj.name.replace("/", "") + " " + modifier 341 342 if compound_group not in peak_group: 343 compound_group = peak_group.create_group(compound_group) 344 345 # compound_group.attrs["retention_time"] = compound_obj.retention_time 346 compound_group.attrs["retention_index"] = compound_obj.ri 347 compound_group.attrs["retention_index_score"] = compound_obj.ri_score 348 compound_group.attrs["spectral_similarity_score"] = ( 349 compound_obj.spectral_similarity_score 350 ) 351 compound_group.attrs["similarity_score"] = compound_obj.similarity_score 352 353 compond_mz = compound_group.create_dataset( 354 "mz", data=np.array(compound_obj.mz), dtype="f8" 355 ) 356 compond_abundance = compound_group.create_dataset( 357 "abundance", data=np.array(compound_obj.abundance), dtype="f8" 358 ) 359 360 if self.gcms.molecular_search_settings.exploratory_mode: 361 compound_group.attrs["Spectral Similarities"] = json.dumps( 362 compound_obj.spectral_similarity_scores, 363 sort_keys=False, 364 indent=4, 365 separators=(",", ":"), 366 ) 367 else: 368 warnings.warn("Skipping duplicate reference compound.") 369 370 import json 371 from datetime import datetime, timezone 372 373 import h5py 374 import numpy as np 375 376 output_path = self.output_file.with_suffix(".hdf5") 377 378 with h5py.File(output_path, "w") as hdf_handle: 379 timenow = str(datetime.now(timezone.utc).strftime("%d/%m/%Y %H:%M:%S %Z")) 380 hdf_handle.attrs["time_stamp"] = timenow 381 hdf_handle.attrs["data_structure"] = "gcms" 382 hdf_handle.attrs["analyzer"] = self.gcms.analyzer 383 hdf_handle.attrs["instrument_label"] = self.gcms.instrument_label 384 385 hdf_handle.attrs["sample_id"] = "self.gcms.id" 386 hdf_handle.attrs["sample_name"] = self.gcms.sample_name 387 hdf_handle.attrs["input_data"] = str(self.gcms.file_location) 388 hdf_handle.attrs["output_data"] = str(output_path) 389 hdf_handle.attrs["output_data_id"] = id_label + uuid.uuid4().hex 390 hdf_handle.attrs["corems_version"] = __version__ 391 392 hdf_handle.attrs["Stats"] = json.dumps( 393 self.get_data_stats(self.gcms), 394 sort_keys=False, 395 indent=4, 396 separators=(",", ": "), 397 ) 398 hdf_handle.attrs["Calibration"] = json.dumps( 399 self.get_calibration_stats(self.gcms, id_label), 400 sort_keys=False, 401 indent=4, 402 separators=(",", ": "), 403 ) 404 hdf_handle.attrs["Blank"] = json.dumps( 405 self.get_blank_stats(self.gcms), 406 sort_keys=False, 407 indent=4, 408 separators=(",", ": "), 409 ) 410 411 corems_dict_setting = parameter_to_dict.get_dict_data_gcms(self.gcms) 412 hdf_handle.attrs["CoreMSParameters"] = json.dumps( 413 corems_dict_setting, sort_keys=False, indent=4, separators=(",", ": ") 414 ) 415 416 scans_dataset = hdf_handle.create_dataset( 417 "scans", data=np.array(self.gcms.scans_number), dtype="f8" 418 ) 419 rt_dataset = hdf_handle.create_dataset( 420 "rt", data=np.array(self.gcms.retention_time), dtype="f8" 421 ) 422 tic_dataset = hdf_handle.create_dataset( 423 "tic", data=np.array(self.gcms.tic), dtype="f8" 424 ) 425 processed_tic_dataset = hdf_handle.create_dataset( 426 "processed_tic", data=np.array(self.gcms.processed_tic), dtype="f8" 427 ) 428 429 output_score_method = ( 430 self.gcms.molecular_search_settings.output_score_method 431 ) 432 433 for gc_peak in self.gcms: 434 # print(gc_peak.retention_time) 435 # print(gc_peak.tic) 436 437 # check if there is a compound candidate 438 peak_group = hdf_handle.create_group(str(gc_peak.retention_time)) 439 peak_group.attrs["deconvolution"] = int( 440 self.gcms.chromatogram_settings.use_deconvolution 441 ) 442 443 peak_group.attrs["start_scan"] = gc_peak.start_scan 444 peak_group.attrs["apex_scan"] = gc_peak.apex_scan 445 peak_group.attrs["final_scan"] = gc_peak.final_scan 446 447 peak_group.attrs["retention_index"] = gc_peak.ri 448 peak_group.attrs["retention_time"] = gc_peak.retention_time 449 peak_group.attrs["area"] = gc_peak.area 450 451 mz = peak_group.create_dataset( 452 "mz", data=np.array(gc_peak.mass_spectrum.mz_exp), dtype="f8" 453 ) 454 abundance = peak_group.create_dataset( 455 "abundance", 456 data=np.array(gc_peak.mass_spectrum.abundance), 457 dtype="f8", 458 ) 459 460 if gc_peak: 461 if output_score_method == "highest_sim_score": 462 compound_obj = gc_peak.highest_score_compound 463 add_compound(gc_peak, compound_obj) 464 465 elif output_score_method == "highest_ss": 466 compound_obj = gc_peak.highest_ss_compound 467 add_compound(gc_peak, compound_obj) 468 469 else: 470 for compound_obj in gc_peak: 471 add_compound(gc_peak, compound_obj) 472 473 def get_data_stats(self, gcms): 474 """Get statistics about the GCMS data. 475 476 Parameters: 477 ---------- 478 gcms : object 479 The low resolution GCMS object. 480 481 Returns: 482 ------- 483 dict 484 A dictionary containing the data statistics. 485 """ 486 487 matched_peaks = gcms.matched_peaks 488 no_matched_peaks = gcms.no_matched_peaks 489 unique_metabolites = gcms.unique_metabolites 490 491 peak_matchs_above_0p85 = 0 492 unique_peak_match_above_0p85 = 0 493 for match_peak in matched_peaks: 494 gc_peak_above_85 = 0 495 matches_above_85 = list( 496 filter(lambda m: m.similarity_score >= 0.85, match_peak) 497 ) 498 if matches_above_85: 499 peak_matchs_above_0p85 += 1 500 if len(matches_above_85) == 1: 501 unique_peak_match_above_0p85 += 1 502 503 data_stats = {} 504 data_stats["average_signal_noise"] = "ni" 505 data_stats["chromatogram_dynamic_range"] = gcms.dynamic_range 506 data_stats["total_number_peaks"] = len(gcms) 507 data_stats["total_peaks_matched"] = len(matched_peaks) 508 data_stats["total_peaks_without_matches"] = len(no_matched_peaks) 509 data_stats["total_matches_above_similarity_score_0.85"] = peak_matchs_above_0p85 510 data_stats["single_matches_above_similarity_score_0.85"] = ( 511 unique_peak_match_above_0p85 512 ) 513 data_stats["unique_metabolites"] = len(unique_metabolites) 514 515 return data_stats 516 517 def get_calibration_stats(self, gcms, id_label): 518 """Get statistics about the GC-MS calibration. 519 520 Parameters: 521 ---------- 522 """ 523 calibration_parameters = {} 524 525 calibration_parameters["calibration_rt_ri_pairs_ref"] = gcms.ri_pairs_ref 526 calibration_parameters["data_url"] = str(gcms.cal_file_path) 527 calibration_parameters["has_input"] = id_label + corems_md5(gcms.cal_file_path) 528 calibration_parameters["data_name"] = str(gcms.cal_file_path.stem) 529 calibration_parameters["calibration_method"] = "" 530 531 return calibration_parameters 532 533 def get_blank_stats(self, gcms): 534 """Get statistics about the GC-MS blank.""" 535 blank_parameters = {} 536 537 blank_parameters["data_name"] = "ni" 538 blank_parameters["blank_id"] = "ni" 539 blank_parameters["data_url"] = "ni" 540 blank_parameters["has_input"] = "ni" 541 blank_parameters["common_features_to_blank"] = "ni" 542 543 return blank_parameters 544 545 def get_instrument_metadata(self, gcms): 546 """Get metadata about the GC-MS instrument.""" 547 instrument_metadata = {} 548 549 instrument_metadata["analyzer"] = gcms.analyzer 550 instrument_metadata["instrument_label"] = gcms.instrument_label 551 instrument_metadata["instrument_id"] = uuid.uuid4().hex 552 553 return instrument_metadata 554 555 def get_data_metadata(self, gcms, id_label, output_path): 556 """Get metadata about the GC-MS data. 557 558 Parameters: 559 ---------- 560 gcms : object 561 The low resolution GCMS object. 562 id_label : str 563 The ID label for the data. 564 output_path : str 565 The output file path. 566 567 Returns: 568 ------- 569 dict 570 A dictionary containing the data metadata. 571 """ 572 if isinstance(output_path, str): 573 output_path = Path(output_path) 574 575 paramaters_path = output_path.with_suffix(".json") 576 577 if paramaters_path.exists(): 578 with paramaters_path.open() as current_param: 579 metadata = json.load(current_param) 580 data_metadata = metadata.get("Data") 581 else: 582 data_metadata = {} 583 data_metadata["data_name"] = [] 584 data_metadata["input_data_url"] = [] 585 data_metadata["has_input"] = [] 586 587 data_metadata["data_name"].append(gcms.sample_name) 588 data_metadata["input_data_url"].append(str(gcms.file_location)) 589 data_metadata["has_input"].append(id_label + corems_md5(gcms.file_location)) 590 591 data_metadata["output_data_name"] = str(output_path.stem) 592 data_metadata["output_data_url"] = str(output_path) 593 data_metadata["has_output"] = id_label + corems_md5(output_path) 594 595 return data_metadata 596 597 def get_parameters_json(self, gcms, id_label, output_path): 598 """Get the parameters as a JSON string. 599 600 Parameters: 601 ---------- 602 gcms : GCMS object 603 The low resolution GCMS object. 604 id_label : str 605 The ID label for the data. 606 output_path : str 607 The output file path. 608 609 Returns: 610 ------- 611 str 612 The parameters as a JSON string. 613 """ 614 615 output_parameters_dict = {} 616 output_parameters_dict["Data"] = self.get_data_metadata( 617 gcms, id_label, output_path 618 ) 619 output_parameters_dict["Stats"] = self.get_data_stats(gcms) 620 output_parameters_dict["Calibration"] = self.get_calibration_stats( 621 gcms, id_label 622 ) 623 output_parameters_dict["Blank"] = self.get_blank_stats(gcms) 624 output_parameters_dict["Instrument"] = self.get_instrument_metadata(gcms) 625 corems_dict_setting = parameter_to_dict.get_dict_data_gcms(gcms) 626 corems_dict_setting["corems_version"] = __version__ 627 output_parameters_dict["CoreMSParameters"] = corems_dict_setting 628 output_parameters_dict["has_metabolite"] = gcms.metabolites_data 629 output = json.dumps( 630 output_parameters_dict, sort_keys=False, indent=4, separators=(",", ": ") 631 ) 632 633 return output 634 635 def write_settings(self, output_path, gcms, id_label="emsl:"): 636 """Write the settings to a JSON file. 637 638 Parameters: 639 ---------- 640 output_path : str 641 The output file path. 642 gcms : GCMS object 643 The low resolution GCMS object. 644 id_label : str 645 The ID label for the data. Default is "emsl:". 646 647 """ 648 649 output = self.get_parameters_json(gcms, id_label, output_path) 650 651 with open( 652 output_path.with_suffix(".json"), 653 "w", 654 encoding="utf8", 655 ) as outfile: 656 outfile.write(output) 657 658 def get_list_dict_data(self, gcms, include_no_match=True, no_match_inline=False): 659 """Get the exported data as a list of dictionaries. 660 661 Parameters: 662 ---------- 663 gcms : object 664 The low resolution GCMS object. 665 include_no_match : bool, optional 666 Whether to include no match data. Default is True. 667 no_match_inline : bool, optional 668 Whether to include no match data inline. Default is False. 669 670 Returns: 671 ------- 672 list 673 The exported data as a list of dictionaries. 674 """ 675 676 output_score_method = gcms.molecular_search_settings.output_score_method 677 678 dict_data_list = [] 679 680 def add_match_dict_data(): 681 derivatization = "{}:{}:{}".format( 682 compound_obj.classify, 683 compound_obj.derivativenum, 684 compound_obj.derivatization, 685 ) 686 out_dict = { 687 "Sample name": gcms.sample_name, 688 "Peak Index": gcpeak_index, 689 "Retention Time": gc_peak.retention_time, 690 "Retention Time Ref": compound_obj.retention_time, 691 "Peak Height": gc_peak.tic, 692 "Peak Area": gc_peak.area, 693 "Retention index": gc_peak.ri, 694 "Retention index Ref": compound_obj.ri, 695 "Retention Index Score": compound_obj.ri_score, 696 "Spectral Similarity Score": compound_obj.spectral_similarity_score, 697 "Similarity Score": compound_obj.similarity_score, 698 "Compound Name": compound_obj.name, 699 "Chebi ID": compound_obj.metadata.chebi, 700 "Kegg Compound ID": compound_obj.metadata.kegg, 701 "Inchi": compound_obj.metadata.inchi, 702 "Inchi Key": compound_obj.metadata.inchikey, 703 "Smiles": compound_obj.metadata.smiles, 704 "Molecular Formula": compound_obj.formula, 705 "IUPAC Name": compound_obj.metadata.iupac_name, 706 "Traditional Name": compound_obj.metadata.traditional_name, 707 "Common Name": compound_obj.metadata.common_name, 708 "Derivatization": derivatization, 709 } 710 711 if self.gcms.molecular_search_settings.exploratory_mode: 712 out_dict.update( 713 { 714 "Weighted Cosine Correlation": compound_obj.spectral_similarity_scores.get( 715 "weighted_cosine_correlation" 716 ), 717 "Cosine Correlation": compound_obj.spectral_similarity_scores.get( 718 "cosine_correlation" 719 ), 720 "Stein Scott Similarity": compound_obj.spectral_similarity_scores.get( 721 "stein_scott_similarity" 722 ), 723 "Pearson Correlation": compound_obj.spectral_similarity_scores.get( 724 "pearson_correlation" 725 ), 726 "Spearman Correlation": compound_obj.spectral_similarity_scores.get( 727 "spearman_correlation" 728 ), 729 "Kendall Tau Correlation": compound_obj.spectral_similarity_scores.get( 730 "kendall_tau_correlation" 731 ), 732 "DFT Correlation": compound_obj.spectral_similarity_scores.get( 733 "dft_correlation" 734 ), 735 "DWT Correlation": compound_obj.spectral_similarity_scores.get( 736 "dwt_correlation" 737 ), 738 "Euclidean Distance": compound_obj.spectral_similarity_scores.get( 739 "euclidean_distance" 740 ), 741 "Manhattan Distance": compound_obj.spectral_similarity_scores.get( 742 "manhattan_distance" 743 ), 744 "Jaccard Distance": compound_obj.spectral_similarity_scores.get( 745 "jaccard_distance" 746 ), 747 } 748 ) 749 for method in methods_name: 750 out_dict[methods_name.get(method)] = ( 751 compound_obj.spectral_similarity_scores.get(method) 752 ) 753 754 dict_data_list.append(out_dict) 755 756 def add_no_match_dict_data(): 757 dict_data_list.append( 758 { 759 "Sample name": gcms.sample_name, 760 "Peak Index": gcpeak_index, 761 "Retention Time": gc_peak.retention_time, 762 "Peak Height": gc_peak.tic, 763 "Peak Area": gc_peak.area, 764 "Retention index": gc_peak.ri, 765 } 766 ) 767 768 for gcpeak_index, gc_peak in enumerate(gcms.sorted_gcpeaks): 769 # check if there is a compound candidate 770 if gc_peak: 771 if output_score_method == "highest_sim_score": 772 compound_obj = gc_peak.highest_score_compound 773 add_match_dict_data() 774 775 elif output_score_method == "highest_ss": 776 compound_obj = gc_peak.highest_ss_compound 777 add_match_dict_data() 778 779 else: 780 for compound_obj in gc_peak: 781 add_match_dict_data() # add monoisotopic peak 782 783 else: 784 # include not_match 785 if include_no_match and no_match_inline: 786 add_no_match_dict_data() 787 788 if include_no_match and not no_match_inline: 789 for gcpeak_index, gc_peak in enumerate(gcms.sorted_gcpeaks): 790 if not gc_peak: 791 add_no_match_dict_data() 792 793 return dict_data_list 794 795 796class HighResMassSpectraExport(HighResMassSpecExport): 797 """A class to export high resolution mass spectra data. 798 799 This class provides methods to export high resolution mass spectra data to various formats 800 such as Excel, CSV, Parquet, HDF5, and Pandas DataFrame. 801 802 Parameters 803 ---------- 804 out_file_path : str | Path 805 The output file path. 806 mass_spectra : object 807 The high resolution mass spectra object. 808 output_type : str, optional 809 The output type. Default is 'excel'. 810 811 Attributes 812 ---------- 813 output_file : Path 814 The output file path without suffix 815 dir_loc : Path 816 The directory location for the output file, 817 by default this will be the output_file + ".corems" and all output files will be 818 written into this location 819 mass_spectra : MassSpectraBase 820 The high resolution mass spectra object. 821 """ 822 823 def __init__(self, out_file_path, mass_spectra, output_type="excel"): 824 super().__init__( 825 out_file_path=out_file_path, mass_spectrum=None, output_type=output_type 826 ) 827 828 self.dir_loc = Path(out_file_path + ".corems") 829 self.dir_loc.mkdir(exist_ok=True) 830 # Place the output file in the directory 831 self.output_file = self.dir_loc / Path(out_file_path).name 832 self._output_type = output_type # 'excel', 'csv', 'pandas', 'parquet' or 'hdf5' 833 self.mass_spectra = mass_spectra 834 self.atoms_order_list = None 835 self._init_columns() 836 837 def get_pandas_df(self): 838 """Get the mass spectra as a list of Pandas DataFrames.""" 839 840 list_df = [] 841 842 for mass_spectrum in self.mass_spectra: 843 columns = self.columns_label + self.get_all_used_atoms_in_order( 844 mass_spectrum 845 ) 846 847 dict_data_list = self.get_list_dict_data(mass_spectrum) 848 849 df = DataFrame(dict_data_list, columns=columns) 850 851 scan_number = mass_spectrum.scan_number 852 853 df.attrs["name"] = str(self.output_file) + "_" + str(scan_number) 854 855 list_df.append(df) 856 857 return list_df 858 859 def to_pandas(self, write_metadata=True): 860 """Export the data to a Pandas DataFrame and save it as a pickle file. 861 862 Parameters: 863 ---------- 864 write_metadata : bool, optional 865 Whether to write metadata to the output file. Default is True. 866 """ 867 868 for mass_spectrum in self.mass_spectra: 869 columns = self.columns_label + self.get_all_used_atoms_in_order( 870 mass_spectrum 871 ) 872 873 dict_data_list = self.get_list_dict_data(mass_spectrum) 874 875 df = DataFrame(dict_data_list, columns=columns) 876 877 scan_number = mass_spectrum.scan_number 878 879 out_filename = Path( 880 "%s_scan%s%s" % (self.output_file, str(scan_number), ".pkl") 881 ) 882 883 df.to_pickle(self.dir_loc / out_filename) 884 885 if write_metadata: 886 self.write_settings( 887 self.dir_loc / out_filename.with_suffix(""), mass_spectrum 888 ) 889 890 def to_excel(self, write_metadata=True): 891 """Export the data to an Excel file. 892 893 Parameters: 894 ---------- 895 write_metadata : bool, optional 896 Whether to write metadata to the output file. Default is True. 897 """ 898 for mass_spectrum in self.mass_spectra: 899 columns = self.columns_label + self.get_all_used_atoms_in_order( 900 mass_spectrum 901 ) 902 903 dict_data_list = self.get_list_dict_data(mass_spectrum) 904 905 df = DataFrame(dict_data_list, columns=columns) 906 907 scan_number = mass_spectrum.scan_number 908 909 out_filename = Path( 910 "%s_scan%s%s" % (self.output_file, str(scan_number), ".xlsx") 911 ) 912 913 df.to_excel(self.dir_loc / out_filename) 914 915 if write_metadata: 916 self.write_settings( 917 self.dir_loc / out_filename.with_suffix(""), mass_spectrum 918 ) 919 920 def to_csv(self, write_metadata=True): 921 """Export the data to a CSV file. 922 923 Parameters: 924 ---------- 925 write_metadata : bool, optional 926 Whether to write metadata to the output file. Default is True. 927 """ 928 import csv 929 930 for mass_spectrum in self.mass_spectra: 931 columns = self.columns_label + self.get_all_used_atoms_in_order( 932 mass_spectrum 933 ) 934 935 scan_number = mass_spectrum.scan_number 936 937 dict_data_list = self.get_list_dict_data(mass_spectrum) 938 939 out_filename = Path( 940 "%s_scan%s%s" % (self.output_file, str(scan_number), ".csv") 941 ) 942 943 with open(self.dir_loc / out_filename, "w", newline="") as csvfile: 944 writer = csv.DictWriter(csvfile, fieldnames=columns) 945 writer.writeheader() 946 for data in dict_data_list: 947 writer.writerow(data) 948 949 if write_metadata: 950 self.write_settings( 951 self.dir_loc / out_filename.with_suffix(""), mass_spectrum 952 ) 953 954 def to_parquet(self, write_metadata=True): 955 """Export the data to a Parquet file. 956 957 Parameters: 958 ---------- 959 write_metadata : bool, optional 960 Whether to write metadata to the output file. Default is True. 961 962 Notes 963 ----- 964 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 965 """ 966 for mass_spectrum in self.mass_spectra: 967 columns = self.columns_label + self.get_all_used_atoms_in_order( 968 mass_spectrum 969 ) 970 971 dict_data_list = self.get_list_dict_data(mass_spectrum) 972 973 df = DataFrame(dict_data_list, columns=columns) 974 975 scan_number = mass_spectrum.scan_number 976 977 out_filename = Path( 978 "%s_scan%s%s" % (self.output_file, str(scan_number), ".parquet") 979 ) 980 981 df.to_parquet(self.dir_loc / out_filename, index=False) 982 983 if write_metadata: 984 self.write_settings( 985 self.dir_loc / out_filename.with_suffix(""), mass_spectrum 986 ) 987 988 def get_mass_spectra_attrs(self): 989 """Get the mass spectra attributes as a JSON string. 990 991 Parameters: 992 ---------- 993 mass_spectra : object 994 The high resolution mass spectra object. 995 996 Returns: 997 ------- 998 str 999 The mass spectra attributes as a JSON string. 1000 """ 1001 dict_ms_attrs = {} 1002 dict_ms_attrs["analyzer"] = self.mass_spectra.analyzer 1003 dict_ms_attrs["instrument_label"] = self.mass_spectra.instrument_label 1004 dict_ms_attrs["sample_name"] = self.mass_spectra.sample_name 1005 1006 return json.dumps( 1007 dict_ms_attrs, sort_keys=False, indent=4, separators=(",", ": ") 1008 ) 1009 1010 def to_hdf( 1011 self, 1012 overwrite=False, 1013 export_raw=True, 1014 compression=DEFAULT_HDF5_COMPRESSION, 1015 profile_dtype="f4", 1016 ): 1017 """Export the data to an HDF5 file. 1018 1019 Parameters 1020 ---------- 1021 overwrite : bool, optional 1022 Whether to overwrite the output file. Default is False. 1023 export_raw : bool, optional 1024 Whether to export the raw mass spectra data. Default is True. 1025 compression : int, str or None, optional 1026 Compression for the written datasets: a gzip level 0-9, 1027 ``"lzf"``, or ``None``/``"none"``. Default is 4. ``"lzf"`` is 1028 h5py's own filter, not a standard HDF5 one -- it needs h5py (or 1029 a tool with the lzf plugin) to read the result back. 1030 profile_dtype : str, optional 1031 dtype used for the stored profile m/z/abundance arrays 1032 (``"f4"`` or ``"f8"``). Default is ``"f4"``. 1033 """ 1034 if overwrite: 1035 if self.output_file.with_suffix(".hdf5").exists(): 1036 self.output_file.with_suffix(".hdf5").unlink() 1037 1038 with h5py.File(self.output_file.with_suffix(".hdf5"), "a") as hdf_handle: 1039 if not hdf_handle.attrs.get("date_utc"): 1040 # Set metadata for all mass spectra 1041 timenow = str( 1042 datetime.now(timezone.utc).strftime("%d/%m/%Y %H:%M:%S %Z") 1043 ) 1044 hdf_handle.attrs["date_utc"] = timenow 1045 hdf_handle.attrs["filename"] = self.mass_spectra.file_location.name 1046 hdf_handle.attrs["data_structure"] = "mass_spectra" 1047 hdf_handle.attrs["analyzer"] = self.mass_spectra.analyzer 1048 hdf_handle.attrs["instrument_label"] = ( 1049 self.mass_spectra.instrument_label 1050 ) 1051 hdf_handle.attrs["sample_name"] = self.mass_spectra.sample_name 1052 hdf_handle.attrs["polarity"] = self.mass_spectra.polarity 1053 hdf_handle.attrs["parser_type"] = ( 1054 self.mass_spectra.spectra_parser_class.__name__ 1055 ) 1056 hdf_handle.attrs["original_file_location"] = ( 1057 self.mass_spectra.file_location._str 1058 ) 1059 1060 # Save creation time from the cached parser when one is configured. 1061 try: 1062 if ( 1063 getattr(self.mass_spectra, "spectra_parser_class", None) 1064 is not None 1065 ): 1066 parser = self.mass_spectra.spectra_parser 1067 if parser is not None and hasattr(parser, "get_creation_time"): 1068 creation_time = parser.get_creation_time() 1069 if creation_time is not None: 1070 hdf_handle.attrs["creation_time"] = ( 1071 creation_time.isoformat() 1072 ) 1073 except Exception: 1074 pass # If creation time cannot be retrieved, skip it 1075 1076 if "mass_spectra" not in hdf_handle: 1077 mass_spectra_group = hdf_handle.create_group("mass_spectra") 1078 else: 1079 mass_spectra_group = hdf_handle.get("mass_spectra") 1080 1081 for mass_spectrum in self.mass_spectra: 1082 group_key = str(int(mass_spectrum.scan_number)) 1083 1084 self.add_mass_spectrum_to_hdf5( 1085 hdf_handle, 1086 mass_spectrum, 1087 group_key, 1088 mass_spectra_group, 1089 export_raw, 1090 compression=compression, 1091 profile_dtype=profile_dtype, 1092 ) 1093 1094 1095class LCMSExport(HighResMassSpectraExport): 1096 """A class to export high resolution LC-MS data. 1097 1098 This class provides methods to export high resolution LC-MS data to HDF5 1099 and Parquet (tabular datasets that mirror the main HDF5 groups). 1100 1101 Parameters 1102 ---------- 1103 out_file_path : str | Path 1104 The output file path, do not include the file extension. 1105 lcms_object : LCMSBase 1106 The high resolution lc-ms object. 1107 """ 1108 1109 def __init__(self, out_file_path, mass_spectra): 1110 super().__init__(out_file_path, mass_spectra, output_type="hdf5") 1111 1112 @staticmethod 1113 def _prepare_df_for_parquet(df): 1114 """Make a DataFrame Parquet-safe by normalizing nested/mixed object values. 1115 1116 Parameters 1117 ---------- 1118 df : DataFrame 1119 Input dataframe. 1120 1121 Returns 1122 ------- 1123 DataFrame 1124 Copy suitable for ``DataFrame.to_parquet``. 1125 1126 Notes 1127 ----- 1128 Object columns with mixed Python types (especially ``bool`` mixed with 1129 ``int``) fail under pyarrow. Nested containers are JSON-serialized; 1130 mixed scalar object columns are coerced to a single type or strings. 1131 """ 1132 if df is None or df.empty: 1133 return df 1134 1135 out = df.copy() 1136 out.columns = [str(c) for c in out.columns] 1137 1138 for col in out.columns: 1139 series = out[col] 1140 if series.dtype != object: 1141 continue 1142 1143 def _normalize(value): 1144 if value is None: 1145 return None 1146 try: 1147 # Avoid treating array-likes as NA; only scalar NA checks 1148 if not isinstance( 1149 value, (list, tuple, dict, np.ndarray) 1150 ) and pd.isna(value): 1151 return None 1152 except (TypeError, ValueError): 1153 pass 1154 if isinstance(value, (list, tuple, dict)): 1155 return json.dumps(value) 1156 if isinstance(value, np.ndarray): 1157 return json.dumps(value.tolist()) 1158 if isinstance(value, (bool, np.bool_)): 1159 return bool(value) 1160 if isinstance(value, (np.integer,)): 1161 return int(value) 1162 if isinstance(value, (np.floating,)): 1163 return float(value) 1164 return value 1165 1166 normalized = series.map(_normalize) 1167 non_null = [v for v in normalized.tolist() if v is not None] 1168 1169 has_bool = any(isinstance(v, bool) for v in non_null) 1170 # bool is a subclass of int; exclude bools from int detection 1171 has_int = any( 1172 isinstance(v, int) and not isinstance(v, bool) for v in non_null 1173 ) 1174 has_float = any(isinstance(v, float) for v in non_null) 1175 has_str = any(isinstance(v, str) for v in non_null) 1176 has_other = any( 1177 not isinstance(v, (bool, int, float, str)) for v in non_null 1178 ) 1179 1180 if ( 1181 has_other 1182 or (has_str and (has_bool or has_int or has_float)) 1183 or (has_bool and has_int) 1184 ): 1185 out[col] = normalized.map(lambda v: None if v is None else str(v)) 1186 elif has_bool and not has_float: 1187 out[col] = normalized.map( 1188 lambda v: None if v is None else bool(v) 1189 ).astype("boolean") 1190 elif has_float or has_int: 1191 out[col] = pd.to_numeric(pd.Series(normalized), errors="coerce") 1192 else: 1193 out[col] = normalized.map(lambda v: None if v is None else str(v)) 1194 1195 return out 1196 1197 def _write_parquet(self, df, filename, overwrite=False): 1198 """Write a dataframe to a Parquet file under the export directory. 1199 1200 Parameters 1201 ---------- 1202 df : DataFrame 1203 Data to write. 1204 filename : str 1205 File name (with or without ``.parquet`` suffix). 1206 overwrite : bool, optional 1207 Whether to replace an existing file. Default is False. 1208 1209 Returns 1210 ------- 1211 Path or None 1212 Output path if written, otherwise None. 1213 """ 1214 if df is None or df.empty: 1215 return None 1216 1217 out_path = self.dir_loc / Path(filename).with_suffix(".parquet") 1218 if out_path.exists() and not overwrite: 1219 return out_path 1220 1221 self._prepare_df_for_parquet(df).to_parquet(out_path, index=False) 1222 return out_path 1223 1224 def _mass_spectra_to_parquet_df(self): 1225 """Build a single DataFrame of peak tables for all loaded mass spectra.""" 1226 frames = [] 1227 for mass_spectrum in self.mass_spectra: 1228 if mass_spectrum is None: 1229 continue 1230 columns = self.columns_label + self.get_all_used_atoms_in_order( 1231 mass_spectrum 1232 ) 1233 dict_data_list = self.get_list_dict_data(mass_spectrum) 1234 df = DataFrame(dict_data_list, columns=columns) 1235 if df.empty: 1236 continue 1237 df.insert(0, "scan_number", mass_spectrum.scan_number) 1238 if getattr(mass_spectrum, "retention_time", None) is not None: 1239 df.insert(1, "retention_time", mass_spectrum.retention_time) 1240 frames.append(df) 1241 1242 if not frames: 1243 return DataFrame() 1244 return pd.concat(frames, ignore_index=True) 1245 1246 @staticmethod 1247 def _sequence_len(value): 1248 """Return length of a sequence/array, or 0 for None/empty/scalar-missing.""" 1249 if value is None: 1250 return 0 1251 try: 1252 return len(value) 1253 except TypeError: 1254 return 0 1255 1256 def _eics_to_parquet_df(self): 1257 """Build a long-format DataFrame of EIC time series.""" 1258 frames = [] 1259 for mz, eic_data in self.mass_spectra.eics.items(): 1260 if eic_data is None: 1261 continue 1262 # Avoid `array or []` — numpy arrays are ambiguous in boolean context 1263 time = getattr(eic_data, "time", None) 1264 n = self._sequence_len(time) 1265 if n == 0: 1266 continue 1267 1268 scans = getattr(eic_data, "scans", None) 1269 eic = getattr(eic_data, "eic", None) 1270 eic_smoothed = getattr(eic_data, "eic_smoothed", None) 1271 1272 frame = { 1273 "mz": [mz] * n, 1274 "scans": list(scans) if self._sequence_len(scans) == n else [None] * n, 1275 "time": list(time), 1276 "eic": list(eic) if self._sequence_len(eic) == n else [None] * n, 1277 } 1278 if self._sequence_len(eic_smoothed) == n: 1279 frame["eic_smoothed"] = list(eic_smoothed) 1280 frames.append(DataFrame(frame)) 1281 1282 if not frames: 1283 return DataFrame() 1284 return pd.concat(frames, ignore_index=True) 1285 1286 def _spectral_search_results_to_parquet_df(self): 1287 """Flatten spectral search results into a single DataFrame.""" 1288 frames = [] 1289 for scan_id, by_precursor in self.mass_spectra.spectral_search_results.items(): 1290 for precursor_key, result in by_precursor.items(): 1291 try: 1292 df = result.to_dataframe() 1293 except Exception: 1294 continue 1295 if df is None or df.empty: 1296 continue 1297 df = df.copy() 1298 # to_dataframe() already includes these fields from the result object; 1299 # only set them when missing so we do not insert duplicate columns. 1300 if "query_spectrum_id" not in df.columns: 1301 df.insert( 1302 0, 1303 "query_spectrum_id", 1304 getattr(result, "query_spectrum_id", scan_id), 1305 ) 1306 if "precursor_mz" not in df.columns: 1307 loc = 1 if "query_spectrum_id" in df.columns else 0 1308 df.insert( 1309 loc, 1310 "precursor_mz", 1311 getattr(result, "precursor_mz", precursor_key), 1312 ) 1313 frames.append(df) 1314 1315 if not frames: 1316 return DataFrame() 1317 return pd.concat(frames, ignore_index=True) 1318 1319 def to_parquet( 1320 self, 1321 overwrite=False, 1322 save_parameters=True, 1323 parameter_format="toml", 1324 export_spectra=True, 1325 ): 1326 """Export LC-MS datasets to Parquet files (tabular mirror of ``to_hdf``). 1327 1328 Writes under ``{out_file_path}.corems/``: 1329 1330 - ``scan_info.parquet`` 1331 - ``mass_features.parquet`` 1332 - ``induced_mass_features.parquet`` (if present) 1333 - ``mass_spectra.parquet`` (optional; all peak tables with scan metadata) 1334 - ``eics.parquet`` (if EICs are present and export is enabled) 1335 - ``spectral_search_results.parquet`` (if search results are present) 1336 - parameters as JSON or TOML when ``save_parameters`` is True 1337 1338 Parameters 1339 ---------- 1340 overwrite : bool, optional 1341 Whether to overwrite existing Parquet files. Default is False. 1342 save_parameters : bool, optional 1343 Whether to save processing parameters. Default is True. 1344 parameter_format : str, optional 1345 Parameter file format, ``'json'`` or ``'toml'``. Default is ``'toml'``. 1346 export_spectra : bool, optional 1347 Whether to export loaded mass-spectrum peak tables. Default is True. 1348 1349 Notes 1350 ----- 1351 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 1352 Raw profile arrays and nested HDF5-only blobs are not reproduced in full; 1353 use ``to_hdf`` for a complete self-contained archive. 1354 """ 1355 self.dir_loc.mkdir(exist_ok=True) 1356 1357 # Scan metadata (same source as the HDF5 scan_info group) 1358 scan_df = self.mass_spectra.scan_df 1359 if scan_df is not None and not scan_df.empty: 1360 self._write_parquet( 1361 scan_df.reset_index(drop=True), "scan_info", overwrite=overwrite 1362 ) 1363 1364 # Mass features 1365 if len(self.mass_spectra.mass_features) > 0: 1366 mf_df = self.mass_spectra.mass_features_to_df().reset_index(drop=False) 1367 self._write_parquet(mf_df, "mass_features", overwrite=overwrite) 1368 1369 if len(getattr(self.mass_spectra, "induced_mass_features", {}) or {}) > 0: 1370 imf_df = self.mass_spectra.mass_features_to_df( 1371 induced_features=True 1372 ).reset_index(drop=False) 1373 self._write_parquet(imf_df, "induced_mass_features", overwrite=overwrite) 1374 1375 # Peak tables for loaded spectra 1376 if export_spectra: 1377 spectra_df = self._mass_spectra_to_parquet_df() 1378 self._write_parquet(spectra_df, "mass_spectra", overwrite=overwrite) 1379 1380 # EICs 1381 export_eics = self.mass_spectra.parameters.lc_ms.export_eics 1382 if export_eics and len(self.mass_spectra.eics) > 0: 1383 eic_df = self._eics_to_parquet_df() 1384 self._write_parquet(eic_df, "eics", overwrite=overwrite) 1385 1386 # Spectral search results 1387 if len(self.mass_spectra.spectral_search_results) > 0: 1388 search_df = self._spectral_search_results_to_parquet_df() 1389 self._write_parquet( 1390 search_df, "spectral_search_results", overwrite=overwrite 1391 ) 1392 1393 if save_parameters: 1394 if parameter_format not in ["json", "toml"]: 1395 raise ValueError("parameter_format must be 'json' or 'toml'") 1396 if parameter_format == "json": 1397 dump_lcms_settings_json( 1398 filename=self.output_file.with_suffix(".json"), 1399 lcms_obj=self.mass_spectra, 1400 ) 1401 elif parameter_format == "toml": 1402 dump_lcms_settings_toml( 1403 filename=self.output_file.with_suffix(".toml"), 1404 lcms_obj=self.mass_spectra, 1405 ) 1406 1407 @staticmethod 1408 def _save_mass_features_dict_to_hdf5( 1409 mass_features_dict, 1410 mass_features_group, 1411 overwrite=False, 1412 compression=DEFAULT_HDF5_COMPRESSION, 1413 ): 1414 """Save a dictionary of mass features to an HDF5 group. 1415 1416 This is a helper method that can be reused by different export classes. 1417 1418 Parameters 1419 ---------- 1420 mass_features_dict : dict 1421 Dictionary of mass features to save, keyed by mass feature ID. 1422 mass_features_group : h5py.Group 1423 The HDF5 group to save the mass features to. 1424 overwrite : bool, optional 1425 Whether to overwrite existing mass features. Default is False. 1426 compression : int, str or None, optional 1427 Compression for the written datasets: a gzip level 0-9, 1428 ``"lzf"``, or ``None``/``"none"``. Default is 4. ``"lzf"`` is 1429 h5py's own filter, not a standard HDF5 one -- it needs h5py (or 1430 a tool with the lzf plugin) to read the result back. 1431 """ 1432 1433 # Create group for each mass feature, with key as the mass feature id 1434 for k, v in mass_features_dict.items(): 1435 if str(k) not in mass_features_group or overwrite: 1436 if str(k) in mass_features_group and overwrite: 1437 del mass_features_group[str(k)] 1438 mass_features_group.create_group(str(k)) 1439 # Loop through each of the mass feature attributes and add them as attributes (if single value) or datasets (if array) 1440 for k2, v2 in v.__dict__.items(): 1441 if v2 is not None: 1442 # Check if the attribute is an integer or float and set as an attribute in the mass feature group 1443 if k2 not in [ 1444 "chromatogram_parent", 1445 "ms2_mass_spectra", 1446 "mass_spectrum", 1447 "_eic_data", 1448 "ms2_similarity_results", 1449 ]: 1450 if k2 == "ms2_scan_numbers": 1451 array = np.array(v2) 1452 # Convert int64 to int32 1453 if array.dtype == np.int64: 1454 array = array.astype(np.int32) 1455 mass_features_group[str(k)].create_dataset( 1456 str(k2), 1457 data=array, 1458 **hdf5_dataset_kwargs( 1459 array.shape, compression=compression 1460 ), 1461 ) 1462 elif k2 == "_half_height_width": 1463 array = np.array(v2) 1464 # Convert float64 to float32 1465 if array.dtype == np.float64: 1466 array = array.astype(np.float32) 1467 mass_features_group[str(k)].create_dataset( 1468 str(k2), 1469 data=array, 1470 **hdf5_dataset_kwargs( 1471 array.shape, compression=compression 1472 ), 1473 ) 1474 elif k2 == "_ms_deconvoluted_idx": 1475 array = np.array(v2) 1476 # Convert int64 to int32 1477 if array.dtype == np.int64: 1478 array = array.astype(np.int32) 1479 mass_features_group[str(k)].create_dataset( 1480 str(k2), 1481 data=array, 1482 **hdf5_dataset_kwargs( 1483 array.shape, compression=compression 1484 ), 1485 ) 1486 elif k2 == "associated_mass_features_deconvoluted": 1487 array = np.array(v2) 1488 # Convert int64 to int32 1489 if array.dtype == np.int64: 1490 array = array.astype(np.int32) 1491 mass_features_group[str(k)].create_dataset( 1492 str(k2), 1493 data=array, 1494 **hdf5_dataset_kwargs( 1495 array.shape, compression=compression 1496 ), 1497 ) 1498 elif k2 == "_noise_score": 1499 array = np.array(v2) 1500 # Convert float64 to float32 1501 if array.dtype == np.float64: 1502 array = array.astype(np.float32) 1503 mass_features_group[str(k)].create_dataset( 1504 str(k2), 1505 data=array, 1506 **hdf5_dataset_kwargs( 1507 array.shape, compression=compression 1508 ), 1509 ) 1510 elif ( 1511 isinstance(v2, int) 1512 or isinstance(v2, float) 1513 or isinstance(v2, str) 1514 or isinstance(v2, np.integer) 1515 or isinstance(v2, np.float32) 1516 or isinstance(v2, np.float64) 1517 or isinstance(v2, np.bool_) 1518 ): 1519 # Convert numpy types to smaller precision for storage 1520 if isinstance(v2, np.int64): 1521 v2 = np.int32(v2) 1522 elif isinstance(v2, np.float64): 1523 v2 = np.float32(v2) 1524 mass_features_group[str(k)].attrs[str(k2)] = v2 1525 1526 @staticmethod 1527 def _save_eics_dict_to_hdf5( 1528 eics_dict, eics_group, overwrite=False, compression=DEFAULT_HDF5_COMPRESSION 1529 ): 1530 """Save a dictionary of EICs to an HDF5 group. 1531 1532 This is a static helper method that can be reused by different export classes 1533 to save EIC data in a consistent format. 1534 1535 Parameters 1536 ---------- 1537 eics_dict : dict 1538 Dictionary of EIC_Data objects, keyed by m/z value. 1539 eics_group : h5py.Group 1540 The HDF5 group to save the EICs to. 1541 overwrite : bool, optional 1542 Whether to overwrite existing EICs. Default is False. 1543 compression : int, str or None, optional 1544 Compression for the written datasets: a gzip level 0-9, 1545 ``"lzf"``, or ``None``/``"none"``. Default is 4. ``"lzf"`` is 1546 h5py's own filter, not a standard HDF5 one -- it needs h5py (or 1547 a tool with the lzf plugin) to read the result back. 1548 """ 1549 for mz, eic_data in eics_dict.items(): 1550 mz_str = str(mz) 1551 if mz_str not in eics_group or overwrite: 1552 if mz_str in eics_group and overwrite: 1553 del eics_group[mz_str] 1554 eic_grp = eics_group.create_group(mz_str) 1555 eic_grp.attrs["mz"] = mz 1556 1557 # Save all EIC_Data attributes as datasets 1558 for attr_name, attr_value in eic_data.__dict__.items(): 1559 if attr_value is not None: 1560 array = np.array(attr_value) 1561 # Apply data type optimization and compression 1562 if array.dtype == np.int64: 1563 array = array.astype(np.int32) 1564 elif array.dtype == np.float64: 1565 array = array.astype(np.float32) 1566 elif array.dtype.str[0:2] == "<U": 1567 # Convert Unicode strings to UTF-8 encoded strings 1568 string_data = [str(item) for item in array] 1569 string_dtype = h5py.string_dtype(encoding="utf-8") 1570 eic_grp.create_dataset( 1571 str(attr_name), 1572 data=string_data, 1573 dtype=string_dtype, 1574 **hdf5_dataset_kwargs( 1575 array.shape, compression=compression, vlen_str=True 1576 ), 1577 ) 1578 continue 1579 eic_grp.create_dataset( 1580 str(attr_name), 1581 data=array, 1582 **hdf5_dataset_kwargs(array.shape, compression=compression), 1583 ) 1584 1585 def _save_mass_features_to_hdf5( 1586 self, 1587 hdf_handle, 1588 group_name="mass_features", 1589 overwrite=False, 1590 compression=DEFAULT_HDF5_COMPRESSION, 1591 ): 1592 """Save the mass features to the HDF5 file. 1593 1594 Parameters 1595 ---------- 1596 hdf_handle : h5py.File 1597 The HDF5 file handle. 1598 group_name : str, optional 1599 The name of the group to save the mass features to. Default is 'mass_features'. 1600 overwrite : bool, optional 1601 Whether to overwrite the group if it exists. Default is False. 1602 compression : int, str or None, optional 1603 Compression for the written datasets: a gzip level 0-9, 1604 ``"lzf"``, or ``None``/``"none"``. Default is 4. ``"lzf"`` is 1605 h5py's own filter, not a standard HDF5 one -- it needs h5py (or 1606 a tool with the lzf plugin) to read the result back. 1607 """ 1608 # Determine which mass features to save based on group_name 1609 if group_name == "induced_mass_features": 1610 if len(self.mass_spectra.induced_mass_features) == 0: 1611 return # No induced mass features to save 1612 mass_features_dict = self.mass_spectra.induced_mass_features 1613 else: 1614 if len(self.mass_spectra.mass_features) == 0: 1615 return # No mass features to save 1616 mass_features_dict = self.mass_spectra.mass_features 1617 1618 # Add LCMS mass features to hdf5 file 1619 if group_name not in hdf_handle: 1620 mass_features_group = hdf_handle.create_group(group_name) 1621 else: 1622 mass_features_group = hdf_handle.get(group_name) 1623 1624 # Use the static helper method to save the mass features 1625 self._save_mass_features_dict_to_hdf5( 1626 mass_features_dict, mass_features_group, overwrite, compression=compression 1627 ) 1628 1629 def to_hdf( 1630 self, 1631 overwrite=False, 1632 save_parameters=True, 1633 parameter_format="toml", 1634 compression=DEFAULT_HDF5_COMPRESSION, 1635 profile_dtype="f4", 1636 ): 1637 """Export the data to an HDF5. 1638 1639 Parameters 1640 ---------- 1641 overwrite : bool, optional 1642 Whether to overwrite the output file. Default is False. 1643 save_parameters : bool, optional 1644 Whether to save the parameters as a separate json or toml file. Default is True. 1645 parameter_format : str, optional 1646 The format to save the parameters in. Default is 'toml'. 1647 compression : int, str or None, optional 1648 Compression for the written datasets: a gzip level 0-9, 1649 ``"lzf"``, or ``None``/``"none"``. Default is 4. ``"lzf"`` is 1650 h5py's own filter, not a standard HDF5 one -- it needs h5py (or 1651 a tool with the lzf plugin) to read the result back. 1652 profile_dtype : str, optional 1653 dtype used for the stored profile m/z and abundance arrays 1654 (``"f4"`` or ``"f8"``). Default is ``"f4"``. 1655 1656 Raises 1657 ------ 1658 ValueError 1659 If parameter_format is not 'json' or 'toml'. 1660 """ 1661 export_profile_spectra = ( 1662 self.mass_spectra.parameters.lc_ms.export_profile_spectra 1663 ) 1664 1665 # Write the mass spectra data to the hdf5 file 1666 super().to_hdf( 1667 overwrite=overwrite, 1668 export_raw=export_profile_spectra, 1669 compression=compression, 1670 profile_dtype=profile_dtype, 1671 ) 1672 1673 # Write scan info, ms_unprocessed, mass features, eics, and ms2_search results to the hdf5 file 1674 with h5py.File(self.output_file.with_suffix(".hdf5"), "a") as hdf_handle: 1675 # Add scan_info to hdf5 file 1676 if "scan_info" not in hdf_handle or overwrite: 1677 if "scan_info" in hdf_handle and overwrite: 1678 del hdf_handle["scan_info"] 1679 scan_info_group = hdf_handle.create_group("scan_info") 1680 for k, v in self.mass_spectra._scan_info.items(): 1681 array = np.array(list(v.values())) 1682 if array.dtype.str[0:2] == "<U": 1683 array = array.astype("S") 1684 scan_info_group.create_dataset( 1685 k, 1686 data=array, 1687 **hdf5_dataset_kwargs(array.shape, compression=compression), 1688 ) 1689 1690 # Add ms_unprocessed to hdf5 file 1691 export_unprocessed_ms1 = ( 1692 self.mass_spectra.parameters.lc_ms.export_unprocessed_ms1 1693 ) 1694 if self.mass_spectra._ms_unprocessed and export_unprocessed_ms1: 1695 if "ms_unprocessed" not in hdf_handle or overwrite: 1696 if "ms_unprocessed" in hdf_handle and overwrite: 1697 del hdf_handle["ms_unprocessed"] 1698 ms_unprocessed_group = hdf_handle.create_group("ms_unprocessed") 1699 else: 1700 ms_unprocessed_group = hdf_handle.get("ms_unprocessed") 1701 for k, v in self.mass_spectra._ms_unprocessed.items(): 1702 if str(k) not in ms_unprocessed_group or overwrite: 1703 if str(k) in ms_unprocessed_group and overwrite: 1704 del ms_unprocessed_group[str(k)] 1705 array = np.array(v) 1706 column_labels = ( 1707 list(v.columns) 1708 if hasattr(v, "columns") 1709 else ["scan", "mz", "intensity"] 1710 ) 1711 ms_dset = ms_unprocessed_group.create_dataset( 1712 str(k), 1713 data=array, 1714 **hdf5_dataset_kwargs(array.shape, compression=compression), 1715 ) 1716 ms_dset.attrs["column_labels"] = json.dumps(column_labels) 1717 1718 # Add LCMS mass features to hdf5 file 1719 self._save_mass_features_to_hdf5( 1720 hdf_handle, 1721 group_name="mass_features", 1722 overwrite=overwrite, 1723 compression=compression, 1724 ) 1725 self._save_mass_features_to_hdf5( 1726 hdf_handle, 1727 group_name="induced_mass_features", 1728 overwrite=overwrite, 1729 compression=compression, 1730 ) 1731 1732 # Add EIC data to hdf5 file 1733 export_eics = self.mass_spectra.parameters.lc_ms.export_eics 1734 if len(self.mass_spectra.eics) > 0 and export_eics: 1735 if "eics" not in hdf_handle or overwrite: 1736 if "eics" in hdf_handle and overwrite: 1737 del hdf_handle["eics"] 1738 eic_group = hdf_handle.create_group("eics") 1739 else: 1740 eic_group = hdf_handle.get("eics") 1741 1742 # Use the static helper method to save the EICs 1743 self._save_eics_dict_to_hdf5( 1744 self.mass_spectra.eics, 1745 eic_group, 1746 overwrite, 1747 compression=compression, 1748 ) 1749 1750 # Add ms2_search results to hdf5 file (parameterized) 1751 if len(self.mass_spectra.spectral_search_results) > 0: 1752 if "spectral_search_results" not in hdf_handle or overwrite: 1753 if "spectral_search_results" in hdf_handle and overwrite: 1754 del hdf_handle["spectral_search_results"] 1755 spectral_search_results = hdf_handle.create_group( 1756 "spectral_search_results" 1757 ) 1758 else: 1759 spectral_search_results = hdf_handle.get("spectral_search_results") 1760 # Create group for each search result by ms2_scan / precursor_mz 1761 for k, v in self.mass_spectra.spectral_search_results.items(): 1762 # TODO KRH: Fix to handle if export_only_relevant and k not in relevant_scan_numbers: continue! 1763 if str(k) not in spectral_search_results or overwrite: 1764 if str(k) in spectral_search_results and overwrite: 1765 del spectral_search_results[str(k)] 1766 spectral_search_results.create_group(str(k)) 1767 for k2, v2 in v.items(): 1768 spectral_search_results[str(k)].create_group(str(k2)) 1769 spectral_search_results[str(k)][str(k2)].attrs[ 1770 "precursor_mz" 1771 ] = v2.precursor_mz 1772 spectral_search_results[str(k)][str(k2)].attrs[ 1773 "query_spectrum_id" 1774 ] = v2.query_spectrum_id 1775 # Loop through each of the attributes and add them as datasets (if array) 1776 for k3, v3 in v2.__dict__.items(): 1777 if v3 is not None and k3 not in [ 1778 "query_spectrum", 1779 "precursor_mz", 1780 "query_spectrum_id", 1781 ]: 1782 if ( 1783 k3 == "query_frag_types" 1784 or k3 == "ref_frag_types" 1785 ): 1786 v3 = [", ".join(x) for x in v3] 1787 if all(v3 is not None for v3 in v3): 1788 array = np.array(v3) 1789 if array.dtype.str[0:2] == "<U": 1790 array = array.astype("S") 1791 spectral_search_results[str(k)][ 1792 str(k2) 1793 ].create_dataset( 1794 str(k3), 1795 data=array, 1796 **hdf5_dataset_kwargs( 1797 array.shape, compression=compression 1798 ), 1799 ) 1800 1801 # Save parameters as separate json 1802 if save_parameters: 1803 # Check if parameter_format is valid 1804 if parameter_format not in ["json", "toml"]: 1805 raise ValueError("parameter_format must be 'json' or 'toml'") 1806 1807 if parameter_format == "json": 1808 dump_lcms_settings_json( 1809 filename=self.output_file.with_suffix(".json"), 1810 lcms_obj=self.mass_spectra, 1811 ) 1812 elif parameter_format == "toml": 1813 dump_lcms_settings_toml( 1814 filename=self.output_file.with_suffix(".toml"), 1815 lcms_obj=self.mass_spectra, 1816 ) 1817 1818 1819class LCMSMetabolomicsExport(LCMSExport): 1820 """A class to export LCMS metabolite data. 1821 1822 This class provides methods to export LCMS metabolite data to various formats 1823 (CSV/Parquet reports, HDF5) and summarize the metabolite report. 1824 1825 Parameters 1826 ---------- 1827 out_file_path : str | Path 1828 The output file path, do not include the file extension. 1829 mass_spectra : object 1830 The high resolution mass spectra object. 1831 """ 1832 1833 def __init__(self, out_file_path, mass_spectra): 1834 super().__init__(out_file_path, mass_spectra) 1835 self.ion_type_dict = MolecularFormulaCalc.ion_type_dict 1836 1837 @staticmethod 1838 def get_ion_formula(neutral_formula, ion_type): 1839 """From a neutral formula and an ion type, return the formula of the ion. 1840 1841 Notes 1842 ----- 1843 Delegates to 1844 :meth:`corems.molecular_formula.calc.MolecularFormulaCalc.MolecularFormulaCalc.get_ion_formula`. 1845 1846 Parameters 1847 ---------- 1848 neutral_formula : str 1849 The neutral formula, this should be a string form from the MolecularFormula class 1850 (e.g. 'C2 H4 O2', isotopes OK), or simple string (e.g. 'C2H4O2', no isotope handling in this case). 1851 In the case of a simple string, the atoms are parsed based on the presence of capital letters, 1852 e.g. MgCl2 is parsed as 'Mg Cl2. 1853 ion_type : str 1854 The ion type, e.g. 'protonated', '[M+H]+', '[M+Na]+', etc. 1855 See ``MolecularFormulaCalc.ion_type_dict`` for the available ion types. 1856 1857 Returns 1858 ------- 1859 str 1860 The formula of the ion as a string (like 'C2 H4 O2'); or None if the neutral_formula is not a string. 1861 """ 1862 return MolecularFormulaCalc.get_ion_formula(neutral_formula, ion_type) 1863 1864 @staticmethod 1865 def precursor_mz_from_formula(neutral_formula, ion_type, charge=None): 1866 """Calculated precursor m/z from neutral formula and adduct / ion type. 1867 1868 Notes 1869 ----- 1870 Delegates to 1871 :meth:`corems.molecular_formula.calc.MolecularFormulaCalc.MolecularFormulaCalc.precursor_mz_from_formula`. 1872 """ 1873 return MolecularFormulaCalc.precursor_mz_from_formula( 1874 neutral_formula, ion_type, charge=charge 1875 ) 1876 1877 @staticmethod 1878 def get_isotope_type(ion_formula): 1879 """From an ion formula, return the 13C isotope type of the ion. 1880 1881 Notes 1882 ----- 1883 This is a static method. 1884 If the ion_formula is not a string, this method will return None. 1885 This is currently only functional for 13C isotopes. 1886 1887 Parameters 1888 ---------- 1889 ion_formula : str 1890 The formula of the ion, expected to be a string like 'C2 H4 O2'. 1891 1892 Returns 1893 ------- 1894 str 1895 The isotope type of the ion, e.g. '13C1', '13C2', etc; or None if the ion_formula does not contain a 13C isotope. 1896 1897 Raises 1898 ------ 1899 ValueError 1900 If the ion_formula is not a string. 1901 """ 1902 if not isinstance(ion_formula, str): 1903 return None 1904 1905 if re.search(r"\s", ion_formula): 1906 ion_formula = MolecularFormula(ion_formula, ion_charge=0) 1907 else: 1908 raise ValueError('ion_formula should be a string like "C2 H4 O2"') 1909 ion_formula_dict = ion_formula.to_dict().copy() 1910 1911 try: 1912 iso_class = "13C" + str(ion_formula_dict.pop("13C")) 1913 except KeyError: 1914 iso_class = None 1915 1916 return iso_class 1917 1918 def report_to_csv(self, molecular_metadata=None): 1919 """Create a report of the mass features and their annotations and save it as a CSV file. 1920 1921 Parameters 1922 ---------- 1923 molecular_metadata : dict, optional 1924 The molecular metadata. Default is None. 1925 """ 1926 report = self.to_report(molecular_metadata=molecular_metadata) 1927 out_file = self.output_file.with_suffix(".csv") 1928 report.to_csv(out_file, index=False) 1929 1930 def report_to_parquet(self, molecular_metadata=None): 1931 """Create a report of the mass features and their annotations and save it as a Parquet file. 1932 1933 Parameters 1934 ---------- 1935 molecular_metadata : dict, optional 1936 The molecular metadata. Default is None. 1937 1938 Notes 1939 ----- 1940 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 1941 Shared by metabolomics and lipidomics exporters via inheritance. 1942 """ 1943 report = self.to_report(molecular_metadata=molecular_metadata) 1944 out_file = self.output_file.with_suffix(".parquet") 1945 report.to_parquet(out_file, index=False) 1946 1947 def clean_ms1_report(self, ms1_summary_full): 1948 """Clean the MS1 report. 1949 1950 Parameters 1951 ---------- 1952 ms1_summary_full : DataFrame 1953 The full MS1 summary DataFrame. 1954 1955 Returns 1956 ------- 1957 DataFrame 1958 The cleaned MS1 summary DataFrame. 1959 """ 1960 ms1_summary_full = ms1_summary_full.reset_index() 1961 cols_to_keep = [ 1962 "mf_id", 1963 "Molecular Formula", 1964 "Ion Type", 1965 "Calculated m/z", 1966 "m/z Error (ppm)", 1967 "m/z Error Score", 1968 "Is Isotopologue", 1969 "Isotopologue Similarity", 1970 "Confidence Score", 1971 ] 1972 ms1_summary = ms1_summary_full[cols_to_keep].copy() 1973 ms1_summary["ion_formula"] = [ 1974 self.get_ion_formula(f, a) 1975 for f, a in zip(ms1_summary["Molecular Formula"], ms1_summary["Ion Type"]) 1976 ] 1977 ms1_summary["isotopologue_type"] = [ 1978 self.get_isotope_type(f) for f in ms1_summary["ion_formula"].tolist() 1979 ] 1980 1981 # Keep MS1 neutral Molecular Formula so it is not confused with library formulas 1982 # from MS2 spectral metadata. 1983 ms1_summary = ms1_summary[ 1984 [ 1985 "mf_id", 1986 "ion_formula", 1987 "isotopologue_type", 1988 "Molecular Formula", 1989 "Calculated m/z", 1990 "m/z Error (ppm)", 1991 "m/z Error Score", 1992 "Isotopologue Similarity", 1993 "Confidence Score", 1994 ] 1995 ] 1996 1997 # Set the index to mf_id 1998 ms1_summary = ms1_summary.set_index("mf_id") 1999 2000 return ms1_summary 2001 2002 def summarize_ms2_report(self, ms2_annot_report): 2003 """ 2004 Summarize the MS2 report. 2005 2006 Parameters 2007 ---------- 2008 ms2_annot_report : DataFrame 2009 The MS2 annotation DataFrame with all annotations, output of mass_features_ms2_annot_to_df. 2010 2011 Returns 2012 ------- 2013 """ 2014 2015 def summarize_metabolomics_report(self, ms2_annot_report): 2016 """Summarize the MS2 hits for a metabolomics report 2017 2018 Parameters 2019 ---------- 2020 ms2_annot : DataFrame 2021 The MS2 annotation DataFrame with all annotations. 2022 2023 Returns 2024 ------- 2025 DataFrame 2026 The summarized metabolomics report. 2027 """ 2028 columns_to_drop = [ 2029 "precursor_mz", 2030 "precursor_mz_error_ppm", 2031 "cas", 2032 "data_id", 2033 "iupac_name", 2034 "traditional_name", 2035 "common_name", 2036 "casno", 2037 ] 2038 ms2_annot = ms2_annot_report.drop( 2039 columns=[col for col in columns_to_drop if col in ms2_annot_report.columns] 2040 ) 2041 2042 # Prepare information about the search results, pulling out the best hit for the single report 2043 # Group by mf_id,ref_mol_id grab row with highest entropy similarity 2044 ms2_annot = ms2_annot.reset_index() 2045 # Add column called "n_spectra_contributing" that is the number of unique values in query_spectrum_id per mf_id,ref_mol_id 2046 ms2_annot["n_spectra_contributing"] = ms2_annot.groupby( 2047 ["mf_id", "ref_mol_id"] 2048 )["query_spectrum_id"].transform("nunique") 2049 # Sort by entropy similarity 2050 ms2_annot = ms2_annot.sort_values( 2051 by=["mf_id", "ref_mol_id", "entropy_similarity"], 2052 ascending=[True, True, False], 2053 ) 2054 best_entropy = ms2_annot.drop_duplicates( 2055 subset=["mf_id", "ref_mol_id"], keep="first" 2056 ) 2057 2058 return best_entropy 2059 2060 def clean_ms2_report(self, metabolite_summary): 2061 """Clean the MS2 report. 2062 2063 Parameters 2064 ---------- 2065 metabolite_summary : DataFrame 2066 The full metabolomics summary DataFrame. 2067 2068 Returns 2069 ------- 2070 DataFrame 2071 The cleaned metabolomics summary DataFrame. 2072 """ 2073 metabolite_summary = metabolite_summary.reset_index() 2074 # Library (MS2 metadata) formulas — keep separate from MS1 search fields 2075 metabolite_summary["library_ion_formula"] = [ 2076 self.get_ion_formula(f, a) 2077 for f, a in zip( 2078 metabolite_summary["formula"], metabolite_summary["ref_ion_type"] 2079 ) 2080 ] 2081 metabolite_summary = metabolite_summary.rename( 2082 columns={"formula": "library_formula"} 2083 ) 2084 2085 col_order = [ 2086 "mf_id", 2087 "library_ion_formula", 2088 "ref_ion_type", 2089 "library_formula", 2090 "inchikey", 2091 "name", 2092 "inchi", 2093 "chebi", 2094 "smiles", 2095 "kegg", 2096 "cas", 2097 "database_name", 2098 "ref_ms_id", 2099 "entropy_similarity", 2100 "ref_mz_in_query_fract", 2101 "n_spectra_contributing", 2102 ] 2103 2104 # Reorder columns 2105 metabolite_summary = metabolite_summary[ 2106 [col for col in col_order if col in metabolite_summary.columns] 2107 ] 2108 2109 # Convert chebi (if present) to int: 2110 if "chebi" in metabolite_summary.columns: 2111 metabolite_summary["chebi"] = metabolite_summary["chebi"].astype( 2112 "Int64", errors="ignore" 2113 ) 2114 2115 # Set the index to mf_id 2116 metabolite_summary = metabolite_summary.set_index("mf_id") 2117 2118 return metabolite_summary 2119 2120 def combine_reports(self, mf_report, ms1_annot_report, ms2_annot_report): 2121 """Combine the mass feature report with the MS1 and MS2 reports. 2122 2123 Parameters 2124 ---------- 2125 mf_report : DataFrame 2126 The mass feature report DataFrame. 2127 ms1_annot_report : DataFrame 2128 The MS1 annotation report DataFrame. 2129 ms2_annot_report : DataFrame 2130 The MS2 annotation report DataFrame. 2131 2132 Notes 2133 ----- 2134 ``Molecular Formula`` / ``Ion Formula`` come only from MS1 molecular 2135 formula search. MS2 spectral library formulas are exposed as 2136 ``Library Molecular Formula`` / ``Library Ion Formula``. 2137 When MS1 and MS2 match (same ion formula), the library formula columns 2138 are cleared so they do not repeat the MS1 values on that row. 2139 """ 2140 # If there is an ms1_annot_report, merge it with the mf_report 2141 if ms1_annot_report is not None and not ms1_annot_report.empty: 2142 # MS1 has been run and has molecular formula information 2143 mf_report = pd.merge( 2144 mf_report, 2145 ms1_annot_report, 2146 how="left", 2147 on=["mf_id", "isotopologue_type"], 2148 ) 2149 if ms2_annot_report is not None: 2150 # Prefer joining MS2 hits to matching MS1 ion formula when both exist. 2151 # MS2 uses library_ion_formula so MS1 Ion Formula is not overwritten. 2152 if ( 2153 "ion_formula" in mf_report.columns 2154 and "library_ion_formula" in ms2_annot_report.columns 2155 ): 2156 # pull out the records without ion_formula and merge on mf_id only 2157 mf_no_ion_formula = mf_report[mf_report["ion_formula"].isna()] 2158 mf_no_ion_formula = ( 2159 mf_no_ion_formula.drop(columns=["ion_formula"]) 2160 if "ion_formula" in mf_no_ion_formula.columns 2161 else mf_no_ion_formula 2162 ) 2163 mf_no_ion_formula = pd.merge( 2164 mf_no_ion_formula, ms2_annot_report, how="left", on=["mf_id"] 2165 ) 2166 2167 # pull out the records with ion_formula and merge on mf_id + formula match 2168 mf_with_ion_formula = mf_report[~mf_report["ion_formula"].isna()] 2169 mf_with_ion_formula = pd.merge( 2170 mf_with_ion_formula, 2171 ms2_annot_report, 2172 how="left", 2173 left_on=["mf_id", "ion_formula"], 2174 right_on=["mf_id", "library_ion_formula"], 2175 ) 2176 2177 # put back together 2178 mf_report = pd.concat([mf_no_ion_formula, mf_with_ion_formula]) 2179 else: 2180 # Fall back to merging on mf_id only (robust when ion_formula missing) 2181 mf_report = pd.merge( 2182 mf_report, ms2_annot_report, how="left", on=["mf_id"] 2183 ) 2184 2185 # Rename colums 2186 rename_dict = { 2187 "mf_id": "Mass Feature ID", 2188 "scan_time": "Retention Time (min)", 2189 "mz": "m/z", 2190 "apex_scan": "Apex Scan Number", 2191 "intensity": "Intensity", 2192 "persistence": "Persistence", 2193 "area": "Area", 2194 "half_height_width": "Half Height Width (min)", 2195 "tailing_factor": "Tailing Factor", 2196 "dispersity_index": "Dispersity Index", 2197 "ms2_spectrum": "MS2 Spectrum", 2198 "monoisotopic_mf_id": "Monoisotopic Mass Feature ID", 2199 "isotopologue_type": "Isotopologue Type", 2200 "mass_spectrum_deconvoluted_parent": "Is Largest Ion after Deconvolution", 2201 "associated_mass_features": "Associated Mass Features after Deconvolution", 2202 "ion_formula": "Ion Formula", 2203 "library_ion_formula": "Library Ion Formula", 2204 "library_formula": "Library Molecular Formula", 2205 "ref_ion_type": "Library Ion Type", 2206 "annot_level": "Lipid Annotation Level", 2207 "lipid_molecular_species_id": "Lipid Molecular Species", 2208 "lipid_summed_name": "Lipid Species", 2209 "lipid_subclass": "Lipid Subclass", 2210 "lipid_class": "Lipid Class", 2211 "lipid_category": "Lipid Category", 2212 "entropy_similarity": "Entropy Similarity", 2213 "ref_mz_in_query_fract": "Library mzs in Query (fraction)", 2214 "n_spectra_contributing": "Spectra with Annotation (n)", 2215 } 2216 mf_report = mf_report.rename(columns=rename_dict) 2217 2218 # When MS1 and MS2 co-annotate the same ion/molecular formula, library formula 2219 # columns only repeat MS1 fields — clear them so the row is not redundant. 2220 # Keep Library * columns populated for MS2-only (or mismatched) hits. 2221 if ( 2222 "Ion Formula" in mf_report.columns 2223 and "Library Ion Formula" in mf_report.columns 2224 ): 2225 ion_match = ( 2226 mf_report["Ion Formula"].notna() 2227 & mf_report["Library Ion Formula"].notna() 2228 & (mf_report["Ion Formula"] == mf_report["Library Ion Formula"]) 2229 ) 2230 mf_report.loc[ion_match, "Library Ion Formula"] = pd.NA 2231 if "Library Molecular Formula" in mf_report.columns: 2232 mf_report.loc[ion_match, "Library Molecular Formula"] = pd.NA 2233 elif ( 2234 "Molecular Formula" in mf_report.columns 2235 and "Library Molecular Formula" in mf_report.columns 2236 ): 2237 mol_match = ( 2238 mf_report["Molecular Formula"].notna() 2239 & mf_report["Library Molecular Formula"].notna() 2240 & ( 2241 mf_report["Molecular Formula"] 2242 == mf_report["Library Molecular Formula"] 2243 ) 2244 ) 2245 mf_report.loc[mol_match, "Library Molecular Formula"] = pd.NA 2246 if "Library Ion Formula" in mf_report.columns: 2247 mf_report.loc[mol_match, "Library Ion Formula"] = pd.NA 2248 2249 mf_report["Sample Name"] = self.mass_spectra.sample_name 2250 mf_report["Polarity"] = self.mass_spectra.polarity 2251 mf_report = mf_report[ 2252 ["Mass Feature ID", "Sample Name", "Polarity"] 2253 + [ 2254 col 2255 for col in mf_report.columns 2256 if col not in ["Mass Feature ID", "Sample Name", "Polarity"] 2257 ] 2258 ] 2259 2260 # Reorder rows by "Mass Feature ID", then "Entropy Similarity" (descending), then "Confidence Score" (descending) 2261 if ( 2262 "Entropy Similarity" in mf_report.columns 2263 and "Confidence Score" in mf_report.columns 2264 ): 2265 mf_report = mf_report.sort_values( 2266 by=["Mass Feature ID", "Entropy Similarity", "Confidence Score"], 2267 ascending=[True, False, False], 2268 ) 2269 elif "Entropy Similarity" in mf_report.columns: 2270 mf_report = mf_report.sort_values( 2271 by=["Mass Feature ID", "Entropy Similarity"], 2272 ascending=[True, False], 2273 ) 2274 elif "Confidence Score" in mf_report.columns: 2275 mf_report = mf_report.sort_values( 2276 by=["Mass Feature ID", "Confidence Score"], 2277 ascending=[True, False], 2278 ) 2279 # If neither "Entropy Similarity" nor "Confidence Score" are in the columns, just sort by "Mass Feature ID" 2280 else: 2281 mf_report = mf_report.sort_values("Mass Feature ID") 2282 2283 # Reset index 2284 mf_report = mf_report.reset_index(drop=True) 2285 2286 return mf_report 2287 2288 def to_report(self, molecular_metadata=None, suppress_warnings=False): 2289 """Create a report of the mass features and their annotations. 2290 2291 Parameters 2292 ---------- 2293 molecular_metadata : dict, optional 2294 The molecular metadata. Default is None. 2295 suppress_warnings : bool, optional 2296 If True, suppresses warnings from mass_features_ms2_annot_to_df. 2297 Default is False. 2298 2299 Returns 2300 ------- 2301 DataFrame 2302 The report as a Pandas DataFrame. 2303 """ 2304 # Get mass feature dataframe 2305 mf_report = self.mass_spectra.mass_features_to_df() 2306 mf_report = mf_report.reset_index(drop=False) 2307 2308 # Get and clean ms1 annotation dataframe 2309 ms1_annot_report = self.mass_spectra.mass_features_ms1_annot_to_df( 2310 suppress_warnings=suppress_warnings 2311 ) 2312 if ms1_annot_report is not None: 2313 ms1_annot_report = ms1_annot_report.copy() 2314 ms1_annot_report = self.clean_ms1_report(ms1_annot_report) 2315 ms1_annot_report = ms1_annot_report.reset_index(drop=False) 2316 else: 2317 ms1_annot_report = None 2318 2319 # Get, summarize, and clean ms2 annotation dataframe 2320 ms2_annot_report = self.mass_spectra.mass_features_ms2_annot_to_df( 2321 molecular_metadata=molecular_metadata, suppress_warnings=suppress_warnings 2322 ) 2323 if ms2_annot_report is not None and molecular_metadata is not None: 2324 ms2_annot_report = self.summarize_metabolomics_report(ms2_annot_report) 2325 ms2_annot_report = self.clean_ms2_report(ms2_annot_report) 2326 ms2_annot_report = ms2_annot_report.dropna(axis=1, how="all") 2327 ms2_annot_report = ms2_annot_report.reset_index(drop=False) 2328 else: 2329 ms2_annot_report = None 2330 2331 report = self.combine_reports( 2332 mf_report=mf_report, 2333 ms1_annot_report=ms1_annot_report, 2334 ms2_annot_report=ms2_annot_report, 2335 ) 2336 2337 return report 2338 2339 2340class LipidomicsExport(LCMSMetabolomicsExport): 2341 """A class to export lipidomics data. 2342 2343 This class provides methods to export lipidomics data to various formats 2344 (CSV/Parquet reports, HDF5) and summarize the lipid report. 2345 2346 Parameters 2347 ---------- 2348 out_file_path : str | Path 2349 The output file path, do not include the file extension. 2350 mass_spectra : object 2351 The high resolution mass spectra object. 2352 """ 2353 2354 def __init__(self, out_file_path, mass_spectra): 2355 super().__init__(out_file_path, mass_spectra) 2356 2357 def summarize_lipid_report(self, ms2_annot): 2358 """Summarize the lipid report. 2359 2360 Parameters 2361 ---------- 2362 ms2_annot : DataFrame 2363 The MS2 annotation DataFrame with all annotations. 2364 2365 Returns 2366 ------- 2367 DataFrame 2368 The summarized lipid report. 2369 """ 2370 # Drop unnecessary columns for easier viewing 2371 columns_to_drop = [ 2372 "precursor_mz", 2373 "precursor_mz_error_ppm", 2374 "ref_mol_id", 2375 "ref_precursor_mz", 2376 "cas", 2377 "inchikey", 2378 "inchi", 2379 "chebi", 2380 "smiles", 2381 "kegg", 2382 "data_id", 2383 "iupac_name", 2384 "traditional_name", 2385 "common_name", 2386 "casno", 2387 ] 2388 ms2_annot = ms2_annot.drop( 2389 columns=[col for col in columns_to_drop if col in ms2_annot.columns] 2390 ) 2391 2392 # If ion_types_excluded is not empty, remove those ion types 2393 ion_types_excluded = self.mass_spectra.parameters.mass_spectrum[ 2394 "ms2" 2395 ].molecular_search.ion_types_excluded 2396 if len(ion_types_excluded) > 0: 2397 ms2_annot = ms2_annot[~ms2_annot["ref_ion_type"].isin(ion_types_excluded)] 2398 2399 # If mf_id is not present, check that the index name is mf_id and reset the index 2400 if "mf_id" not in ms2_annot.columns: 2401 if ms2_annot.index.name == "mf_id": 2402 ms2_annot = ms2_annot.reset_index() 2403 else: 2404 raise ValueError("mf_id is not present in the dataframe") 2405 2406 # Attempt to get consensus annotations to the MLF level 2407 mlf_results_all = [] 2408 for mf_id in ms2_annot["mf_id"].unique(): 2409 mlf_results_perid = [] 2410 ms2_annot_mf = ms2_annot[ms2_annot["mf_id"] == mf_id].copy() 2411 ms2_annot_mf["n_spectra_contributing"] = ( 2412 ms2_annot_mf.query_spectrum_id.nunique() 2413 ) 2414 2415 for query_scan in ms2_annot["query_spectrum_id"].unique(): 2416 ms2_annot_sub = ms2_annot_mf[ 2417 ms2_annot_mf["query_spectrum_id"] == query_scan 2418 ].copy() 2419 2420 if ms2_annot_sub["lipid_summed_name"].nunique() == 1: 2421 # If there is only one lipid_summed_name, let's try to get consensus molecular species annotation 2422 if ms2_annot_sub["lipid_summed_name"].nunique() == 1: 2423 ms2_annot_sub["entropy_max"] = ( 2424 ms2_annot_sub["entropy_similarity"] 2425 == ms2_annot_sub["entropy_similarity"].max() 2426 ) 2427 ms2_annot_sub["ref_match_fract_max"] = ( 2428 ms2_annot_sub["ref_mz_in_query_fract"] 2429 == ms2_annot_sub["ref_mz_in_query_fract"].max() 2430 ) 2431 ms2_annot_sub["frag_max"] = ms2_annot_sub[ 2432 "query_frag_types" 2433 ].apply(lambda x: True if "MLF" in x else False) 2434 2435 # New column that looks if there is a consensus between the ranks (one row that is highest in all ranks) 2436 ms2_annot_sub["consensus"] = ms2_annot_sub[ 2437 ["entropy_max", "ref_match_fract_max", "frag_max"] 2438 ].all(axis=1) 2439 2440 # If there is a consensus, take the row with the highest entropy_similarity 2441 if ms2_annot_sub["consensus"].any(): 2442 ms2_annot_sub = ms2_annot_sub[ 2443 ms2_annot_sub["entropy_similarity"] 2444 == ms2_annot_sub["entropy_similarity"].max() 2445 ].head(1) 2446 mlf_results_perid.append(ms2_annot_sub) 2447 if len(mlf_results_perid) == 0: 2448 mlf_results_perid = pd.DataFrame() 2449 else: 2450 mlf_results_perid = pd.concat(mlf_results_perid) 2451 if mlf_results_perid["name"].nunique() == 1: 2452 mlf_results_perid = mlf_results_perid[ 2453 mlf_results_perid["entropy_similarity"] 2454 == mlf_results_perid["entropy_similarity"].max() 2455 ].head(1) 2456 else: 2457 mlf_results_perid = pd.DataFrame() 2458 mlf_results_all.append(mlf_results_perid) 2459 2460 # These are the consensus annotations to the MLF level 2461 if len(mlf_results_all) > 0: 2462 mlf_results_all = pd.concat(mlf_results_all) 2463 mlf_results_all["annot_level"] = mlf_results_all["structure_level"] 2464 else: 2465 # Make an empty dataframe 2466 mlf_results_all = ms2_annot.head(0) 2467 2468 # For remaining mf_ids, try to get a consensus annotation to the species level 2469 species_results_all = [] 2470 # Remove mf_ids that have consensus annotations to the MLF level 2471 ms2_annot_spec = ms2_annot[ 2472 ~ms2_annot["mf_id"].isin(mlf_results_all["mf_id"].unique()) 2473 ] 2474 for mf_id in ms2_annot_spec["mf_id"].unique(): 2475 # Do all the hits have the same lipid_summed_name? 2476 ms2_annot_sub = ms2_annot_spec[ms2_annot_spec["mf_id"] == mf_id].copy() 2477 ms2_annot_sub["n_spectra_contributing"] = len(ms2_annot_sub) 2478 2479 if ms2_annot_sub["lipid_summed_name"].nunique() == 1: 2480 # Grab the highest entropy_similarity result 2481 ms2_annot_sub = ms2_annot_sub[ 2482 ms2_annot_sub["entropy_similarity"] 2483 == ms2_annot_sub["entropy_similarity"].max() 2484 ].head(1) 2485 species_results_all.append(ms2_annot_sub) 2486 2487 # These are the consensus annotations to the species level 2488 if len(species_results_all) > 0: 2489 species_results_all = pd.concat(species_results_all) 2490 species_results_all["annot_level"] = "species" 2491 else: 2492 # Make an empty dataframe 2493 species_results_all = ms2_annot.head(0) 2494 2495 # Deal with the remaining mf_ids that do not have consensus annotations to the species level or MLF level 2496 # Remove mf_ids that have consensus annotations to the species level 2497 ms2_annot_remaining = ms2_annot_spec[ 2498 ~ms2_annot_spec["mf_id"].isin(species_results_all["mf_id"].unique()) 2499 ] 2500 no_consensus = [] 2501 for mf_id in ms2_annot_remaining["mf_id"].unique(): 2502 id_sub = [] 2503 id_no_con = [] 2504 ms2_annot_sub_mf = ms2_annot_remaining[ 2505 ms2_annot_remaining["mf_id"] == mf_id 2506 ].copy() 2507 for query_scan in ms2_annot_sub_mf["query_spectrum_id"].unique(): 2508 ms2_annot_sub = ms2_annot_sub_mf[ 2509 ms2_annot_sub_mf["query_spectrum_id"] == query_scan 2510 ].copy() 2511 2512 # New columns for ranking [HIGHER RANK = BETTER] 2513 ms2_annot_sub["entropy_max"] = ( 2514 ms2_annot_sub["entropy_similarity"] 2515 == ms2_annot_sub["entropy_similarity"].max() 2516 ) 2517 ms2_annot_sub["ref_match_fract_max"] = ( 2518 ms2_annot_sub["ref_mz_in_query_fract"] 2519 == ms2_annot_sub["ref_mz_in_query_fract"].max() 2520 ) 2521 ms2_annot_sub["frag_max"] = ms2_annot_sub["query_frag_types"].apply( 2522 lambda x: True if "MLF" in x else False 2523 ) 2524 2525 # New column that looks if there is a consensus between the ranks (one row that is highest in all ranks) 2526 ms2_annot_sub["consensus"] = ms2_annot_sub[ 2527 ["entropy_max", "ref_match_fract_max", "frag_max"] 2528 ].all(axis=1) 2529 ms2_annot_sub_con = ms2_annot_sub[ms2_annot_sub["consensus"]] 2530 id_sub.append(ms2_annot_sub_con) 2531 id_no_con.append(ms2_annot_sub) 2532 id_sub = pd.concat(id_sub) 2533 id_no_con = pd.concat(id_no_con) 2534 2535 # Scenario 1: Multiple scans are being resolved to different MLFs [could be coelutions and should both be kept and annotated to MS level] 2536 if ( 2537 id_sub["query_frag_types"] 2538 .apply(lambda x: True if "MLF" in x else False) 2539 .all() 2540 and len(id_sub) > 0 2541 ): 2542 idx = id_sub.groupby("name")["entropy_similarity"].idxmax() 2543 id_sub = id_sub.loc[idx] 2544 # Reorder so highest entropy_similarity is first 2545 id_sub = id_sub.sort_values("entropy_similarity", ascending=False) 2546 id_sub["annot_level"] = id_sub["structure_level"] 2547 no_consensus.append(id_sub) 2548 2549 # Scenario 2: Multiple scans are being resolved to different species, keep both and annotate to appropriate level 2550 elif len(id_sub) == 0: 2551 for lipid_summed_name in id_no_con["lipid_summed_name"].unique(): 2552 summed_sub = id_no_con[ 2553 id_no_con["lipid_summed_name"] == lipid_summed_name 2554 ] 2555 # Any consensus to MLF? 2556 if summed_sub["consensus"].any(): 2557 summed_sub = summed_sub[summed_sub["consensus"]] 2558 summed_sub["annot_level"] = summed_sub["structure_level"] 2559 no_consensus.append(summed_sub) 2560 else: 2561 # Grab the highest entropy_similarity, if there are multiple, grab the first one 2562 summed_sub = summed_sub[ 2563 summed_sub["entropy_similarity"] 2564 == summed_sub["entropy_similarity"].max() 2565 ].head(1) 2566 # get first row 2567 summed_sub["annot_level"] = "species" 2568 summed_sub["name"] = "" 2569 no_consensus.append(summed_sub) 2570 else: 2571 raise ValueError("Unexpected scenario for summarizing mf_id: ", mf_id) 2572 2573 if len(no_consensus) > 0: 2574 no_consensus = pd.concat(no_consensus) 2575 else: 2576 no_consensus = ms2_annot.head(0) 2577 2578 # Combine all the consensus annotations and reformat the dataframe for output 2579 species_results_all = species_results_all.drop(columns=["name"]) 2580 species_results_all["lipid_molecular_species_id"] = "" 2581 mlf_results_all["lipid_molecular_species_id"] = mlf_results_all["name"] 2582 no_consensus["lipid_molecular_species_id"] = no_consensus["name"] 2583 consensus_annotations = pd.concat( 2584 [mlf_results_all, species_results_all, no_consensus] 2585 ) 2586 consensus_annotations = consensus_annotations.sort_values( 2587 "mf_id", ascending=True 2588 ) 2589 cols_to_keep = [ 2590 "mf_id", 2591 "ref_ion_type", 2592 "entropy_similarity", 2593 "ref_mz_in_query_fract", 2594 "lipid_molecular_species_id", 2595 "lipid_summed_name", 2596 "lipid_subclass", 2597 "lipid_class", 2598 "lipid_category", 2599 "formula", 2600 "annot_level", 2601 "n_spectra_contributing", 2602 ] 2603 consensus_annotations = consensus_annotations[cols_to_keep] 2604 consensus_annotations = consensus_annotations.set_index("mf_id") 2605 2606 return consensus_annotations 2607 2608 def clean_ms2_report(self, lipid_summary): 2609 """Clean the MS2 report. 2610 2611 Parameters 2612 ---------- 2613 lipid_summary : DataFrame 2614 The full lipid summary DataFrame. 2615 2616 Returns 2617 ------- 2618 DataFrame 2619 The cleaned lipid summary DataFrame. 2620 """ 2621 lipid_summary = lipid_summary.reset_index() 2622 # Library (MS2 metadata) formulas — keep separate from MS1 search fields 2623 lipid_summary["library_ion_formula"] = [ 2624 self.get_ion_formula(f, a) 2625 for f, a in zip(lipid_summary["formula"], lipid_summary["ref_ion_type"]) 2626 ] 2627 lipid_summary = lipid_summary.rename(columns={"formula": "library_formula"}) 2628 2629 # Reorder columns 2630 lipid_summary = lipid_summary[ 2631 [ 2632 "mf_id", 2633 "library_ion_formula", 2634 "ref_ion_type", 2635 "library_formula", 2636 "annot_level", 2637 "lipid_molecular_species_id", 2638 "lipid_summed_name", 2639 "lipid_subclass", 2640 "lipid_class", 2641 "lipid_category", 2642 "entropy_similarity", 2643 "ref_mz_in_query_fract", 2644 "n_spectra_contributing", 2645 ] 2646 ] 2647 2648 # Set the index to mf_id 2649 lipid_summary = lipid_summary.set_index("mf_id") 2650 2651 return lipid_summary 2652 2653 def to_report(self, molecular_metadata=None): 2654 """Create a report of the mass features and their annotations. 2655 2656 Parameters 2657 ---------- 2658 molecular_metadata : dict, optional 2659 The molecular metadata. Default is None. 2660 2661 Returns 2662 ------- 2663 DataFrame 2664 The report of the mass features and their annotations. 2665 2666 Notes 2667 ----- 2668 The report will contain the mass features and their annotations from MS1 and MS2 (if available). 2669 """ 2670 # Get mass feature dataframe 2671 mf_report = self.mass_spectra.mass_features_to_df() 2672 mf_report = mf_report.reset_index(drop=False) 2673 2674 # Get and clean ms1 annotation dataframe 2675 ms1_annot_report = self.mass_spectra.mass_features_ms1_annot_to_df().copy() 2676 ms1_annot_report = self.clean_ms1_report(ms1_annot_report) 2677 ms1_annot_report = ms1_annot_report.reset_index(drop=False) 2678 2679 # Get, summarize, and clean ms2 annotation dataframe 2680 ms2_annot_report = self.mass_spectra.mass_features_ms2_annot_to_df( 2681 molecular_metadata=molecular_metadata 2682 ) 2683 if ms2_annot_report is not None and molecular_metadata is not None: 2684 ms2_annot_report = self.summarize_lipid_report(ms2_annot_report) 2685 ms2_annot_report = self.clean_ms2_report(ms2_annot_report) 2686 ms2_annot_report = ms2_annot_report.dropna(axis=1, how="all") 2687 ms2_annot_report = ms2_annot_report.reset_index(drop=False) 2688 report = self.combine_reports( 2689 mf_report=mf_report, 2690 ms1_annot_report=ms1_annot_report, 2691 ms2_annot_report=ms2_annot_report, 2692 ) 2693 return report 2694 2695 2696class LCMSCollectionExport: 2697 """A class to export an LCMS collection to HDF5 or Parquet format. 2698 2699 This class provides methods to export collection-level data from multi-sample LC-MS 2700 experiments to HDF5 or Parquet files. It handles the export of metadata, retention time 2701 alignments, cluster assignments, and induced mass features (gap-filled features) across 2702 the collection. 2703 2704 The exporter is designed to work with LCMSCollection objects and complements the individual 2705 LCMSExport class by focusing on collection-wide data rather than individual sample data. 2706 2707 Parameters 2708 ---------- 2709 out_file_path : str | Path 2710 The output file path, do not include the file extension. The .hdf5 / .parquet 2711 extensions are added automatically as needed. 2712 mass_spectra_collection : LCMSCollection 2713 The LCMS collection object containing multiple LCMS samples with processed mass features, 2714 alignments, and clustering information. 2715 2716 Attributes 2717 ---------- 2718 out_file_path : Path 2719 The output file path as a Path object. 2720 mass_spectra_collection : LCMSCollection 2721 The LCMS collection object to be exported. 2722 2723 Methods 2724 ------- 2725 export_to_hdf5(overwrite=False) 2726 Export the LCMS collection to an HDF5 file with collection-level data. 2727 export_to_parquet(overwrite=False) 2728 Export collection-level tabular datasets to Parquet files. 2729 2730 Notes 2731 ----- 2732 This class exports collection-level data including: 2733 - Sample manifest (metadata about all samples in the collection) 2734 - Retention time alignment data (if RT alignment has been performed) 2735 - Cluster assignments (consensus mass feature groupings across samples) 2736 - Induced mass features (gap-filled features saved to individual LCMS object HDF5 files) 2737 2738 Individual sample data (mass spectra, mass features, EICs, etc.) should be exported 2739 separately using the LCMSExport class for each LCMS object in the collection. 2740 2741 Examples 2742 -------- 2743 Export a collection after clustering and gap-filling: 2744 2745 >>> from corems.mass_spectra.output.export import LCMSCollectionExporter 2746 >>> exporter = LCMSCollectionExporter("my_collection", lcms_collection) 2747 >>> exporter.export_to_hdf5(overwrite=True) 2748 2749 The resulting HDF5 file will contain collection-level metadata and can be used 2750 to reconstruct the collection state for further analysis. 2751 2752 See Also 2753 -------- 2754 LCMSExport : Export individual LCMS objects to HDF5 2755 LCMSCollection : The collection object being exported 2756 """ 2757 2758 def __init__(self, out_file_path, mass_spectra_collection): 2759 self.out_file_path = Path(out_file_path) 2760 self.mass_spectra_collection = mass_spectra_collection 2761 2762 def export_to_hdf5( 2763 self, 2764 overwrite=False, 2765 save_parameters=True, 2766 parameter_format="toml", 2767 update_lcms_objects=True, 2768 ): 2769 """Export the LCMS collection to an HDF5 file. 2770 2771 This method saves the collection-level data to an HDF5 file, including: 2772 - Basic metadata (date, folder location, gap-filling status) 2773 - Sample manifest 2774 - Retention time alignments (if available) 2775 - Cluster assignments (if available) 2776 - Induced mass features for each LCMS object (if gap-filling was performed) 2777 2778 Individual LCMS objects in the collection are not exported by this method. 2779 Use LCMSExport for exporting individual LCMS objects. 2780 2781 Parameters 2782 ---------- 2783 overwrite : bool, optional 2784 If True, overwrites the output file if it already exists and replaces 2785 existing groups within the HDF5 file. If False, appends new data to 2786 existing file without overwriting existing groups. Default is False. 2787 save_parameters : bool, optional 2788 If True, saves the collection-level parameters to a separate file in the specified format. 2789 Default is True. 2790 parameter_format : str, optional 2791 The format for saving parameters, either "json" or "toml". Default is "toml". 2792 update_lcms_objects : bool, optional 2793 If True, updates the individual LCMS object HDF5 files with new raw file locations and any additional 2794 information produced during the processing of the collection (e.g. cluster mass feature associations). Default is True. 2795 2796 Notes 2797 ----- 2798 The HDF5 file structure includes: 2799 - Attributes: date_utc, lcms_objects_folder, missing_mass_features_searched, manifest 2800 - Groups: rt_alignments, cluster_assignments (if available) 2801 2802 Induced mass features are saved to the individual LCMS object HDF5 files 2803 within the .corems folder structure, not in the collection-level HDF5 file. 2804 2805 Examples 2806 -------- 2807 >>> exporter = LCMSCollectionExporter("my_collection", lcms_collection) 2808 >>> exporter.export_to_hdf5(overwrite=True) 2809 """ 2810 if overwrite: 2811 if self.out_file_path.with_suffix(".hdf5").exists(): 2812 self.out_file_path.with_suffix(".hdf5").unlink() 2813 2814 with h5py.File(self.out_file_path.with_suffix(".hdf5"), "a") as hdf_handle: 2815 # Add basic attributes to the HDF5 file, always overwrite these 2816 timenow = str(datetime.now(timezone.utc).strftime("%d/%m/%Y %H:%M:%S %Z")) 2817 hdf_handle.attrs["date_utc"] = timenow 2818 hdf_handle.attrs["lcms_objects_folder"] = str( 2819 self.mass_spectra_collection.collection_parser.folder_location 2820 ) 2821 hdf_handle.attrs["missing_mass_features_searched"] = ( 2822 self.mass_spectra_collection.missing_mass_features_searched 2823 ) 2824 hdf_handle.attrs["rt_aligned"] = self.mass_spectra_collection.rt_aligned 2825 hdf_handle.attrs["rt_alignment_attempted"] = ( 2826 self.mass_spectra_collection.rt_alignment_attempted 2827 ) 2828 2829 # Add the manifest to the HDF5 file, always overwrite this 2830 hdf_handle.attrs["manifest"] = self._convert_manifest_to_json() 2831 2832 # Save retention time alignments if they exist, only overwrite if specified 2833 self._save_rt_alignments_to_hdf5(hdf_handle, overwrite) 2834 2835 # Save cluster assignments if they exist, only overwrite if specified 2836 self._save_cluster_assignments_to_hdf5(hdf_handle, overwrite) 2837 2838 # Save consensus feature-group labels if present 2839 self._save_feature_group_labels_to_hdf5(hdf_handle, overwrite) 2840 2841 # Save new raw file locations to each LCMS object's HDF5 file if needed 2842 if ( 2843 hasattr(self.mass_spectra_collection, "raw_files_relocated") 2844 and self.mass_spectra_collection.raw_files_relocated 2845 ): 2846 self._update_raw_file_locations_in_hdf5() 2847 2848 # Save induced mass features to the collection with associations to each individual, only if lcms_collection.missing_mass_features_searched is True 2849 if self.mass_spectra_collection.missing_mass_features_searched: 2850 self._save_induced_mass_features_to_hdf5(overwrite) 2851 # Save EICs for induced mass features at collection level 2852 self._save_induced_eics_to_hdf5(overwrite) 2853 2854 # Build cluster mass feature map to know which features to update 2855 # This uses the same logic as process_consensus_features to determine loaded features 2856 cluster_mf_map = self._build_cluster_mf_map() 2857 2858 # Save updated mass features for each LCMS object 2859 # This implements selective update: only loaded features are updated, non-cluster features are preserved 2860 if update_lcms_objects: 2861 self._save_lcms_objects_to_hdf5(cluster_mf_map, overwrite) 2862 2863 # Save collection-level parameters as separate file 2864 if save_parameters: 2865 # Check if parameter_format is valid 2866 if parameter_format not in ["json", "toml"]: 2867 raise ValueError("parameter_format must be 'json' or 'toml'") 2868 2869 if parameter_format == "json": 2870 dump_lcms_collection_settings_json( 2871 filename=self.out_file_path.with_suffix(".json"), 2872 lcms_collection=self.mass_spectra_collection, 2873 ) 2874 elif parameter_format == "toml": 2875 dump_lcms_collection_settings_toml( 2876 filename=self.out_file_path.with_suffix(".toml"), 2877 lcms_collection=self.mass_spectra_collection, 2878 ) 2879 2880 def export_to_parquet( 2881 self, overwrite=False, save_parameters=True, parameter_format="toml" 2882 ): 2883 """Export collection-level LC-MS datasets to Parquet files. 2884 2885 Writes next to ``out_file_path`` (same stem as the HDF5 export): 2886 2887 - ``{stem}_manifest.parquet`` — sample manifest 2888 - ``{stem}_cluster_assignments.parquet`` — cluster labels (if present) 2889 - ``{stem}_mass_features.parquet`` — collection mass-features table (if present) 2890 - ``{stem}_rt_alignments.parquet`` — RT alignment curves (if aligned) 2891 - parameters as JSON or TOML when ``save_parameters`` is True 2892 2893 Parameters 2894 ---------- 2895 overwrite : bool, optional 2896 Whether to overwrite existing Parquet files. Default is False. 2897 save_parameters : bool, optional 2898 Whether to save collection parameters. Default is True. 2899 parameter_format : str, optional 2900 Parameter file format, ``'json'`` or ``'toml'``. Default is ``'toml'``. 2901 2902 Notes 2903 ----- 2904 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 2905 This is a tabular companion to ``export_to_hdf5``; it does not replace 2906 per-sample HDF5 archives or gap-fill side effects on individual LCMS files. 2907 """ 2908 stem = self.out_file_path 2909 2910 def _write(df, suffix): 2911 if df is None or df.empty: 2912 return 2913 out_path = Path(f"{stem}_{suffix}").with_suffix(".parquet") 2914 if out_path.exists() and not overwrite: 2915 return 2916 LCMSExport._prepare_df_for_parquet(df).to_parquet(out_path, index=False) 2917 2918 # Manifest — prefer the tidy samples-as-rows dataframe 2919 manifest_df = getattr(self.mass_spectra_collection, "manifest_dataframe", None) 2920 if manifest_df is None or getattr(manifest_df, "empty", True): 2921 manifest = getattr( 2922 self.mass_spectra_collection, "manifest", None 2923 ) or getattr( 2924 getattr(self.mass_spectra_collection, "collection_parser", None), 2925 "manifest", 2926 None, 2927 ) 2928 if isinstance(manifest, DataFrame): 2929 manifest_df = manifest 2930 elif isinstance(manifest, dict) and manifest: 2931 try: 2932 # Collection manifests are sample -> attrs; transpose to rows 2933 manifest_df = pd.DataFrame(manifest).T 2934 except ValueError: 2935 manifest_df = DataFrame( 2936 [{"manifest_json": self._convert_manifest_to_json()}] 2937 ) 2938 else: 2939 manifest_df = None 2940 2941 if manifest_df is not None and not manifest_df.empty: 2942 _write(manifest_df.reset_index(drop=False), "manifest") 2943 2944 # Cluster assignments / mass features table 2945 mf_df = getattr(self.mass_spectra_collection, "mass_features_dataframe", None) 2946 if mf_df is not None and not mf_df.empty: 2947 _write(mf_df.reset_index(drop=False), "mass_features") 2948 if "cluster" in mf_df.columns: 2949 cluster_df = mf_df[["cluster"]].copy().reset_index(drop=False) 2950 _write(cluster_df, "cluster_assignments") 2951 2952 # RT alignments as long-format table 2953 if getattr(self.mass_spectra_collection, "rt_aligned", False) and getattr( 2954 self.mass_spectra_collection, "rt_alignments", None 2955 ): 2956 frames = [] 2957 for ( 2958 sample_idx, 2959 alignment_data, 2960 ) in self.mass_spectra_collection.rt_alignments.items(): 2961 arr = np.asarray(alignment_data) 2962 if arr.ndim == 1: 2963 frames.append( 2964 DataFrame( 2965 { 2966 "sample_idx": sample_idx, 2967 "point": np.arange(len(arr)), 2968 "value": arr, 2969 } 2970 ) 2971 ) 2972 elif arr.ndim == 2 and arr.shape[1] >= 2: 2973 frames.append( 2974 DataFrame( 2975 { 2976 "sample_idx": sample_idx, 2977 "rt_original": arr[:, 0], 2978 "rt_aligned": arr[:, 1], 2979 } 2980 ) 2981 ) 2982 else: 2983 frames.append( 2984 DataFrame( 2985 { 2986 "sample_idx": [sample_idx], 2987 "alignment_json": [json.dumps(arr.tolist())], 2988 } 2989 ) 2990 ) 2991 if frames: 2992 _write(pd.concat(frames, ignore_index=True), "rt_alignments") 2993 2994 if save_parameters: 2995 if parameter_format not in ["json", "toml"]: 2996 raise ValueError("parameter_format must be 'json' or 'toml'") 2997 if parameter_format == "json": 2998 dump_lcms_collection_settings_json( 2999 filename=self.out_file_path.with_suffix(".json"), 3000 lcms_collection=self.mass_spectra_collection, 3001 ) 3002 elif parameter_format == "toml": 3003 dump_lcms_collection_settings_toml( 3004 filename=self.out_file_path.with_suffix(".toml"), 3005 lcms_collection=self.mass_spectra_collection, 3006 ) 3007 3008 def _save_rt_alignments_to_hdf5(self, hdf_handle, overwrite): 3009 """Save retention time alignments to HDF5 file.""" 3010 # If no rt_alignments, return early 3011 if not self.mass_spectra_collection.rt_aligned: 3012 return 3013 3014 # If rt_alignments exist, save them 3015 if self.mass_spectra_collection.rt_aligned: 3016 group_name = "rt_alignments" 3017 # grab dictionary of rt_alignments 3018 rt_alignments = self.mass_spectra_collection.rt_alignments 3019 3020 if rt_alignments: 3021 # Check if group exists and handle overwrite logic 3022 if group_name in hdf_handle: 3023 if not overwrite: 3024 return 3025 del hdf_handle[group_name] 3026 3027 grp = hdf_handle.create_group(group_name) 3028 3029 # Save each alignment as a dataset 3030 for sample_idx, alignment_data in rt_alignments.items(): 3031 grp.create_dataset(str(sample_idx), data=alignment_data) 3032 3033 def _convert_manifest_to_json(self): 3034 """Clean the manifest for export to HDF5.""" 3035 manifest = self.mass_spectra_collection.collection_parser.manifest 3036 3037 # Process the manifest to convert numpy.bool_ or bool values for the 'use_rt_alignment' key 3038 def convert_bool_values(data): 3039 if isinstance(data, dict): 3040 # Process each key-value pair recursively 3041 return { 3042 k: ( 3043 int(v) 3044 if k == "use_rt_alignment" and isinstance(v, (bool, np.bool_)) 3045 else convert_bool_values(v) 3046 ) 3047 for k, v in data.items() 3048 } 3049 elif isinstance(data, list): 3050 # Recursively process lists 3051 return [convert_bool_values(item) for item in data] 3052 else: 3053 # Return non-dict/list types unchanged 3054 return data 3055 3056 # Clean the whole manifest 3057 cleaned_manifest = convert_bool_values(manifest) 3058 3059 # Serialize the cleaned manifest into JSON format 3060 json_manifest = json.dumps(cleaned_manifest) 3061 return json_manifest 3062 3063 def _save_cluster_assignments_to_hdf5(self, hdf_handle, overwrite): 3064 """Save cluster assignments to HDF5 file.""" 3065 # Check if column "cluster" is present in self.mass_features_dataframe 3066 if "cluster" in self.mass_spectra_collection.mass_features_dataframe.columns: 3067 group_name = "cluster_assignments" 3068 cluster_assignments = self.mass_spectra_collection.mass_features_dataframe[ 3069 ["cluster"] 3070 ].copy() 3071 3072 # Check if group exists and handle overwrite logic 3073 if group_name in hdf_handle: 3074 if not overwrite: 3075 return 3076 del hdf_handle[group_name] 3077 3078 grp = hdf_handle.create_group(group_name) 3079 3080 # Save the index, converting strings to bytes. 3081 # Avoid Index.astype(str).values under pandas 3 string/CoW semantics; 3082 # build a plain numpy bytes array instead. 3083 index_bytes = np.asarray( 3084 [str(x).encode("utf-8") for x in cluster_assignments.index], 3085 dtype="S", 3086 ) 3087 grp.create_dataset("index", data=index_bytes) 3088 3089 # Save the "cluster" column 3090 grp.create_dataset( 3091 "cluster", data=cluster_assignments["cluster"].to_numpy() 3092 ) 3093 3094 def _save_feature_group_labels_to_hdf5(self, hdf_handle, overwrite): 3095 """Save cluster-indexed feature-group labels to the collection HDF5.""" 3096 from corems.mass_spectra.calc.feature_grouping import GROUP_COLUMNS 3097 3098 labels = getattr(self.mass_spectra_collection, "feature_group_dataframe", None) 3099 if labels is None or len(labels) == 0: 3100 return 3101 3102 group_name = "feature_group_labels" 3103 if group_name in hdf_handle: 3104 if not overwrite: 3105 return 3106 del hdf_handle[group_name] 3107 3108 grp = hdf_handle.create_group(group_name) 3109 index_vals = np.asarray( 3110 [int(x) for x in labels.index], 3111 dtype=np.int64, 3112 ) 3113 grp.create_dataset("cluster", data=index_vals) 3114 3115 for col in GROUP_COLUMNS: 3116 if col not in labels.columns: 3117 continue 3118 series = labels[col] 3119 if col in ("feature_group_id", "mono_cluster_id"): 3120 # Nullable ints → float with NaN for missing 3121 data = series.astype("Float64").to_numpy(dtype=float) 3122 grp.create_dataset(col, data=data) 3123 else: 3124 # Object / string columns 3125 encoded = np.asarray( 3126 [ 3127 ( 3128 b"" 3129 if v is None 3130 or (isinstance(v, float) and np.isnan(v)) 3131 or (hasattr(pd, "isna") and pd.isna(v)) 3132 else str(v).encode("utf-8") 3133 ) 3134 for v in series.tolist() 3135 ], 3136 dtype="S", 3137 ) 3138 grp.create_dataset(col, data=encoded) 3139 3140 def _build_cluster_mf_map(self): 3141 """Build a mapping of which mass features should be saved for each sample. 3142 3143 This uses the same logic as process_consensus_features to determine which 3144 mass features were loaded and should be updated in HDF5 files. 3145 3146 Returns 3147 ------- 3148 dict 3149 Dictionary mapping sample_id to list of tuples (mf_id, cluster_id). 3150 Only includes samples that have loaded representative features. 3151 Returns empty dict if no clusters exist. 3152 3153 Notes 3154 ----- 3155 This follows the DRY principle by using the same get_sample_mf_map_for_representatives 3156 method used by process_consensus_features and ReadSavedLCMSCollection. 3157 """ 3158 # Check if clusters exist 3159 if ( 3160 "cluster" 3161 not in self.mass_spectra_collection.mass_features_dataframe.columns 3162 ): 3163 return {} 3164 3165 # Check if cluster_summary_dataframe exists (needed by get_sample_mf_map_for_representatives) 3166 if ( 3167 not hasattr(self.mass_spectra_collection, "cluster_summary_dataframe") 3168 or self.mass_spectra_collection.cluster_summary_dataframe is None 3169 ): 3170 return {} 3171 3172 # Use the same DRY helper method that process_consensus_features uses 3173 # This ensures consistency across the codebase 3174 cluster_mf_map = ( 3175 self.mass_spectra_collection.get_sample_mf_map_for_representatives( 3176 include_cluster_id=True 3177 ) 3178 ) 3179 3180 return cluster_mf_map 3181 3182 def _update_raw_file_locations_in_hdf5(self): 3183 """Update raw file locations in each LCMS object's HDF5 file. 3184 3185 This method updates the 'original_file_location' attribute in each LCMS object's 3186 HDF5 file to reflect the new raw file location after files have been relocated. 3187 """ 3188 for lcms_obj in self.mass_spectra_collection: 3189 # Get the HDF5 file path for this LCMS object 3190 hdf5_path = lcms_obj.file_location.with_suffix(".hdf5") 3191 3192 if hdf5_path.exists(): 3193 with h5py.File(hdf5_path, "a") as hdf_handle: 3194 # Update the original_file_location attribute 3195 if "original_file_location" in hdf_handle.attrs: 3196 hdf_handle.attrs["original_file_location"] = str( 3197 lcms_obj.raw_file_location 3198 ) 3199 # If the attribute does not exist, create it 3200 else: 3201 hdf_handle.attrs.create( 3202 "original_file_location", str(lcms_obj.raw_file_location) 3203 ) 3204 3205 def _save_induced_mass_features_to_hdf5(self, overwrite): 3206 """Save induced mass features to the collection HDF5 file. 3207 3208 Induced mass features are gap-filled features that only exist at the collection level. 3209 They are saved with full detail (all attributes and datasets) in the collection HDF5 file 3210 and distributed to individual LCMS objects when the collection is loaded. 3211 3212 The induced mass features are stored in the collection's induced_mass_features_dataframe 3213 and are regenerated as LCMSMassFeature objects for saving. 3214 3215 Parameters 3216 ---------- 3217 overwrite : bool 3218 If True, overwrites existing induced mass features group. If False, skips if group exists. 3219 """ 3220 # Check if we have any induced mass features to save 3221 if ( 3222 self.mass_spectra_collection.induced_mass_features_dataframe is None 3223 or self.mass_spectra_collection.induced_mass_features_dataframe.empty 3224 ): 3225 return 3226 3227 # Open the collection HDF5 file to save induced mass features 3228 with h5py.File(self.out_file_path.with_suffix(".hdf5"), "a") as hdf_handle: 3229 group_name = "induced_mass_features" 3230 3231 # Check if group exists and handle overwrite logic 3232 if group_name in hdf_handle: 3233 if not overwrite: 3234 return 3235 del hdf_handle[group_name] 3236 3237 # Create top-level group for induced mass features 3238 imf_group = hdf_handle.create_group(group_name) 3239 3240 # Get the induced mass features dataframe 3241 induced_df = self.mass_spectra_collection.induced_mass_features_dataframe 3242 3243 # Get unique sample IDs from the dataframe 3244 sample_ids = induced_df["sample_id"].unique() 3245 3246 # Iterate through each sample and save its induced mass features 3247 for sample_id in sample_ids: 3248 # Filter dataframe to this sample 3249 sample_df = induced_df[induced_df["sample_id"] == sample_id].copy() 3250 3251 if sample_df.empty: 3252 continue 3253 3254 # Regenerate mass features from the dataframe 3255 regenerated_features = self._regenerate_mass_features_from_sample_df( 3256 sample_df, sample_id 3257 ) 3258 3259 if not regenerated_features: 3260 continue 3261 3262 # Create a subgroup for this sample's induced mass features 3263 sample_group = imf_group.create_group(str(sample_id)) 3264 3265 # Use the static helper method from LCMSExport to save the mass features 3266 LCMSExport._save_mass_features_dict_to_hdf5( 3267 regenerated_features, sample_group, overwrite=overwrite 3268 ) 3269 3270 def _save_induced_eics_to_hdf5(self, overwrite): 3271 """Save EICs for induced mass features to the collection HDF5 file. 3272 3273 Induced mass features are gap-filled features created during process_consensus_features. 3274 Their associated EICs need to be saved at the collection level so they can be reloaded. 3275 3276 The induced mass features are identified from the collection's induced_mass_features_dataframe, 3277 and their EICs are retrieved from the individual LCMS objects. 3278 3279 Parameters 3280 ---------- 3281 overwrite : bool 3282 If True, overwrites existing induced EICs group. If False, skips if group exists. 3283 """ 3284 # Check if we have any induced mass features to save 3285 if ( 3286 self.mass_spectra_collection.induced_mass_features_dataframe is None 3287 or self.mass_spectra_collection.induced_mass_features_dataframe.empty 3288 ): 3289 return 3290 3291 # Open the collection HDF5 file to save induced EICs 3292 with h5py.File(self.out_file_path.with_suffix(".hdf5"), "a") as hdf_handle: 3293 group_name = "induced_eics" 3294 3295 # Check if group exists and handle overwrite logic 3296 if group_name in hdf_handle: 3297 if not overwrite: 3298 return 3299 del hdf_handle[group_name] 3300 3301 # Create top-level group for induced EICs 3302 induced_eics_group = hdf_handle.create_group(group_name) 3303 3304 # Get the induced mass features dataframe 3305 induced_df = self.mass_spectra_collection.induced_mass_features_dataframe 3306 3307 # Get unique sample IDs from the dataframe 3308 sample_ids = induced_df["sample_id"].unique() 3309 3310 # Iterate through each sample and save EICs for its induced mass features 3311 for sample_id in sample_ids: 3312 lcms_obj = self.mass_spectra_collection[sample_id] 3313 3314 # Filter dataframe to this sample 3315 sample_df = induced_df[induced_df["sample_id"] == sample_id].copy() 3316 3317 if sample_df.empty: 3318 continue 3319 3320 # Collect EICs for induced mass features using _eic_mz from dataframe 3321 induced_eics = {} 3322 for _, row in sample_df.iterrows(): 3323 # Get the EIC m/z from the dataframe 3324 eic_mz = row.get("_eic_mz") 3325 3326 if eic_mz is not None and pd.notna(eic_mz): 3327 # Try to get the EIC from the LCMS object 3328 if ( 3329 hasattr(lcms_obj, "eics") 3330 and lcms_obj.eics 3331 and eic_mz in lcms_obj.eics 3332 ): 3333 induced_eics[eic_mz] = lcms_obj.eics[eic_mz] 3334 3335 if not induced_eics: 3336 continue 3337 3338 # Create a subgroup for this sample's induced EICs 3339 sample_group = induced_eics_group.create_group(str(sample_id)) 3340 3341 # Use the static helper method from LCMSExport to save the EICs 3342 LCMSExport._save_eics_dict_to_hdf5( 3343 induced_eics, sample_group, overwrite 3344 ) 3345 3346 def _regenerate_mass_features_from_sample_df(self, sample_df, sample_id): 3347 """Regenerate induced mass features from a sample-specific dataframe. 3348 3349 This method creates LCMSMassFeature objects from rows in the induced_mass_features_dataframe 3350 for a specific sample. The regenerated features are used for saving to HDF5. 3351 3352 Parameters 3353 ---------- 3354 sample_df : pd.DataFrame 3355 DataFrame containing induced mass features for a specific sample. 3356 sample_id : int 3357 The sample ID (index in the collection). 3358 3359 Returns 3360 ------- 3361 dict 3362 Dictionary of regenerated LCMSMassFeature objects keyed by feature ID. 3363 """ 3364 from corems.chroma_peak.factory.chroma_peak_classes import LCMSMassFeature 3365 3366 if sample_df.empty: 3367 return {} 3368 3369 # Get the corresponding LCMS object for proper parent reference 3370 lcms_obj = self.mass_spectra_collection[sample_id] 3371 3372 # Regenerate mass features from the dataframe 3373 regenerated_features = {} 3374 3375 for _, row in sample_df.iterrows(): 3376 # Extract the original ID from mf_id (format: c{cluster}_{index}_i) 3377 # This is the ID used in lcms_obj.induced_mass_features dict 3378 original_id = row["mf_id"] 3379 3380 # Create a new LCMSMassFeature with proper parent reference 3381 # Note: dataframe uses 'scan_time' but __init__ parameter is 'retention_time' 3382 mass_feature = LCMSMassFeature( 3383 lcms_parent=lcms_obj, 3384 mz=row["mz"], 3385 retention_time=row["scan_time"], # Column is 'scan_time' in dataframe 3386 intensity=row["intensity"], 3387 apex_scan=int(row["apex_scan"]), 3388 persistence=row.get("persistence", None) 3389 if "persistence" in row 3390 else None, 3391 id=original_id, # Use the original string ID from gap-filling 3392 ) 3393 3394 # Set additional attributes dynamically from dataframe columns 3395 # Skip columns already handled in __init__ or structural metadata 3396 skip_cols = { 3397 "sample_id", 3398 "mf_id", 3399 "mz", 3400 "scan_time", 3401 "scan_time_aligned", 3402 "intensity", 3403 "apex_scan", 3404 "persistence", 3405 } 3406 3407 # Iterate through all columns and set via property setters 3408 for col_name in row.index: 3409 if col_name in skip_cols: 3410 continue 3411 value = row[col_name] 3412 try: 3413 if pd.isna(value): 3414 continue 3415 except (TypeError, ValueError): 3416 pass # value is array-like; not NA, proceed 3417 3418 # Set via property (public interface handles private attributes) 3419 # Don't save empty lists 3420 if isinstance(value, list) and len(value) == 0: 3421 continue 3422 try: 3423 setattr(mass_feature, col_name, value) 3424 except (AttributeError, TypeError): 3425 pass # Skip attributes that don't exist or can't be set 3426 3427 # Set cluster_index if present 3428 if "cluster" in row and pd.notna(row["cluster"]): 3429 mass_feature.cluster_index = int(row["cluster"]) 3430 3431 regenerated_features[mass_feature.id] = mass_feature 3432 3433 return regenerated_features 3434 3435 def _save_lcms_objects_to_hdf5(self, cluster_mf_map, overwrite): 3436 """Save updated mass features for each LCMS object. 3437 3438 This method implements a "selective update" strategy for mass features: 3439 - For mass features specified in cluster_mf_map (loaded representatives), we selectively 3440 update them by deleting their old entries and re-saving with new attributes. 3441 - Non-cluster features (not loaded) are never touched/overwritten. 3442 3443 Note: EICs are NOT saved here. Induced feature EICs are saved at the collection level. 3444 3445 Parameters 3446 ---------- 3447 cluster_mf_map : dict 3448 Dictionary mapping sample_id to list of tuples (mf_id, cluster_id). 3449 This explicitly defines which mass features should be updated. 3450 overwrite : bool 3451 If True, allows overwriting of existing data. If False, skips if data exists. 3452 """ 3453 for sample_id, lcms_obj in enumerate(self.mass_spectra_collection): 3454 hdf5_path = lcms_obj.file_location.with_suffix(".hdf5") 3455 3456 if not hdf5_path.exists(): 3457 # If HDF5 doesn't exist, we can't do selective update, raise error 3458 raise FileNotFoundError( 3459 f"HDF5 file for LCMS object {lcms_obj.sample_name} not found at {hdf5_path}" 3460 ) 3461 3462 # Check if this sample has any loaded features in the map 3463 if sample_id not in cluster_mf_map or not cluster_mf_map[sample_id]: 3464 # Nothing loaded for this sample, nothing to update 3465 continue 3466 3467 # Extract mf_ids from the map (cluster_mf_map contains tuples of (mf_id, cluster_id)) 3468 mf_ids_to_update = [ 3469 mf_id for mf_id, cluster_id in cluster_mf_map[sample_id] 3470 ] 3471 3472 # Perform selective update of mass features 3473 self._selective_update_mass_features( 3474 lcms_obj, hdf5_path, mf_ids_to_update, overwrite 3475 ) 3476 3477 # Save any new mass spectra that were added during processing 3478 self._save_new_mass_spectra(lcms_obj, hdf5_path, overwrite) 3479 3480 def _save_new_mass_spectra(self, lcms_obj, hdf5_path, overwrite): 3481 """Save new mass spectra that were added during processing. 3482 3483 This method checks what mass spectra are in lcms_obj._ms and saves any 3484 that aren't already in the HDF5 file's mass_spectra group. Uses the 3485 existing add_mass_spectrum_to_hdf5 method for consistency with original 3486 export logic. 3487 3488 Parameters 3489 ---------- 3490 lcms_obj : LCMSBase 3491 The LCMS object with potentially new mass spectra. 3492 hdf5_path : Path 3493 Path to the HDF5 file. 3494 overwrite : bool 3495 If True, allows overwriting existing spectra. 3496 """ 3497 # Check if there are any mass spectra to save 3498 if not hasattr(lcms_obj, "_ms") or not lcms_obj._ms: 3499 return 3500 3501 # Create an LCMS exporter instance for this LCMS object 3502 # This gives us access to add_mass_spectrum_to_hdf5 method inherited from HighResMassSpecExport 3503 # Turn hdf5_path into str without suffix for LCMSExport 3504 hdf5_path_str = str(hdf5_path.with_suffix("")) 3505 exporter = LCMSExport(out_file_path=hdf5_path_str, mass_spectra=lcms_obj) 3506 3507 # Open HDF5 file and check existing mass spectra 3508 with h5py.File(hdf5_path, "a") as hdf_handle: 3509 # Create mass_spectra group if it doesn't exist 3510 if "mass_spectra" not in hdf_handle: 3511 ms_group = hdf_handle.create_group("mass_spectra") 3512 existing_scan_numbers = set() 3513 else: 3514 ms_group = hdf_handle["mass_spectra"] 3515 existing_scan_numbers = set(int(k) for k in ms_group.keys()) 3516 3517 # Find new mass spectra (in _ms but not in HDF5) 3518 new_scan_numbers = set(lcms_obj._ms.keys()) - existing_scan_numbers 3519 3520 if not new_scan_numbers: 3521 return 3522 3523 # Save new mass spectra using existing add_mass_spectrum_to_hdf5 method 3524 export_profile = lcms_obj.parameters.lc_ms.export_profile_spectra 3525 for scan_number in new_scan_numbers: 3526 mass_spec = lcms_obj._ms[scan_number] 3527 scan_group_name = str(scan_number) 3528 3529 # Delete existing group if overwrite is True 3530 if scan_group_name in ms_group and overwrite: 3531 del ms_group[scan_group_name] 3532 elif scan_group_name in ms_group: 3533 continue 3534 3535 # Use the existing method from HighResMassSpecExport 3536 exporter.add_mass_spectrum_to_hdf5( 3537 hdf_handle=hdf_handle, 3538 mass_spectrum=mass_spec, 3539 group_key=scan_group_name, 3540 mass_spectra_group=ms_group, 3541 export_raw=export_profile, 3542 ) 3543 3544 def _selective_update_mass_features( 3545 self, lcms_obj, hdf5_path, mf_ids_to_update, overwrite 3546 ): 3547 """Selectively update mass features in HDF5 file. 3548 3549 This method deletes only the mass features specified in mf_ids_to_update, 3550 then re-saves them with their potentially updated attributes. Non-cluster features 3551 in the HDF5 file are left untouched. 3552 3553 Parameters 3554 ---------- 3555 lcms_obj : LCMSBase 3556 The LCMS object with mass features to update. 3557 hdf5_path : Path 3558 Path to the HDF5 file. 3559 mf_ids_to_update : list of int 3560 List of mass feature IDs that should be updated. This explicitly defines 3561 which features were loaded and should be saved. 3562 overwrite : bool 3563 If True, allows overwriting. If False, skips if group exists. 3564 """ 3565 if not mf_ids_to_update: 3566 return 3567 3568 # Open HDF5 file and delete specified feature IDs, then re-save 3569 with h5py.File(hdf5_path, "a") as hdf_handle: 3570 if "mass_features" not in hdf_handle: 3571 return 3572 3573 mf_group = hdf_handle["mass_features"] 3574 3575 # Delete features that are being updated 3576 for feature_id in mf_ids_to_update: 3577 feature_id_str = str(feature_id) 3578 if feature_id_str in mf_group: 3579 del mf_group[feature_id_str] 3580 3581 # Re-save updated features (only those that exist in mass_features dict) 3582 updated_features = { 3583 mf.id: mf 3584 for mf in lcms_obj.mass_features.values() 3585 if mf.id in mf_ids_to_update 3586 } 3587 3588 if updated_features: 3589 LCMSExport._save_mass_features_dict_to_hdf5( 3590 updated_features, mf_group, overwrite=overwrite 3591 )
ion_type_dict =
{'M+': {'add': {}, 'sub': {}, 'polarity': 'positive'}, '[M]+': {'add': {}, 'sub': {}, 'polarity': 'positive'}, 'protonated': {'add': {'H': 1}, 'sub': {}, 'polarity': 'positive'}, '[M+H]+': {'add': {'H': 1}, 'sub': {}, 'polarity': 'positive'}, '[M+2H]2+': {'add': {'H': 2}, 'sub': {}, 'polarity': 'positive'}, '[M+3H]3+': {'add': {'H': 3}, 'sub': {}, 'polarity': 'positive'}, '[M+NH4]+': {'add': {'N': 1, 'H': 4}, 'sub': {}, 'polarity': 'positive'}, '[M+Na]+': {'add': {'Na': 1}, 'sub': {}, 'polarity': 'positive'}, '[M+K]+': {'add': {'K': 1}, 'sub': {}, 'polarity': 'positive'}, '[M+2Na]2+': {'add': {'Na': 2}, 'sub': {}, 'polarity': 'positive'}, '[M+H+Na]2+': {'add': {'H': 1, 'Na': 1}, 'sub': {}, 'polarity': 'positive'}, '[M+H+K]2+': {'add': {'H': 1, 'K': 1}, 'sub': {}, 'polarity': 'positive'}, '[M+2Na+Cl]+': {'add': {'Na': 2, 'Cl': 1}, 'sub': {}, 'polarity': 'positive'}, '[M+2Na-H]+': {'add': {'Na': 2}, 'sub': {'H': 1}, 'polarity': 'positive'}, '[M-H+2Na]+': {'add': {'Na': 2}, 'sub': {'H': 1}, 'polarity': 'positive'}, '[M+C2H3Na2O2]+': {'add': {'C': 2, 'H': 3, 'Na': 2, 'O': 2}, 'sub': {}, 'polarity': 'positive'}, '[M+C4H10N3]+': {'add': {'C': 4, 'H': 10, 'N': 3}, 'sub': {}, 'polarity': 'positive'}, '[M+NH4+ACN]+': {'add': {'C': 2, 'H': 7, 'N': 2}, 'sub': {}, 'polarity': 'positive'}, '[M+H-H2O]+': {'add': {}, 'sub': {'H': 1, 'O': 1}, 'polarity': 'positive'}, '[M+H-2H2O]+': {'add': {}, 'sub': {'H': 3, 'O': 2}, 'polarity': 'positive'}, '[M+H-NH3]+': {'add': {}, 'sub': {'N': 1, 'H': 2}, 'polarity': 'positive'}, '[M+2H-NH3]2+': {'add': {}, 'sub': {'N': 1, 'H': 1}, 'polarity': 'positive'}, '[M+2H-H2O]2+': {'add': {}, 'sub': {'O': 1}, 'polarity': 'positive'}, '[M+NH4-H2O]+': {'add': {'N': 1, 'H': 2}, 'sub': {}, 'polarity': 'positive'}, '[M+H+H2O]+': {'add': {'H': 3, 'O': 1}, 'sub': {}, 'polarity': 'positive'}, 'de-protonated': {'add': {}, 'sub': {'H': 1}, 'polarity': 'negative'}, '[M-H]-': {'add': {}, 'sub': {'H': 1}, 'polarity': 'negative'}, '[M-2H]2-': {'add': {}, 'sub': {'H': 2}, 'polarity': 'negative'}, '[M-H-H2O]-': {'add': {}, 'sub': {'H': 3, 'O': 1}, 'polarity': 'negative'}, '[M-H+H2O]-': {'add': {'H': 1, 'O': 1}, 'sub': {}, 'polarity': 'negative'}, '[M+Cl]-': {'add': {'Cl': 1}, 'sub': {}, 'polarity': 'negative'}, '[M+HCOO]-': {'add': {'C': 1, 'H': 1, 'O': 2}, 'sub': {}, 'polarity': 'negative'}, '[M+CH3COO]-': {'add': {'C': 2, 'H': 3, 'O': 2}, 'sub': {}, 'polarity': 'negative'}, '[M+2NaAc+Cl]-': {'add': {'Na': 2, 'C': 2, 'H': 3, 'O': 2, 'Cl': 1}, 'sub': {}, 'polarity': 'negative'}, '[M+K-2H]-': {'add': {'K': 1}, 'sub': {'H': 2}, 'polarity': 'negative'}, '[M+Na-2H]-': {'add': {'Na': 1}, 'sub': {'H': 2}, 'polarity': 'negative'}}
class
LowResGCMSExport:
43class LowResGCMSExport: 44 """A class to export low resolution GC-MS data. 45 46 This class provides methods to export low resolution GC-MS data to various formats such as Excel, CSV, Parquet, HDF5, and Pandas DataFrame. 47 48 Parameters: 49 ---------- 50 out_file_path : str 51 The output file path. 52 gcms : object 53 The low resolution GCMS object. 54 55 Attributes: 56 ---------- 57 output_file : Path 58 The output file path as a Path object. 59 gcms : object 60 The low resolution GCMS object. 61 62 Methods: 63 ------- 64 * get_pandas_df(id_label="corems:"). Get the exported data as a Pandas DataFrame. 65 * get_json(nan=False, id_label="corems:"). Get the exported data as a JSON string. 66 * to_pandas(write_metadata=True, id_label="corems:"). Export the data to a Pandas DataFrame and save it as a pickle file. 67 * to_excel(write_mode='a', write_metadata=True, id_label="corems:"), 68 Export the data to an Excel file. 69 * to_csv(separate_output=False, write_mode="w", write_metadata=True, id_label="corems:"). 70 Export the data to a CSV file. 71 * to_parquet(write_metadata=True, id_label="corems:"). 72 Export the data to a Parquet file. 73 * to_hdf(id_label="corems:"). 74 Export the data to an HDF5 file. 75 * get_data_stats(gcms). 76 Get statistics about the GCMS data. 77 78 """ 79 80 def __init__(self, out_file_path, gcms): 81 self.output_file = Path(out_file_path) 82 83 self.gcms = gcms 84 85 self._init_columns() 86 87 def _init_columns(self): 88 """Initialize the column names for the exported data. 89 90 Returns: 91 ------- 92 list 93 The list of column names. 94 """ 95 96 columns = [ 97 "Sample name", 98 "Peak Index", 99 "Retention Time", 100 "Retention Time Ref", 101 "Peak Height", 102 "Peak Area", 103 "Retention index", 104 "Retention index Ref", 105 "Retention Index Score", 106 "Similarity Score", 107 "Spectral Similarity Score", 108 "Compound Name", 109 "Chebi ID", 110 "Kegg Compound ID", 111 "Inchi", 112 "Inchi Key", 113 "Smiles", 114 "Molecular Formula", 115 "IUPAC Name", 116 "Traditional Name", 117 "Common Name", 118 "Derivatization", 119 ] 120 121 if self.gcms.molecular_search_settings.exploratory_mode: 122 columns.extend( 123 [ 124 "Weighted Cosine Correlation", 125 "Cosine Correlation", 126 "Stein Scott Similarity", 127 "Pearson Correlation", 128 "Spearman Correlation", 129 "Kendall Tau Correlation", 130 "Euclidean Distance", 131 "Manhattan Distance", 132 "Jaccard Distance", 133 "DWT Correlation", 134 "DFT Correlation", 135 ] 136 ) 137 138 columns.extend(list(methods_name.values())) 139 140 return columns 141 142 def get_pandas_df(self, id_label="corems:"): 143 """Get the exported data as a Pandas DataFrame. 144 145 Parameters: 146 ---------- 147 id_label : str, optional 148 The ID label for the data. Default is "corems:". 149 150 Returns: 151 ------- 152 DataFrame 153 The exported data as a Pandas DataFrame. 154 """ 155 156 columns = self._init_columns() 157 158 dict_data_list = self.get_list_dict_data(self.gcms) 159 160 df = DataFrame(dict_data_list, columns=columns) 161 162 df.attrs["name"] = self.gcms.sample_name 163 164 return df 165 166 def get_json(self, nan=False, id_label="corems:"): 167 """Get the exported data as a JSON string. 168 169 Parameters: 170 ---------- 171 nan : bool, optional 172 Whether to include NaN values in the JSON string. Default is False. 173 id_label : str, optional 174 The ID label for the data. Default is "corems:". 175 176 """ 177 178 import json 179 180 dict_data_list = self.get_list_dict_data(self.gcms) 181 182 return json.dumps( 183 dict_data_list, sort_keys=False, indent=4, separators=(",", ": ") 184 ) 185 186 def to_pandas(self, write_metadata=True, id_label="corems:"): 187 """Export the data to a Pandas DataFrame and save it as a pickle file. 188 189 Parameters: 190 ---------- 191 write_metadata : bool, optional 192 Whether to write metadata to the output file. 193 id_label : str, optional 194 The ID label for the data. 195 """ 196 197 columns = self._init_columns() 198 199 dict_data_list = self.get_list_dict_data(self.gcms) 200 201 df = DataFrame(dict_data_list, columns=columns) 202 203 df.to_pickle(self.output_file.with_suffix(".pkl")) 204 205 if write_metadata: 206 self.write_settings( 207 self.output_file.with_suffix(".pkl"), self.gcms, id_label="corems:" 208 ) 209 210 def to_excel(self, write_mode="a", write_metadata=True, id_label="corems:"): 211 """Export the data to an Excel file. 212 213 Parameters: 214 ---------- 215 write_mode : str, optional 216 The write mode for the Excel file. Default is 'a' (append). 217 write_metadata : bool, optional 218 Whether to write metadata to the output file. Default is True. 219 id_label : str, optional 220 The ID label for the data. Default is "corems:". 221 """ 222 223 out_put_path = self.output_file.with_suffix(".xlsx") 224 225 columns = self._init_columns() 226 227 dict_data_list = self.get_list_dict_data(self.gcms) 228 229 df = DataFrame(dict_data_list, columns=columns) 230 231 if write_mode == "a" and out_put_path.exists(): 232 writer = ExcelWriter(out_put_path, engine="openpyxl") 233 # try to open an existing workbook 234 writer.book = load_workbook(out_put_path) 235 # copy existing sheets 236 writer.sheets = dict((ws.title, ws) for ws in writer.book.worksheets) 237 # read existing file 238 reader = read_excel(out_put_path) 239 # write out the new sheet 240 df.to_excel(writer, index=False, header=False, startrow=len(reader) + 1) 241 242 writer.close() 243 else: 244 df.to_excel( 245 self.output_file.with_suffix(".xlsx"), index=False, engine="openpyxl" 246 ) 247 248 if write_metadata: 249 self.write_settings(out_put_path, self.gcms, id_label=id_label) 250 251 def to_csv( 252 self, 253 separate_output=False, 254 write_mode="w", 255 write_metadata=True, 256 id_label="corems:", 257 ): 258 """Export the data to a CSV file. 259 260 Parameters: 261 ---------- 262 separate_output : bool, optional 263 Whether to separate the output into multiple files. Default is False. 264 write_mode : str, optional 265 The write mode for the CSV file. Default is 'w' (write). 266 write_metadata : bool, optional 267 Whether to write metadata to the output file. Default is True. 268 id_label : str, optional 269 The ID label for the data. Default is "corems:". 270 """ 271 272 if separate_output: 273 # set write mode to write 274 # this mode will overwrite the file without warning 275 write_mode = "w" 276 else: 277 # set write mode to append 278 write_mode = "a" 279 280 columns = self._init_columns() 281 282 dict_data_list = self.get_list_dict_data(self.gcms) 283 284 out_put_path = self.output_file.with_suffix(".csv") 285 286 write_header = not out_put_path.exists() 287 288 try: 289 with open(out_put_path, write_mode, newline="") as csvfile: 290 writer = csv.DictWriter(csvfile, fieldnames=columns) 291 if write_header: 292 writer.writeheader() 293 for data in dict_data_list: 294 writer.writerow(data) 295 296 if write_metadata: 297 self.write_settings(out_put_path, self.gcms, id_label=id_label) 298 299 except IOError as ioerror: 300 print(ioerror) 301 302 def to_parquet(self, write_metadata=True, id_label="corems:"): 303 """Export the data to a Parquet file. 304 305 Parameters: 306 ---------- 307 write_metadata : bool, optional 308 Whether to write metadata to the output file. Default is True. 309 id_label : str, optional 310 The ID label for the data. Default is "corems:". 311 312 Notes 313 ----- 314 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 315 """ 316 317 columns = self._init_columns() 318 319 dict_data_list = self.get_list_dict_data(self.gcms) 320 321 df = DataFrame(dict_data_list, columns=columns) 322 323 out_put_path = self.output_file.with_suffix(".parquet") 324 df.to_parquet(out_put_path, index=False) 325 326 if write_metadata: 327 self.write_settings(out_put_path, self.gcms, id_label=id_label) 328 329 def to_hdf(self, id_label="corems:"): 330 """Export the data to an HDF5 file. 331 332 Parameters: 333 ---------- 334 id_label : str, optional 335 The ID label for the data. Default is "corems:". 336 """ 337 338 # save sample at a time 339 def add_compound(gc_peak, compound_obj): 340 modifier = compound_obj.classify if compound_obj.classify else "" 341 compound_group = compound_obj.name.replace("/", "") + " " + modifier 342 343 if compound_group not in peak_group: 344 compound_group = peak_group.create_group(compound_group) 345 346 # compound_group.attrs["retention_time"] = compound_obj.retention_time 347 compound_group.attrs["retention_index"] = compound_obj.ri 348 compound_group.attrs["retention_index_score"] = compound_obj.ri_score 349 compound_group.attrs["spectral_similarity_score"] = ( 350 compound_obj.spectral_similarity_score 351 ) 352 compound_group.attrs["similarity_score"] = compound_obj.similarity_score 353 354 compond_mz = compound_group.create_dataset( 355 "mz", data=np.array(compound_obj.mz), dtype="f8" 356 ) 357 compond_abundance = compound_group.create_dataset( 358 "abundance", data=np.array(compound_obj.abundance), dtype="f8" 359 ) 360 361 if self.gcms.molecular_search_settings.exploratory_mode: 362 compound_group.attrs["Spectral Similarities"] = json.dumps( 363 compound_obj.spectral_similarity_scores, 364 sort_keys=False, 365 indent=4, 366 separators=(",", ":"), 367 ) 368 else: 369 warnings.warn("Skipping duplicate reference compound.") 370 371 import json 372 from datetime import datetime, timezone 373 374 import h5py 375 import numpy as np 376 377 output_path = self.output_file.with_suffix(".hdf5") 378 379 with h5py.File(output_path, "w") as hdf_handle: 380 timenow = str(datetime.now(timezone.utc).strftime("%d/%m/%Y %H:%M:%S %Z")) 381 hdf_handle.attrs["time_stamp"] = timenow 382 hdf_handle.attrs["data_structure"] = "gcms" 383 hdf_handle.attrs["analyzer"] = self.gcms.analyzer 384 hdf_handle.attrs["instrument_label"] = self.gcms.instrument_label 385 386 hdf_handle.attrs["sample_id"] = "self.gcms.id" 387 hdf_handle.attrs["sample_name"] = self.gcms.sample_name 388 hdf_handle.attrs["input_data"] = str(self.gcms.file_location) 389 hdf_handle.attrs["output_data"] = str(output_path) 390 hdf_handle.attrs["output_data_id"] = id_label + uuid.uuid4().hex 391 hdf_handle.attrs["corems_version"] = __version__ 392 393 hdf_handle.attrs["Stats"] = json.dumps( 394 self.get_data_stats(self.gcms), 395 sort_keys=False, 396 indent=4, 397 separators=(",", ": "), 398 ) 399 hdf_handle.attrs["Calibration"] = json.dumps( 400 self.get_calibration_stats(self.gcms, id_label), 401 sort_keys=False, 402 indent=4, 403 separators=(",", ": "), 404 ) 405 hdf_handle.attrs["Blank"] = json.dumps( 406 self.get_blank_stats(self.gcms), 407 sort_keys=False, 408 indent=4, 409 separators=(",", ": "), 410 ) 411 412 corems_dict_setting = parameter_to_dict.get_dict_data_gcms(self.gcms) 413 hdf_handle.attrs["CoreMSParameters"] = json.dumps( 414 corems_dict_setting, sort_keys=False, indent=4, separators=(",", ": ") 415 ) 416 417 scans_dataset = hdf_handle.create_dataset( 418 "scans", data=np.array(self.gcms.scans_number), dtype="f8" 419 ) 420 rt_dataset = hdf_handle.create_dataset( 421 "rt", data=np.array(self.gcms.retention_time), dtype="f8" 422 ) 423 tic_dataset = hdf_handle.create_dataset( 424 "tic", data=np.array(self.gcms.tic), dtype="f8" 425 ) 426 processed_tic_dataset = hdf_handle.create_dataset( 427 "processed_tic", data=np.array(self.gcms.processed_tic), dtype="f8" 428 ) 429 430 output_score_method = ( 431 self.gcms.molecular_search_settings.output_score_method 432 ) 433 434 for gc_peak in self.gcms: 435 # print(gc_peak.retention_time) 436 # print(gc_peak.tic) 437 438 # check if there is a compound candidate 439 peak_group = hdf_handle.create_group(str(gc_peak.retention_time)) 440 peak_group.attrs["deconvolution"] = int( 441 self.gcms.chromatogram_settings.use_deconvolution 442 ) 443 444 peak_group.attrs["start_scan"] = gc_peak.start_scan 445 peak_group.attrs["apex_scan"] = gc_peak.apex_scan 446 peak_group.attrs["final_scan"] = gc_peak.final_scan 447 448 peak_group.attrs["retention_index"] = gc_peak.ri 449 peak_group.attrs["retention_time"] = gc_peak.retention_time 450 peak_group.attrs["area"] = gc_peak.area 451 452 mz = peak_group.create_dataset( 453 "mz", data=np.array(gc_peak.mass_spectrum.mz_exp), dtype="f8" 454 ) 455 abundance = peak_group.create_dataset( 456 "abundance", 457 data=np.array(gc_peak.mass_spectrum.abundance), 458 dtype="f8", 459 ) 460 461 if gc_peak: 462 if output_score_method == "highest_sim_score": 463 compound_obj = gc_peak.highest_score_compound 464 add_compound(gc_peak, compound_obj) 465 466 elif output_score_method == "highest_ss": 467 compound_obj = gc_peak.highest_ss_compound 468 add_compound(gc_peak, compound_obj) 469 470 else: 471 for compound_obj in gc_peak: 472 add_compound(gc_peak, compound_obj) 473 474 def get_data_stats(self, gcms): 475 """Get statistics about the GCMS data. 476 477 Parameters: 478 ---------- 479 gcms : object 480 The low resolution GCMS object. 481 482 Returns: 483 ------- 484 dict 485 A dictionary containing the data statistics. 486 """ 487 488 matched_peaks = gcms.matched_peaks 489 no_matched_peaks = gcms.no_matched_peaks 490 unique_metabolites = gcms.unique_metabolites 491 492 peak_matchs_above_0p85 = 0 493 unique_peak_match_above_0p85 = 0 494 for match_peak in matched_peaks: 495 gc_peak_above_85 = 0 496 matches_above_85 = list( 497 filter(lambda m: m.similarity_score >= 0.85, match_peak) 498 ) 499 if matches_above_85: 500 peak_matchs_above_0p85 += 1 501 if len(matches_above_85) == 1: 502 unique_peak_match_above_0p85 += 1 503 504 data_stats = {} 505 data_stats["average_signal_noise"] = "ni" 506 data_stats["chromatogram_dynamic_range"] = gcms.dynamic_range 507 data_stats["total_number_peaks"] = len(gcms) 508 data_stats["total_peaks_matched"] = len(matched_peaks) 509 data_stats["total_peaks_without_matches"] = len(no_matched_peaks) 510 data_stats["total_matches_above_similarity_score_0.85"] = peak_matchs_above_0p85 511 data_stats["single_matches_above_similarity_score_0.85"] = ( 512 unique_peak_match_above_0p85 513 ) 514 data_stats["unique_metabolites"] = len(unique_metabolites) 515 516 return data_stats 517 518 def get_calibration_stats(self, gcms, id_label): 519 """Get statistics about the GC-MS calibration. 520 521 Parameters: 522 ---------- 523 """ 524 calibration_parameters = {} 525 526 calibration_parameters["calibration_rt_ri_pairs_ref"] = gcms.ri_pairs_ref 527 calibration_parameters["data_url"] = str(gcms.cal_file_path) 528 calibration_parameters["has_input"] = id_label + corems_md5(gcms.cal_file_path) 529 calibration_parameters["data_name"] = str(gcms.cal_file_path.stem) 530 calibration_parameters["calibration_method"] = "" 531 532 return calibration_parameters 533 534 def get_blank_stats(self, gcms): 535 """Get statistics about the GC-MS blank.""" 536 blank_parameters = {} 537 538 blank_parameters["data_name"] = "ni" 539 blank_parameters["blank_id"] = "ni" 540 blank_parameters["data_url"] = "ni" 541 blank_parameters["has_input"] = "ni" 542 blank_parameters["common_features_to_blank"] = "ni" 543 544 return blank_parameters 545 546 def get_instrument_metadata(self, gcms): 547 """Get metadata about the GC-MS instrument.""" 548 instrument_metadata = {} 549 550 instrument_metadata["analyzer"] = gcms.analyzer 551 instrument_metadata["instrument_label"] = gcms.instrument_label 552 instrument_metadata["instrument_id"] = uuid.uuid4().hex 553 554 return instrument_metadata 555 556 def get_data_metadata(self, gcms, id_label, output_path): 557 """Get metadata about the GC-MS data. 558 559 Parameters: 560 ---------- 561 gcms : object 562 The low resolution GCMS object. 563 id_label : str 564 The ID label for the data. 565 output_path : str 566 The output file path. 567 568 Returns: 569 ------- 570 dict 571 A dictionary containing the data metadata. 572 """ 573 if isinstance(output_path, str): 574 output_path = Path(output_path) 575 576 paramaters_path = output_path.with_suffix(".json") 577 578 if paramaters_path.exists(): 579 with paramaters_path.open() as current_param: 580 metadata = json.load(current_param) 581 data_metadata = metadata.get("Data") 582 else: 583 data_metadata = {} 584 data_metadata["data_name"] = [] 585 data_metadata["input_data_url"] = [] 586 data_metadata["has_input"] = [] 587 588 data_metadata["data_name"].append(gcms.sample_name) 589 data_metadata["input_data_url"].append(str(gcms.file_location)) 590 data_metadata["has_input"].append(id_label + corems_md5(gcms.file_location)) 591 592 data_metadata["output_data_name"] = str(output_path.stem) 593 data_metadata["output_data_url"] = str(output_path) 594 data_metadata["has_output"] = id_label + corems_md5(output_path) 595 596 return data_metadata 597 598 def get_parameters_json(self, gcms, id_label, output_path): 599 """Get the parameters as a JSON string. 600 601 Parameters: 602 ---------- 603 gcms : GCMS object 604 The low resolution GCMS object. 605 id_label : str 606 The ID label for the data. 607 output_path : str 608 The output file path. 609 610 Returns: 611 ------- 612 str 613 The parameters as a JSON string. 614 """ 615 616 output_parameters_dict = {} 617 output_parameters_dict["Data"] = self.get_data_metadata( 618 gcms, id_label, output_path 619 ) 620 output_parameters_dict["Stats"] = self.get_data_stats(gcms) 621 output_parameters_dict["Calibration"] = self.get_calibration_stats( 622 gcms, id_label 623 ) 624 output_parameters_dict["Blank"] = self.get_blank_stats(gcms) 625 output_parameters_dict["Instrument"] = self.get_instrument_metadata(gcms) 626 corems_dict_setting = parameter_to_dict.get_dict_data_gcms(gcms) 627 corems_dict_setting["corems_version"] = __version__ 628 output_parameters_dict["CoreMSParameters"] = corems_dict_setting 629 output_parameters_dict["has_metabolite"] = gcms.metabolites_data 630 output = json.dumps( 631 output_parameters_dict, sort_keys=False, indent=4, separators=(",", ": ") 632 ) 633 634 return output 635 636 def write_settings(self, output_path, gcms, id_label="emsl:"): 637 """Write the settings to a JSON file. 638 639 Parameters: 640 ---------- 641 output_path : str 642 The output file path. 643 gcms : GCMS object 644 The low resolution GCMS object. 645 id_label : str 646 The ID label for the data. Default is "emsl:". 647 648 """ 649 650 output = self.get_parameters_json(gcms, id_label, output_path) 651 652 with open( 653 output_path.with_suffix(".json"), 654 "w", 655 encoding="utf8", 656 ) as outfile: 657 outfile.write(output) 658 659 def get_list_dict_data(self, gcms, include_no_match=True, no_match_inline=False): 660 """Get the exported data as a list of dictionaries. 661 662 Parameters: 663 ---------- 664 gcms : object 665 The low resolution GCMS object. 666 include_no_match : bool, optional 667 Whether to include no match data. Default is True. 668 no_match_inline : bool, optional 669 Whether to include no match data inline. Default is False. 670 671 Returns: 672 ------- 673 list 674 The exported data as a list of dictionaries. 675 """ 676 677 output_score_method = gcms.molecular_search_settings.output_score_method 678 679 dict_data_list = [] 680 681 def add_match_dict_data(): 682 derivatization = "{}:{}:{}".format( 683 compound_obj.classify, 684 compound_obj.derivativenum, 685 compound_obj.derivatization, 686 ) 687 out_dict = { 688 "Sample name": gcms.sample_name, 689 "Peak Index": gcpeak_index, 690 "Retention Time": gc_peak.retention_time, 691 "Retention Time Ref": compound_obj.retention_time, 692 "Peak Height": gc_peak.tic, 693 "Peak Area": gc_peak.area, 694 "Retention index": gc_peak.ri, 695 "Retention index Ref": compound_obj.ri, 696 "Retention Index Score": compound_obj.ri_score, 697 "Spectral Similarity Score": compound_obj.spectral_similarity_score, 698 "Similarity Score": compound_obj.similarity_score, 699 "Compound Name": compound_obj.name, 700 "Chebi ID": compound_obj.metadata.chebi, 701 "Kegg Compound ID": compound_obj.metadata.kegg, 702 "Inchi": compound_obj.metadata.inchi, 703 "Inchi Key": compound_obj.metadata.inchikey, 704 "Smiles": compound_obj.metadata.smiles, 705 "Molecular Formula": compound_obj.formula, 706 "IUPAC Name": compound_obj.metadata.iupac_name, 707 "Traditional Name": compound_obj.metadata.traditional_name, 708 "Common Name": compound_obj.metadata.common_name, 709 "Derivatization": derivatization, 710 } 711 712 if self.gcms.molecular_search_settings.exploratory_mode: 713 out_dict.update( 714 { 715 "Weighted Cosine Correlation": compound_obj.spectral_similarity_scores.get( 716 "weighted_cosine_correlation" 717 ), 718 "Cosine Correlation": compound_obj.spectral_similarity_scores.get( 719 "cosine_correlation" 720 ), 721 "Stein Scott Similarity": compound_obj.spectral_similarity_scores.get( 722 "stein_scott_similarity" 723 ), 724 "Pearson Correlation": compound_obj.spectral_similarity_scores.get( 725 "pearson_correlation" 726 ), 727 "Spearman Correlation": compound_obj.spectral_similarity_scores.get( 728 "spearman_correlation" 729 ), 730 "Kendall Tau Correlation": compound_obj.spectral_similarity_scores.get( 731 "kendall_tau_correlation" 732 ), 733 "DFT Correlation": compound_obj.spectral_similarity_scores.get( 734 "dft_correlation" 735 ), 736 "DWT Correlation": compound_obj.spectral_similarity_scores.get( 737 "dwt_correlation" 738 ), 739 "Euclidean Distance": compound_obj.spectral_similarity_scores.get( 740 "euclidean_distance" 741 ), 742 "Manhattan Distance": compound_obj.spectral_similarity_scores.get( 743 "manhattan_distance" 744 ), 745 "Jaccard Distance": compound_obj.spectral_similarity_scores.get( 746 "jaccard_distance" 747 ), 748 } 749 ) 750 for method in methods_name: 751 out_dict[methods_name.get(method)] = ( 752 compound_obj.spectral_similarity_scores.get(method) 753 ) 754 755 dict_data_list.append(out_dict) 756 757 def add_no_match_dict_data(): 758 dict_data_list.append( 759 { 760 "Sample name": gcms.sample_name, 761 "Peak Index": gcpeak_index, 762 "Retention Time": gc_peak.retention_time, 763 "Peak Height": gc_peak.tic, 764 "Peak Area": gc_peak.area, 765 "Retention index": gc_peak.ri, 766 } 767 ) 768 769 for gcpeak_index, gc_peak in enumerate(gcms.sorted_gcpeaks): 770 # check if there is a compound candidate 771 if gc_peak: 772 if output_score_method == "highest_sim_score": 773 compound_obj = gc_peak.highest_score_compound 774 add_match_dict_data() 775 776 elif output_score_method == "highest_ss": 777 compound_obj = gc_peak.highest_ss_compound 778 add_match_dict_data() 779 780 else: 781 for compound_obj in gc_peak: 782 add_match_dict_data() # add monoisotopic peak 783 784 else: 785 # include not_match 786 if include_no_match and no_match_inline: 787 add_no_match_dict_data() 788 789 if include_no_match and not no_match_inline: 790 for gcpeak_index, gc_peak in enumerate(gcms.sorted_gcpeaks): 791 if not gc_peak: 792 add_no_match_dict_data() 793 794 return dict_data_list
A class to export low resolution GC-MS data.
This class provides methods to export low resolution GC-MS data to various formats such as Excel, CSV, Parquet, HDF5, and Pandas DataFrame.
Parameters:
out_file_path : str The output file path. gcms : object The low resolution GCMS object.
Attributes:
output_file : Path The output file path as a Path object. gcms : object The low resolution GCMS object.
Methods:
- get_pandas_df(id_label="corems:"). Get the exported data as a Pandas DataFrame.
- get_json(nan=False, id_label="corems:"). Get the exported data as a JSON string.
- to_pandas(write_metadata=True, id_label="corems:"). Export the data to a Pandas DataFrame and save it as a pickle file.
- to_excel(write_mode='a', write_metadata=True, id_label="corems:"), Export the data to an Excel file.
- to_csv(separate_output=False, write_mode="w", write_metadata=True, id_label="corems:"). Export the data to a CSV file.
- to_parquet(write_metadata=True, id_label="corems:"). Export the data to a Parquet file.
- to_hdf(id_label="corems:"). Export the data to an HDF5 file.
- get_data_stats(gcms). Get statistics about the GCMS data.
def
get_pandas_df(self, id_label='corems:'):
142 def get_pandas_df(self, id_label="corems:"): 143 """Get the exported data as a Pandas DataFrame. 144 145 Parameters: 146 ---------- 147 id_label : str, optional 148 The ID label for the data. Default is "corems:". 149 150 Returns: 151 ------- 152 DataFrame 153 The exported data as a Pandas DataFrame. 154 """ 155 156 columns = self._init_columns() 157 158 dict_data_list = self.get_list_dict_data(self.gcms) 159 160 df = DataFrame(dict_data_list, columns=columns) 161 162 df.attrs["name"] = self.gcms.sample_name 163 164 return df
Get the exported data as a Pandas DataFrame.
Parameters:
id_label : str, optional The ID label for the data. Default is "corems:".
Returns:
DataFrame The exported data as a Pandas DataFrame.
def
get_json(self, nan=False, id_label='corems:'):
166 def get_json(self, nan=False, id_label="corems:"): 167 """Get the exported data as a JSON string. 168 169 Parameters: 170 ---------- 171 nan : bool, optional 172 Whether to include NaN values in the JSON string. Default is False. 173 id_label : str, optional 174 The ID label for the data. Default is "corems:". 175 176 """ 177 178 import json 179 180 dict_data_list = self.get_list_dict_data(self.gcms) 181 182 return json.dumps( 183 dict_data_list, sort_keys=False, indent=4, separators=(",", ": ") 184 )
Get the exported data as a JSON string.
Parameters:
nan : bool, optional Whether to include NaN values in the JSON string. Default is False. id_label : str, optional The ID label for the data. Default is "corems:".
def
to_pandas(self, write_metadata=True, id_label='corems:'):
186 def to_pandas(self, write_metadata=True, id_label="corems:"): 187 """Export the data to a Pandas DataFrame and save it as a pickle file. 188 189 Parameters: 190 ---------- 191 write_metadata : bool, optional 192 Whether to write metadata to the output file. 193 id_label : str, optional 194 The ID label for the data. 195 """ 196 197 columns = self._init_columns() 198 199 dict_data_list = self.get_list_dict_data(self.gcms) 200 201 df = DataFrame(dict_data_list, columns=columns) 202 203 df.to_pickle(self.output_file.with_suffix(".pkl")) 204 205 if write_metadata: 206 self.write_settings( 207 self.output_file.with_suffix(".pkl"), self.gcms, id_label="corems:" 208 )
Export the data to a Pandas DataFrame and save it as a pickle file.
Parameters:
write_metadata : bool, optional Whether to write metadata to the output file. id_label : str, optional The ID label for the data.
def
to_excel(self, write_mode='a', write_metadata=True, id_label='corems:'):
210 def to_excel(self, write_mode="a", write_metadata=True, id_label="corems:"): 211 """Export the data to an Excel file. 212 213 Parameters: 214 ---------- 215 write_mode : str, optional 216 The write mode for the Excel file. Default is 'a' (append). 217 write_metadata : bool, optional 218 Whether to write metadata to the output file. Default is True. 219 id_label : str, optional 220 The ID label for the data. Default is "corems:". 221 """ 222 223 out_put_path = self.output_file.with_suffix(".xlsx") 224 225 columns = self._init_columns() 226 227 dict_data_list = self.get_list_dict_data(self.gcms) 228 229 df = DataFrame(dict_data_list, columns=columns) 230 231 if write_mode == "a" and out_put_path.exists(): 232 writer = ExcelWriter(out_put_path, engine="openpyxl") 233 # try to open an existing workbook 234 writer.book = load_workbook(out_put_path) 235 # copy existing sheets 236 writer.sheets = dict((ws.title, ws) for ws in writer.book.worksheets) 237 # read existing file 238 reader = read_excel(out_put_path) 239 # write out the new sheet 240 df.to_excel(writer, index=False, header=False, startrow=len(reader) + 1) 241 242 writer.close() 243 else: 244 df.to_excel( 245 self.output_file.with_suffix(".xlsx"), index=False, engine="openpyxl" 246 ) 247 248 if write_metadata: 249 self.write_settings(out_put_path, self.gcms, id_label=id_label)
Export the data to an Excel file.
Parameters:
write_mode : str, optional The write mode for the Excel file. Default is 'a' (append). write_metadata : bool, optional Whether to write metadata to the output file. Default is True. id_label : str, optional The ID label for the data. Default is "corems:".
def
to_csv( self, separate_output=False, write_mode='w', write_metadata=True, id_label='corems:'):
251 def to_csv( 252 self, 253 separate_output=False, 254 write_mode="w", 255 write_metadata=True, 256 id_label="corems:", 257 ): 258 """Export the data to a CSV file. 259 260 Parameters: 261 ---------- 262 separate_output : bool, optional 263 Whether to separate the output into multiple files. Default is False. 264 write_mode : str, optional 265 The write mode for the CSV file. Default is 'w' (write). 266 write_metadata : bool, optional 267 Whether to write metadata to the output file. Default is True. 268 id_label : str, optional 269 The ID label for the data. Default is "corems:". 270 """ 271 272 if separate_output: 273 # set write mode to write 274 # this mode will overwrite the file without warning 275 write_mode = "w" 276 else: 277 # set write mode to append 278 write_mode = "a" 279 280 columns = self._init_columns() 281 282 dict_data_list = self.get_list_dict_data(self.gcms) 283 284 out_put_path = self.output_file.with_suffix(".csv") 285 286 write_header = not out_put_path.exists() 287 288 try: 289 with open(out_put_path, write_mode, newline="") as csvfile: 290 writer = csv.DictWriter(csvfile, fieldnames=columns) 291 if write_header: 292 writer.writeheader() 293 for data in dict_data_list: 294 writer.writerow(data) 295 296 if write_metadata: 297 self.write_settings(out_put_path, self.gcms, id_label=id_label) 298 299 except IOError as ioerror: 300 print(ioerror)
Export the data to a CSV file.
Parameters:
separate_output : bool, optional Whether to separate the output into multiple files. Default is False. write_mode : str, optional The write mode for the CSV file. Default is 'w' (write). write_metadata : bool, optional Whether to write metadata to the output file. Default is True. id_label : str, optional The ID label for the data. Default is "corems:".
def
to_parquet(self, write_metadata=True, id_label='corems:'):
302 def to_parquet(self, write_metadata=True, id_label="corems:"): 303 """Export the data to a Parquet file. 304 305 Parameters: 306 ---------- 307 write_metadata : bool, optional 308 Whether to write metadata to the output file. Default is True. 309 id_label : str, optional 310 The ID label for the data. Default is "corems:". 311 312 Notes 313 ----- 314 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 315 """ 316 317 columns = self._init_columns() 318 319 dict_data_list = self.get_list_dict_data(self.gcms) 320 321 df = DataFrame(dict_data_list, columns=columns) 322 323 out_put_path = self.output_file.with_suffix(".parquet") 324 df.to_parquet(out_put_path, index=False) 325 326 if write_metadata: 327 self.write_settings(out_put_path, self.gcms, id_label=id_label)
Export the data to a Parquet file.
Parameters:
write_metadata : bool, optional Whether to write metadata to the output file. Default is True. id_label : str, optional The ID label for the data. Default is "corems:".
Notes
Requires pyarrow (or fastparquet) for pandas Parquet I/O.
def
to_hdf(self, id_label='corems:'):
329 def to_hdf(self, id_label="corems:"): 330 """Export the data to an HDF5 file. 331 332 Parameters: 333 ---------- 334 id_label : str, optional 335 The ID label for the data. Default is "corems:". 336 """ 337 338 # save sample at a time 339 def add_compound(gc_peak, compound_obj): 340 modifier = compound_obj.classify if compound_obj.classify else "" 341 compound_group = compound_obj.name.replace("/", "") + " " + modifier 342 343 if compound_group not in peak_group: 344 compound_group = peak_group.create_group(compound_group) 345 346 # compound_group.attrs["retention_time"] = compound_obj.retention_time 347 compound_group.attrs["retention_index"] = compound_obj.ri 348 compound_group.attrs["retention_index_score"] = compound_obj.ri_score 349 compound_group.attrs["spectral_similarity_score"] = ( 350 compound_obj.spectral_similarity_score 351 ) 352 compound_group.attrs["similarity_score"] = compound_obj.similarity_score 353 354 compond_mz = compound_group.create_dataset( 355 "mz", data=np.array(compound_obj.mz), dtype="f8" 356 ) 357 compond_abundance = compound_group.create_dataset( 358 "abundance", data=np.array(compound_obj.abundance), dtype="f8" 359 ) 360 361 if self.gcms.molecular_search_settings.exploratory_mode: 362 compound_group.attrs["Spectral Similarities"] = json.dumps( 363 compound_obj.spectral_similarity_scores, 364 sort_keys=False, 365 indent=4, 366 separators=(",", ":"), 367 ) 368 else: 369 warnings.warn("Skipping duplicate reference compound.") 370 371 import json 372 from datetime import datetime, timezone 373 374 import h5py 375 import numpy as np 376 377 output_path = self.output_file.with_suffix(".hdf5") 378 379 with h5py.File(output_path, "w") as hdf_handle: 380 timenow = str(datetime.now(timezone.utc).strftime("%d/%m/%Y %H:%M:%S %Z")) 381 hdf_handle.attrs["time_stamp"] = timenow 382 hdf_handle.attrs["data_structure"] = "gcms" 383 hdf_handle.attrs["analyzer"] = self.gcms.analyzer 384 hdf_handle.attrs["instrument_label"] = self.gcms.instrument_label 385 386 hdf_handle.attrs["sample_id"] = "self.gcms.id" 387 hdf_handle.attrs["sample_name"] = self.gcms.sample_name 388 hdf_handle.attrs["input_data"] = str(self.gcms.file_location) 389 hdf_handle.attrs["output_data"] = str(output_path) 390 hdf_handle.attrs["output_data_id"] = id_label + uuid.uuid4().hex 391 hdf_handle.attrs["corems_version"] = __version__ 392 393 hdf_handle.attrs["Stats"] = json.dumps( 394 self.get_data_stats(self.gcms), 395 sort_keys=False, 396 indent=4, 397 separators=(",", ": "), 398 ) 399 hdf_handle.attrs["Calibration"] = json.dumps( 400 self.get_calibration_stats(self.gcms, id_label), 401 sort_keys=False, 402 indent=4, 403 separators=(",", ": "), 404 ) 405 hdf_handle.attrs["Blank"] = json.dumps( 406 self.get_blank_stats(self.gcms), 407 sort_keys=False, 408 indent=4, 409 separators=(",", ": "), 410 ) 411 412 corems_dict_setting = parameter_to_dict.get_dict_data_gcms(self.gcms) 413 hdf_handle.attrs["CoreMSParameters"] = json.dumps( 414 corems_dict_setting, sort_keys=False, indent=4, separators=(",", ": ") 415 ) 416 417 scans_dataset = hdf_handle.create_dataset( 418 "scans", data=np.array(self.gcms.scans_number), dtype="f8" 419 ) 420 rt_dataset = hdf_handle.create_dataset( 421 "rt", data=np.array(self.gcms.retention_time), dtype="f8" 422 ) 423 tic_dataset = hdf_handle.create_dataset( 424 "tic", data=np.array(self.gcms.tic), dtype="f8" 425 ) 426 processed_tic_dataset = hdf_handle.create_dataset( 427 "processed_tic", data=np.array(self.gcms.processed_tic), dtype="f8" 428 ) 429 430 output_score_method = ( 431 self.gcms.molecular_search_settings.output_score_method 432 ) 433 434 for gc_peak in self.gcms: 435 # print(gc_peak.retention_time) 436 # print(gc_peak.tic) 437 438 # check if there is a compound candidate 439 peak_group = hdf_handle.create_group(str(gc_peak.retention_time)) 440 peak_group.attrs["deconvolution"] = int( 441 self.gcms.chromatogram_settings.use_deconvolution 442 ) 443 444 peak_group.attrs["start_scan"] = gc_peak.start_scan 445 peak_group.attrs["apex_scan"] = gc_peak.apex_scan 446 peak_group.attrs["final_scan"] = gc_peak.final_scan 447 448 peak_group.attrs["retention_index"] = gc_peak.ri 449 peak_group.attrs["retention_time"] = gc_peak.retention_time 450 peak_group.attrs["area"] = gc_peak.area 451 452 mz = peak_group.create_dataset( 453 "mz", data=np.array(gc_peak.mass_spectrum.mz_exp), dtype="f8" 454 ) 455 abundance = peak_group.create_dataset( 456 "abundance", 457 data=np.array(gc_peak.mass_spectrum.abundance), 458 dtype="f8", 459 ) 460 461 if gc_peak: 462 if output_score_method == "highest_sim_score": 463 compound_obj = gc_peak.highest_score_compound 464 add_compound(gc_peak, compound_obj) 465 466 elif output_score_method == "highest_ss": 467 compound_obj = gc_peak.highest_ss_compound 468 add_compound(gc_peak, compound_obj) 469 470 else: 471 for compound_obj in gc_peak: 472 add_compound(gc_peak, compound_obj)
Export the data to an HDF5 file.
Parameters:
id_label : str, optional The ID label for the data. Default is "corems:".
def
get_data_stats(self, gcms):
474 def get_data_stats(self, gcms): 475 """Get statistics about the GCMS data. 476 477 Parameters: 478 ---------- 479 gcms : object 480 The low resolution GCMS object. 481 482 Returns: 483 ------- 484 dict 485 A dictionary containing the data statistics. 486 """ 487 488 matched_peaks = gcms.matched_peaks 489 no_matched_peaks = gcms.no_matched_peaks 490 unique_metabolites = gcms.unique_metabolites 491 492 peak_matchs_above_0p85 = 0 493 unique_peak_match_above_0p85 = 0 494 for match_peak in matched_peaks: 495 gc_peak_above_85 = 0 496 matches_above_85 = list( 497 filter(lambda m: m.similarity_score >= 0.85, match_peak) 498 ) 499 if matches_above_85: 500 peak_matchs_above_0p85 += 1 501 if len(matches_above_85) == 1: 502 unique_peak_match_above_0p85 += 1 503 504 data_stats = {} 505 data_stats["average_signal_noise"] = "ni" 506 data_stats["chromatogram_dynamic_range"] = gcms.dynamic_range 507 data_stats["total_number_peaks"] = len(gcms) 508 data_stats["total_peaks_matched"] = len(matched_peaks) 509 data_stats["total_peaks_without_matches"] = len(no_matched_peaks) 510 data_stats["total_matches_above_similarity_score_0.85"] = peak_matchs_above_0p85 511 data_stats["single_matches_above_similarity_score_0.85"] = ( 512 unique_peak_match_above_0p85 513 ) 514 data_stats["unique_metabolites"] = len(unique_metabolites) 515 516 return data_stats
Get statistics about the GCMS data.
Parameters:
gcms : object The low resolution GCMS object.
Returns:
dict A dictionary containing the data statistics.
def
get_calibration_stats(self, gcms, id_label):
518 def get_calibration_stats(self, gcms, id_label): 519 """Get statistics about the GC-MS calibration. 520 521 Parameters: 522 ---------- 523 """ 524 calibration_parameters = {} 525 526 calibration_parameters["calibration_rt_ri_pairs_ref"] = gcms.ri_pairs_ref 527 calibration_parameters["data_url"] = str(gcms.cal_file_path) 528 calibration_parameters["has_input"] = id_label + corems_md5(gcms.cal_file_path) 529 calibration_parameters["data_name"] = str(gcms.cal_file_path.stem) 530 calibration_parameters["calibration_method"] = "" 531 532 return calibration_parameters
Get statistics about the GC-MS calibration.
Parameters:
def
get_blank_stats(self, gcms):
534 def get_blank_stats(self, gcms): 535 """Get statistics about the GC-MS blank.""" 536 blank_parameters = {} 537 538 blank_parameters["data_name"] = "ni" 539 blank_parameters["blank_id"] = "ni" 540 blank_parameters["data_url"] = "ni" 541 blank_parameters["has_input"] = "ni" 542 blank_parameters["common_features_to_blank"] = "ni" 543 544 return blank_parameters
Get statistics about the GC-MS blank.
def
get_instrument_metadata(self, gcms):
546 def get_instrument_metadata(self, gcms): 547 """Get metadata about the GC-MS instrument.""" 548 instrument_metadata = {} 549 550 instrument_metadata["analyzer"] = gcms.analyzer 551 instrument_metadata["instrument_label"] = gcms.instrument_label 552 instrument_metadata["instrument_id"] = uuid.uuid4().hex 553 554 return instrument_metadata
Get metadata about the GC-MS instrument.
def
get_data_metadata(self, gcms, id_label, output_path):
556 def get_data_metadata(self, gcms, id_label, output_path): 557 """Get metadata about the GC-MS data. 558 559 Parameters: 560 ---------- 561 gcms : object 562 The low resolution GCMS object. 563 id_label : str 564 The ID label for the data. 565 output_path : str 566 The output file path. 567 568 Returns: 569 ------- 570 dict 571 A dictionary containing the data metadata. 572 """ 573 if isinstance(output_path, str): 574 output_path = Path(output_path) 575 576 paramaters_path = output_path.with_suffix(".json") 577 578 if paramaters_path.exists(): 579 with paramaters_path.open() as current_param: 580 metadata = json.load(current_param) 581 data_metadata = metadata.get("Data") 582 else: 583 data_metadata = {} 584 data_metadata["data_name"] = [] 585 data_metadata["input_data_url"] = [] 586 data_metadata["has_input"] = [] 587 588 data_metadata["data_name"].append(gcms.sample_name) 589 data_metadata["input_data_url"].append(str(gcms.file_location)) 590 data_metadata["has_input"].append(id_label + corems_md5(gcms.file_location)) 591 592 data_metadata["output_data_name"] = str(output_path.stem) 593 data_metadata["output_data_url"] = str(output_path) 594 data_metadata["has_output"] = id_label + corems_md5(output_path) 595 596 return data_metadata
Get metadata about the GC-MS data.
Parameters:
gcms : object The low resolution GCMS object. id_label : str The ID label for the data. output_path : str The output file path.
Returns:
dict A dictionary containing the data metadata.
def
get_parameters_json(self, gcms, id_label, output_path):
598 def get_parameters_json(self, gcms, id_label, output_path): 599 """Get the parameters as a JSON string. 600 601 Parameters: 602 ---------- 603 gcms : GCMS object 604 The low resolution GCMS object. 605 id_label : str 606 The ID label for the data. 607 output_path : str 608 The output file path. 609 610 Returns: 611 ------- 612 str 613 The parameters as a JSON string. 614 """ 615 616 output_parameters_dict = {} 617 output_parameters_dict["Data"] = self.get_data_metadata( 618 gcms, id_label, output_path 619 ) 620 output_parameters_dict["Stats"] = self.get_data_stats(gcms) 621 output_parameters_dict["Calibration"] = self.get_calibration_stats( 622 gcms, id_label 623 ) 624 output_parameters_dict["Blank"] = self.get_blank_stats(gcms) 625 output_parameters_dict["Instrument"] = self.get_instrument_metadata(gcms) 626 corems_dict_setting = parameter_to_dict.get_dict_data_gcms(gcms) 627 corems_dict_setting["corems_version"] = __version__ 628 output_parameters_dict["CoreMSParameters"] = corems_dict_setting 629 output_parameters_dict["has_metabolite"] = gcms.metabolites_data 630 output = json.dumps( 631 output_parameters_dict, sort_keys=False, indent=4, separators=(",", ": ") 632 ) 633 634 return output
Get the parameters as a JSON string.
Parameters:
gcms : GCMS object The low resolution GCMS object. id_label : str The ID label for the data. output_path : str The output file path.
Returns:
str The parameters as a JSON string.
def
write_settings(self, output_path, gcms, id_label='emsl:'):
636 def write_settings(self, output_path, gcms, id_label="emsl:"): 637 """Write the settings to a JSON file. 638 639 Parameters: 640 ---------- 641 output_path : str 642 The output file path. 643 gcms : GCMS object 644 The low resolution GCMS object. 645 id_label : str 646 The ID label for the data. Default is "emsl:". 647 648 """ 649 650 output = self.get_parameters_json(gcms, id_label, output_path) 651 652 with open( 653 output_path.with_suffix(".json"), 654 "w", 655 encoding="utf8", 656 ) as outfile: 657 outfile.write(output)
Write the settings to a JSON file.
Parameters:
output_path : str The output file path. gcms : GCMS object The low resolution GCMS object. id_label : str The ID label for the data. Default is "emsl:".
def
get_list_dict_data(self, gcms, include_no_match=True, no_match_inline=False):
659 def get_list_dict_data(self, gcms, include_no_match=True, no_match_inline=False): 660 """Get the exported data as a list of dictionaries. 661 662 Parameters: 663 ---------- 664 gcms : object 665 The low resolution GCMS object. 666 include_no_match : bool, optional 667 Whether to include no match data. Default is True. 668 no_match_inline : bool, optional 669 Whether to include no match data inline. Default is False. 670 671 Returns: 672 ------- 673 list 674 The exported data as a list of dictionaries. 675 """ 676 677 output_score_method = gcms.molecular_search_settings.output_score_method 678 679 dict_data_list = [] 680 681 def add_match_dict_data(): 682 derivatization = "{}:{}:{}".format( 683 compound_obj.classify, 684 compound_obj.derivativenum, 685 compound_obj.derivatization, 686 ) 687 out_dict = { 688 "Sample name": gcms.sample_name, 689 "Peak Index": gcpeak_index, 690 "Retention Time": gc_peak.retention_time, 691 "Retention Time Ref": compound_obj.retention_time, 692 "Peak Height": gc_peak.tic, 693 "Peak Area": gc_peak.area, 694 "Retention index": gc_peak.ri, 695 "Retention index Ref": compound_obj.ri, 696 "Retention Index Score": compound_obj.ri_score, 697 "Spectral Similarity Score": compound_obj.spectral_similarity_score, 698 "Similarity Score": compound_obj.similarity_score, 699 "Compound Name": compound_obj.name, 700 "Chebi ID": compound_obj.metadata.chebi, 701 "Kegg Compound ID": compound_obj.metadata.kegg, 702 "Inchi": compound_obj.metadata.inchi, 703 "Inchi Key": compound_obj.metadata.inchikey, 704 "Smiles": compound_obj.metadata.smiles, 705 "Molecular Formula": compound_obj.formula, 706 "IUPAC Name": compound_obj.metadata.iupac_name, 707 "Traditional Name": compound_obj.metadata.traditional_name, 708 "Common Name": compound_obj.metadata.common_name, 709 "Derivatization": derivatization, 710 } 711 712 if self.gcms.molecular_search_settings.exploratory_mode: 713 out_dict.update( 714 { 715 "Weighted Cosine Correlation": compound_obj.spectral_similarity_scores.get( 716 "weighted_cosine_correlation" 717 ), 718 "Cosine Correlation": compound_obj.spectral_similarity_scores.get( 719 "cosine_correlation" 720 ), 721 "Stein Scott Similarity": compound_obj.spectral_similarity_scores.get( 722 "stein_scott_similarity" 723 ), 724 "Pearson Correlation": compound_obj.spectral_similarity_scores.get( 725 "pearson_correlation" 726 ), 727 "Spearman Correlation": compound_obj.spectral_similarity_scores.get( 728 "spearman_correlation" 729 ), 730 "Kendall Tau Correlation": compound_obj.spectral_similarity_scores.get( 731 "kendall_tau_correlation" 732 ), 733 "DFT Correlation": compound_obj.spectral_similarity_scores.get( 734 "dft_correlation" 735 ), 736 "DWT Correlation": compound_obj.spectral_similarity_scores.get( 737 "dwt_correlation" 738 ), 739 "Euclidean Distance": compound_obj.spectral_similarity_scores.get( 740 "euclidean_distance" 741 ), 742 "Manhattan Distance": compound_obj.spectral_similarity_scores.get( 743 "manhattan_distance" 744 ), 745 "Jaccard Distance": compound_obj.spectral_similarity_scores.get( 746 "jaccard_distance" 747 ), 748 } 749 ) 750 for method in methods_name: 751 out_dict[methods_name.get(method)] = ( 752 compound_obj.spectral_similarity_scores.get(method) 753 ) 754 755 dict_data_list.append(out_dict) 756 757 def add_no_match_dict_data(): 758 dict_data_list.append( 759 { 760 "Sample name": gcms.sample_name, 761 "Peak Index": gcpeak_index, 762 "Retention Time": gc_peak.retention_time, 763 "Peak Height": gc_peak.tic, 764 "Peak Area": gc_peak.area, 765 "Retention index": gc_peak.ri, 766 } 767 ) 768 769 for gcpeak_index, gc_peak in enumerate(gcms.sorted_gcpeaks): 770 # check if there is a compound candidate 771 if gc_peak: 772 if output_score_method == "highest_sim_score": 773 compound_obj = gc_peak.highest_score_compound 774 add_match_dict_data() 775 776 elif output_score_method == "highest_ss": 777 compound_obj = gc_peak.highest_ss_compound 778 add_match_dict_data() 779 780 else: 781 for compound_obj in gc_peak: 782 add_match_dict_data() # add monoisotopic peak 783 784 else: 785 # include not_match 786 if include_no_match and no_match_inline: 787 add_no_match_dict_data() 788 789 if include_no_match and not no_match_inline: 790 for gcpeak_index, gc_peak in enumerate(gcms.sorted_gcpeaks): 791 if not gc_peak: 792 add_no_match_dict_data() 793 794 return dict_data_list
Get the exported data as a list of dictionaries.
Parameters:
gcms : object The low resolution GCMS object. include_no_match : bool, optional Whether to include no match data. Default is True. no_match_inline : bool, optional Whether to include no match data inline. Default is False.
Returns:
list The exported data as a list of dictionaries.
797class HighResMassSpectraExport(HighResMassSpecExport): 798 """A class to export high resolution mass spectra data. 799 800 This class provides methods to export high resolution mass spectra data to various formats 801 such as Excel, CSV, Parquet, HDF5, and Pandas DataFrame. 802 803 Parameters 804 ---------- 805 out_file_path : str | Path 806 The output file path. 807 mass_spectra : object 808 The high resolution mass spectra object. 809 output_type : str, optional 810 The output type. Default is 'excel'. 811 812 Attributes 813 ---------- 814 output_file : Path 815 The output file path without suffix 816 dir_loc : Path 817 The directory location for the output file, 818 by default this will be the output_file + ".corems" and all output files will be 819 written into this location 820 mass_spectra : MassSpectraBase 821 The high resolution mass spectra object. 822 """ 823 824 def __init__(self, out_file_path, mass_spectra, output_type="excel"): 825 super().__init__( 826 out_file_path=out_file_path, mass_spectrum=None, output_type=output_type 827 ) 828 829 self.dir_loc = Path(out_file_path + ".corems") 830 self.dir_loc.mkdir(exist_ok=True) 831 # Place the output file in the directory 832 self.output_file = self.dir_loc / Path(out_file_path).name 833 self._output_type = output_type # 'excel', 'csv', 'pandas', 'parquet' or 'hdf5' 834 self.mass_spectra = mass_spectra 835 self.atoms_order_list = None 836 self._init_columns() 837 838 def get_pandas_df(self): 839 """Get the mass spectra as a list of Pandas DataFrames.""" 840 841 list_df = [] 842 843 for mass_spectrum in self.mass_spectra: 844 columns = self.columns_label + self.get_all_used_atoms_in_order( 845 mass_spectrum 846 ) 847 848 dict_data_list = self.get_list_dict_data(mass_spectrum) 849 850 df = DataFrame(dict_data_list, columns=columns) 851 852 scan_number = mass_spectrum.scan_number 853 854 df.attrs["name"] = str(self.output_file) + "_" + str(scan_number) 855 856 list_df.append(df) 857 858 return list_df 859 860 def to_pandas(self, write_metadata=True): 861 """Export the data to a Pandas DataFrame and save it as a pickle file. 862 863 Parameters: 864 ---------- 865 write_metadata : bool, optional 866 Whether to write metadata to the output file. Default is True. 867 """ 868 869 for mass_spectrum in self.mass_spectra: 870 columns = self.columns_label + self.get_all_used_atoms_in_order( 871 mass_spectrum 872 ) 873 874 dict_data_list = self.get_list_dict_data(mass_spectrum) 875 876 df = DataFrame(dict_data_list, columns=columns) 877 878 scan_number = mass_spectrum.scan_number 879 880 out_filename = Path( 881 "%s_scan%s%s" % (self.output_file, str(scan_number), ".pkl") 882 ) 883 884 df.to_pickle(self.dir_loc / out_filename) 885 886 if write_metadata: 887 self.write_settings( 888 self.dir_loc / out_filename.with_suffix(""), mass_spectrum 889 ) 890 891 def to_excel(self, write_metadata=True): 892 """Export the data to an Excel file. 893 894 Parameters: 895 ---------- 896 write_metadata : bool, optional 897 Whether to write metadata to the output file. Default is True. 898 """ 899 for mass_spectrum in self.mass_spectra: 900 columns = self.columns_label + self.get_all_used_atoms_in_order( 901 mass_spectrum 902 ) 903 904 dict_data_list = self.get_list_dict_data(mass_spectrum) 905 906 df = DataFrame(dict_data_list, columns=columns) 907 908 scan_number = mass_spectrum.scan_number 909 910 out_filename = Path( 911 "%s_scan%s%s" % (self.output_file, str(scan_number), ".xlsx") 912 ) 913 914 df.to_excel(self.dir_loc / out_filename) 915 916 if write_metadata: 917 self.write_settings( 918 self.dir_loc / out_filename.with_suffix(""), mass_spectrum 919 ) 920 921 def to_csv(self, write_metadata=True): 922 """Export the data to a CSV file. 923 924 Parameters: 925 ---------- 926 write_metadata : bool, optional 927 Whether to write metadata to the output file. Default is True. 928 """ 929 import csv 930 931 for mass_spectrum in self.mass_spectra: 932 columns = self.columns_label + self.get_all_used_atoms_in_order( 933 mass_spectrum 934 ) 935 936 scan_number = mass_spectrum.scan_number 937 938 dict_data_list = self.get_list_dict_data(mass_spectrum) 939 940 out_filename = Path( 941 "%s_scan%s%s" % (self.output_file, str(scan_number), ".csv") 942 ) 943 944 with open(self.dir_loc / out_filename, "w", newline="") as csvfile: 945 writer = csv.DictWriter(csvfile, fieldnames=columns) 946 writer.writeheader() 947 for data in dict_data_list: 948 writer.writerow(data) 949 950 if write_metadata: 951 self.write_settings( 952 self.dir_loc / out_filename.with_suffix(""), mass_spectrum 953 ) 954 955 def to_parquet(self, write_metadata=True): 956 """Export the data to a Parquet file. 957 958 Parameters: 959 ---------- 960 write_metadata : bool, optional 961 Whether to write metadata to the output file. Default is True. 962 963 Notes 964 ----- 965 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 966 """ 967 for mass_spectrum in self.mass_spectra: 968 columns = self.columns_label + self.get_all_used_atoms_in_order( 969 mass_spectrum 970 ) 971 972 dict_data_list = self.get_list_dict_data(mass_spectrum) 973 974 df = DataFrame(dict_data_list, columns=columns) 975 976 scan_number = mass_spectrum.scan_number 977 978 out_filename = Path( 979 "%s_scan%s%s" % (self.output_file, str(scan_number), ".parquet") 980 ) 981 982 df.to_parquet(self.dir_loc / out_filename, index=False) 983 984 if write_metadata: 985 self.write_settings( 986 self.dir_loc / out_filename.with_suffix(""), mass_spectrum 987 ) 988 989 def get_mass_spectra_attrs(self): 990 """Get the mass spectra attributes as a JSON string. 991 992 Parameters: 993 ---------- 994 mass_spectra : object 995 The high resolution mass spectra object. 996 997 Returns: 998 ------- 999 str 1000 The mass spectra attributes as a JSON string. 1001 """ 1002 dict_ms_attrs = {} 1003 dict_ms_attrs["analyzer"] = self.mass_spectra.analyzer 1004 dict_ms_attrs["instrument_label"] = self.mass_spectra.instrument_label 1005 dict_ms_attrs["sample_name"] = self.mass_spectra.sample_name 1006 1007 return json.dumps( 1008 dict_ms_attrs, sort_keys=False, indent=4, separators=(",", ": ") 1009 ) 1010 1011 def to_hdf( 1012 self, 1013 overwrite=False, 1014 export_raw=True, 1015 compression=DEFAULT_HDF5_COMPRESSION, 1016 profile_dtype="f4", 1017 ): 1018 """Export the data to an HDF5 file. 1019 1020 Parameters 1021 ---------- 1022 overwrite : bool, optional 1023 Whether to overwrite the output file. Default is False. 1024 export_raw : bool, optional 1025 Whether to export the raw mass spectra data. Default is True. 1026 compression : int, str or None, optional 1027 Compression for the written datasets: a gzip level 0-9, 1028 ``"lzf"``, or ``None``/``"none"``. Default is 4. ``"lzf"`` is 1029 h5py's own filter, not a standard HDF5 one -- it needs h5py (or 1030 a tool with the lzf plugin) to read the result back. 1031 profile_dtype : str, optional 1032 dtype used for the stored profile m/z/abundance arrays 1033 (``"f4"`` or ``"f8"``). Default is ``"f4"``. 1034 """ 1035 if overwrite: 1036 if self.output_file.with_suffix(".hdf5").exists(): 1037 self.output_file.with_suffix(".hdf5").unlink() 1038 1039 with h5py.File(self.output_file.with_suffix(".hdf5"), "a") as hdf_handle: 1040 if not hdf_handle.attrs.get("date_utc"): 1041 # Set metadata for all mass spectra 1042 timenow = str( 1043 datetime.now(timezone.utc).strftime("%d/%m/%Y %H:%M:%S %Z") 1044 ) 1045 hdf_handle.attrs["date_utc"] = timenow 1046 hdf_handle.attrs["filename"] = self.mass_spectra.file_location.name 1047 hdf_handle.attrs["data_structure"] = "mass_spectra" 1048 hdf_handle.attrs["analyzer"] = self.mass_spectra.analyzer 1049 hdf_handle.attrs["instrument_label"] = ( 1050 self.mass_spectra.instrument_label 1051 ) 1052 hdf_handle.attrs["sample_name"] = self.mass_spectra.sample_name 1053 hdf_handle.attrs["polarity"] = self.mass_spectra.polarity 1054 hdf_handle.attrs["parser_type"] = ( 1055 self.mass_spectra.spectra_parser_class.__name__ 1056 ) 1057 hdf_handle.attrs["original_file_location"] = ( 1058 self.mass_spectra.file_location._str 1059 ) 1060 1061 # Save creation time from the cached parser when one is configured. 1062 try: 1063 if ( 1064 getattr(self.mass_spectra, "spectra_parser_class", None) 1065 is not None 1066 ): 1067 parser = self.mass_spectra.spectra_parser 1068 if parser is not None and hasattr(parser, "get_creation_time"): 1069 creation_time = parser.get_creation_time() 1070 if creation_time is not None: 1071 hdf_handle.attrs["creation_time"] = ( 1072 creation_time.isoformat() 1073 ) 1074 except Exception: 1075 pass # If creation time cannot be retrieved, skip it 1076 1077 if "mass_spectra" not in hdf_handle: 1078 mass_spectra_group = hdf_handle.create_group("mass_spectra") 1079 else: 1080 mass_spectra_group = hdf_handle.get("mass_spectra") 1081 1082 for mass_spectrum in self.mass_spectra: 1083 group_key = str(int(mass_spectrum.scan_number)) 1084 1085 self.add_mass_spectrum_to_hdf5( 1086 hdf_handle, 1087 mass_spectrum, 1088 group_key, 1089 mass_spectra_group, 1090 export_raw, 1091 compression=compression, 1092 profile_dtype=profile_dtype, 1093 )
A class to export high resolution mass spectra data.
This class provides methods to export high resolution mass spectra data to various formats such as Excel, CSV, Parquet, HDF5, and Pandas DataFrame.
Parameters
- out_file_path (str | Path): The output file path.
- mass_spectra (object): The high resolution mass spectra object.
- output_type (str, optional): The output type. Default is 'excel'.
Attributes
- output_file (Path): The output file path without suffix
- dir_loc (Path): The directory location for the output file, by default this will be the output_file + ".corems" and all output files will be written into this location
- mass_spectra (MassSpectraBase): The high resolution mass spectra object.
HighResMassSpectraExport(out_file_path, mass_spectra, output_type='excel')
824 def __init__(self, out_file_path, mass_spectra, output_type="excel"): 825 super().__init__( 826 out_file_path=out_file_path, mass_spectrum=None, output_type=output_type 827 ) 828 829 self.dir_loc = Path(out_file_path + ".corems") 830 self.dir_loc.mkdir(exist_ok=True) 831 # Place the output file in the directory 832 self.output_file = self.dir_loc / Path(out_file_path).name 833 self._output_type = output_type # 'excel', 'csv', 'pandas', 'parquet' or 'hdf5' 834 self.mass_spectra = mass_spectra 835 self.atoms_order_list = None 836 self._init_columns()
This constructor should always be called with keyword arguments. Arguments are:
group should be None; reserved for future extension when a ThreadGroup class is implemented.
target is the callable object to be invoked by the run() method. Defaults to None, meaning nothing is called.
name is the thread name. By default, a unique name is constructed of the form "Thread-N" where N is a small decimal number.
args is a list or tuple of arguments for the target invocation. Defaults to ().
kwargs is a dictionary of keyword arguments for the target invocation. Defaults to {}.
If a subclass overrides the constructor, it must make sure to invoke the base class constructor (Thread.__init__()) before doing anything else to the thread.
def
get_pandas_df(self):
838 def get_pandas_df(self): 839 """Get the mass spectra as a list of Pandas DataFrames.""" 840 841 list_df = [] 842 843 for mass_spectrum in self.mass_spectra: 844 columns = self.columns_label + self.get_all_used_atoms_in_order( 845 mass_spectrum 846 ) 847 848 dict_data_list = self.get_list_dict_data(mass_spectrum) 849 850 df = DataFrame(dict_data_list, columns=columns) 851 852 scan_number = mass_spectrum.scan_number 853 854 df.attrs["name"] = str(self.output_file) + "_" + str(scan_number) 855 856 list_df.append(df) 857 858 return list_df
Get the mass spectra as a list of Pandas DataFrames.
def
to_pandas(self, write_metadata=True):
860 def to_pandas(self, write_metadata=True): 861 """Export the data to a Pandas DataFrame and save it as a pickle file. 862 863 Parameters: 864 ---------- 865 write_metadata : bool, optional 866 Whether to write metadata to the output file. Default is True. 867 """ 868 869 for mass_spectrum in self.mass_spectra: 870 columns = self.columns_label + self.get_all_used_atoms_in_order( 871 mass_spectrum 872 ) 873 874 dict_data_list = self.get_list_dict_data(mass_spectrum) 875 876 df = DataFrame(dict_data_list, columns=columns) 877 878 scan_number = mass_spectrum.scan_number 879 880 out_filename = Path( 881 "%s_scan%s%s" % (self.output_file, str(scan_number), ".pkl") 882 ) 883 884 df.to_pickle(self.dir_loc / out_filename) 885 886 if write_metadata: 887 self.write_settings( 888 self.dir_loc / out_filename.with_suffix(""), mass_spectrum 889 )
Export the data to a Pandas DataFrame and save it as a pickle file.
Parameters:
write_metadata : bool, optional Whether to write metadata to the output file. Default is True.
def
to_excel(self, write_metadata=True):
891 def to_excel(self, write_metadata=True): 892 """Export the data to an Excel file. 893 894 Parameters: 895 ---------- 896 write_metadata : bool, optional 897 Whether to write metadata to the output file. Default is True. 898 """ 899 for mass_spectrum in self.mass_spectra: 900 columns = self.columns_label + self.get_all_used_atoms_in_order( 901 mass_spectrum 902 ) 903 904 dict_data_list = self.get_list_dict_data(mass_spectrum) 905 906 df = DataFrame(dict_data_list, columns=columns) 907 908 scan_number = mass_spectrum.scan_number 909 910 out_filename = Path( 911 "%s_scan%s%s" % (self.output_file, str(scan_number), ".xlsx") 912 ) 913 914 df.to_excel(self.dir_loc / out_filename) 915 916 if write_metadata: 917 self.write_settings( 918 self.dir_loc / out_filename.with_suffix(""), mass_spectrum 919 )
Export the data to an Excel file.
Parameters:
write_metadata : bool, optional Whether to write metadata to the output file. Default is True.
def
to_csv(self, write_metadata=True):
921 def to_csv(self, write_metadata=True): 922 """Export the data to a CSV file. 923 924 Parameters: 925 ---------- 926 write_metadata : bool, optional 927 Whether to write metadata to the output file. Default is True. 928 """ 929 import csv 930 931 for mass_spectrum in self.mass_spectra: 932 columns = self.columns_label + self.get_all_used_atoms_in_order( 933 mass_spectrum 934 ) 935 936 scan_number = mass_spectrum.scan_number 937 938 dict_data_list = self.get_list_dict_data(mass_spectrum) 939 940 out_filename = Path( 941 "%s_scan%s%s" % (self.output_file, str(scan_number), ".csv") 942 ) 943 944 with open(self.dir_loc / out_filename, "w", newline="") as csvfile: 945 writer = csv.DictWriter(csvfile, fieldnames=columns) 946 writer.writeheader() 947 for data in dict_data_list: 948 writer.writerow(data) 949 950 if write_metadata: 951 self.write_settings( 952 self.dir_loc / out_filename.with_suffix(""), mass_spectrum 953 )
Export the data to a CSV file.
Parameters:
write_metadata : bool, optional Whether to write metadata to the output file. Default is True.
def
to_parquet(self, write_metadata=True):
955 def to_parquet(self, write_metadata=True): 956 """Export the data to a Parquet file. 957 958 Parameters: 959 ---------- 960 write_metadata : bool, optional 961 Whether to write metadata to the output file. Default is True. 962 963 Notes 964 ----- 965 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 966 """ 967 for mass_spectrum in self.mass_spectra: 968 columns = self.columns_label + self.get_all_used_atoms_in_order( 969 mass_spectrum 970 ) 971 972 dict_data_list = self.get_list_dict_data(mass_spectrum) 973 974 df = DataFrame(dict_data_list, columns=columns) 975 976 scan_number = mass_spectrum.scan_number 977 978 out_filename = Path( 979 "%s_scan%s%s" % (self.output_file, str(scan_number), ".parquet") 980 ) 981 982 df.to_parquet(self.dir_loc / out_filename, index=False) 983 984 if write_metadata: 985 self.write_settings( 986 self.dir_loc / out_filename.with_suffix(""), mass_spectrum 987 )
Export the data to a Parquet file.
Parameters:
write_metadata : bool, optional Whether to write metadata to the output file. Default is True.
Notes
Requires pyarrow (or fastparquet) for pandas Parquet I/O.
def
get_mass_spectra_attrs(self):
989 def get_mass_spectra_attrs(self): 990 """Get the mass spectra attributes as a JSON string. 991 992 Parameters: 993 ---------- 994 mass_spectra : object 995 The high resolution mass spectra object. 996 997 Returns: 998 ------- 999 str 1000 The mass spectra attributes as a JSON string. 1001 """ 1002 dict_ms_attrs = {} 1003 dict_ms_attrs["analyzer"] = self.mass_spectra.analyzer 1004 dict_ms_attrs["instrument_label"] = self.mass_spectra.instrument_label 1005 dict_ms_attrs["sample_name"] = self.mass_spectra.sample_name 1006 1007 return json.dumps( 1008 dict_ms_attrs, sort_keys=False, indent=4, separators=(",", ": ") 1009 )
Get the mass spectra attributes as a JSON string.
Parameters:
mass_spectra : object The high resolution mass spectra object.
Returns:
str The mass spectra attributes as a JSON string.
def
to_hdf( self, overwrite=False, export_raw=True, compression=4, profile_dtype='f4'):
1011 def to_hdf( 1012 self, 1013 overwrite=False, 1014 export_raw=True, 1015 compression=DEFAULT_HDF5_COMPRESSION, 1016 profile_dtype="f4", 1017 ): 1018 """Export the data to an HDF5 file. 1019 1020 Parameters 1021 ---------- 1022 overwrite : bool, optional 1023 Whether to overwrite the output file. Default is False. 1024 export_raw : bool, optional 1025 Whether to export the raw mass spectra data. Default is True. 1026 compression : int, str or None, optional 1027 Compression for the written datasets: a gzip level 0-9, 1028 ``"lzf"``, or ``None``/``"none"``. Default is 4. ``"lzf"`` is 1029 h5py's own filter, not a standard HDF5 one -- it needs h5py (or 1030 a tool with the lzf plugin) to read the result back. 1031 profile_dtype : str, optional 1032 dtype used for the stored profile m/z/abundance arrays 1033 (``"f4"`` or ``"f8"``). Default is ``"f4"``. 1034 """ 1035 if overwrite: 1036 if self.output_file.with_suffix(".hdf5").exists(): 1037 self.output_file.with_suffix(".hdf5").unlink() 1038 1039 with h5py.File(self.output_file.with_suffix(".hdf5"), "a") as hdf_handle: 1040 if not hdf_handle.attrs.get("date_utc"): 1041 # Set metadata for all mass spectra 1042 timenow = str( 1043 datetime.now(timezone.utc).strftime("%d/%m/%Y %H:%M:%S %Z") 1044 ) 1045 hdf_handle.attrs["date_utc"] = timenow 1046 hdf_handle.attrs["filename"] = self.mass_spectra.file_location.name 1047 hdf_handle.attrs["data_structure"] = "mass_spectra" 1048 hdf_handle.attrs["analyzer"] = self.mass_spectra.analyzer 1049 hdf_handle.attrs["instrument_label"] = ( 1050 self.mass_spectra.instrument_label 1051 ) 1052 hdf_handle.attrs["sample_name"] = self.mass_spectra.sample_name 1053 hdf_handle.attrs["polarity"] = self.mass_spectra.polarity 1054 hdf_handle.attrs["parser_type"] = ( 1055 self.mass_spectra.spectra_parser_class.__name__ 1056 ) 1057 hdf_handle.attrs["original_file_location"] = ( 1058 self.mass_spectra.file_location._str 1059 ) 1060 1061 # Save creation time from the cached parser when one is configured. 1062 try: 1063 if ( 1064 getattr(self.mass_spectra, "spectra_parser_class", None) 1065 is not None 1066 ): 1067 parser = self.mass_spectra.spectra_parser 1068 if parser is not None and hasattr(parser, "get_creation_time"): 1069 creation_time = parser.get_creation_time() 1070 if creation_time is not None: 1071 hdf_handle.attrs["creation_time"] = ( 1072 creation_time.isoformat() 1073 ) 1074 except Exception: 1075 pass # If creation time cannot be retrieved, skip it 1076 1077 if "mass_spectra" not in hdf_handle: 1078 mass_spectra_group = hdf_handle.create_group("mass_spectra") 1079 else: 1080 mass_spectra_group = hdf_handle.get("mass_spectra") 1081 1082 for mass_spectrum in self.mass_spectra: 1083 group_key = str(int(mass_spectrum.scan_number)) 1084 1085 self.add_mass_spectrum_to_hdf5( 1086 hdf_handle, 1087 mass_spectrum, 1088 group_key, 1089 mass_spectra_group, 1090 export_raw, 1091 compression=compression, 1092 profile_dtype=profile_dtype, 1093 )
Export the data to an HDF5 file.
Parameters
- overwrite (bool, optional): Whether to overwrite the output file. Default is False.
- export_raw (bool, optional): Whether to export the raw mass spectra data. Default is True.
- compression (int, str or None, optional):
Compression for the written datasets: a gzip level 0-9,
"lzf", orNone/"none". Default is 4."lzf"is h5py's own filter, not a standard HDF5 one -- it needs h5py (or a tool with the lzf plugin) to read the result back. - profile_dtype (str, optional):
dtype used for the stored profile m/z/abundance arrays
(
"f4"or"f8"). Default is"f4".
1096class LCMSExport(HighResMassSpectraExport): 1097 """A class to export high resolution LC-MS data. 1098 1099 This class provides methods to export high resolution LC-MS data to HDF5 1100 and Parquet (tabular datasets that mirror the main HDF5 groups). 1101 1102 Parameters 1103 ---------- 1104 out_file_path : str | Path 1105 The output file path, do not include the file extension. 1106 lcms_object : LCMSBase 1107 The high resolution lc-ms object. 1108 """ 1109 1110 def __init__(self, out_file_path, mass_spectra): 1111 super().__init__(out_file_path, mass_spectra, output_type="hdf5") 1112 1113 @staticmethod 1114 def _prepare_df_for_parquet(df): 1115 """Make a DataFrame Parquet-safe by normalizing nested/mixed object values. 1116 1117 Parameters 1118 ---------- 1119 df : DataFrame 1120 Input dataframe. 1121 1122 Returns 1123 ------- 1124 DataFrame 1125 Copy suitable for ``DataFrame.to_parquet``. 1126 1127 Notes 1128 ----- 1129 Object columns with mixed Python types (especially ``bool`` mixed with 1130 ``int``) fail under pyarrow. Nested containers are JSON-serialized; 1131 mixed scalar object columns are coerced to a single type or strings. 1132 """ 1133 if df is None or df.empty: 1134 return df 1135 1136 out = df.copy() 1137 out.columns = [str(c) for c in out.columns] 1138 1139 for col in out.columns: 1140 series = out[col] 1141 if series.dtype != object: 1142 continue 1143 1144 def _normalize(value): 1145 if value is None: 1146 return None 1147 try: 1148 # Avoid treating array-likes as NA; only scalar NA checks 1149 if not isinstance( 1150 value, (list, tuple, dict, np.ndarray) 1151 ) and pd.isna(value): 1152 return None 1153 except (TypeError, ValueError): 1154 pass 1155 if isinstance(value, (list, tuple, dict)): 1156 return json.dumps(value) 1157 if isinstance(value, np.ndarray): 1158 return json.dumps(value.tolist()) 1159 if isinstance(value, (bool, np.bool_)): 1160 return bool(value) 1161 if isinstance(value, (np.integer,)): 1162 return int(value) 1163 if isinstance(value, (np.floating,)): 1164 return float(value) 1165 return value 1166 1167 normalized = series.map(_normalize) 1168 non_null = [v for v in normalized.tolist() if v is not None] 1169 1170 has_bool = any(isinstance(v, bool) for v in non_null) 1171 # bool is a subclass of int; exclude bools from int detection 1172 has_int = any( 1173 isinstance(v, int) and not isinstance(v, bool) for v in non_null 1174 ) 1175 has_float = any(isinstance(v, float) for v in non_null) 1176 has_str = any(isinstance(v, str) for v in non_null) 1177 has_other = any( 1178 not isinstance(v, (bool, int, float, str)) for v in non_null 1179 ) 1180 1181 if ( 1182 has_other 1183 or (has_str and (has_bool or has_int or has_float)) 1184 or (has_bool and has_int) 1185 ): 1186 out[col] = normalized.map(lambda v: None if v is None else str(v)) 1187 elif has_bool and not has_float: 1188 out[col] = normalized.map( 1189 lambda v: None if v is None else bool(v) 1190 ).astype("boolean") 1191 elif has_float or has_int: 1192 out[col] = pd.to_numeric(pd.Series(normalized), errors="coerce") 1193 else: 1194 out[col] = normalized.map(lambda v: None if v is None else str(v)) 1195 1196 return out 1197 1198 def _write_parquet(self, df, filename, overwrite=False): 1199 """Write a dataframe to a Parquet file under the export directory. 1200 1201 Parameters 1202 ---------- 1203 df : DataFrame 1204 Data to write. 1205 filename : str 1206 File name (with or without ``.parquet`` suffix). 1207 overwrite : bool, optional 1208 Whether to replace an existing file. Default is False. 1209 1210 Returns 1211 ------- 1212 Path or None 1213 Output path if written, otherwise None. 1214 """ 1215 if df is None or df.empty: 1216 return None 1217 1218 out_path = self.dir_loc / Path(filename).with_suffix(".parquet") 1219 if out_path.exists() and not overwrite: 1220 return out_path 1221 1222 self._prepare_df_for_parquet(df).to_parquet(out_path, index=False) 1223 return out_path 1224 1225 def _mass_spectra_to_parquet_df(self): 1226 """Build a single DataFrame of peak tables for all loaded mass spectra.""" 1227 frames = [] 1228 for mass_spectrum in self.mass_spectra: 1229 if mass_spectrum is None: 1230 continue 1231 columns = self.columns_label + self.get_all_used_atoms_in_order( 1232 mass_spectrum 1233 ) 1234 dict_data_list = self.get_list_dict_data(mass_spectrum) 1235 df = DataFrame(dict_data_list, columns=columns) 1236 if df.empty: 1237 continue 1238 df.insert(0, "scan_number", mass_spectrum.scan_number) 1239 if getattr(mass_spectrum, "retention_time", None) is not None: 1240 df.insert(1, "retention_time", mass_spectrum.retention_time) 1241 frames.append(df) 1242 1243 if not frames: 1244 return DataFrame() 1245 return pd.concat(frames, ignore_index=True) 1246 1247 @staticmethod 1248 def _sequence_len(value): 1249 """Return length of a sequence/array, or 0 for None/empty/scalar-missing.""" 1250 if value is None: 1251 return 0 1252 try: 1253 return len(value) 1254 except TypeError: 1255 return 0 1256 1257 def _eics_to_parquet_df(self): 1258 """Build a long-format DataFrame of EIC time series.""" 1259 frames = [] 1260 for mz, eic_data in self.mass_spectra.eics.items(): 1261 if eic_data is None: 1262 continue 1263 # Avoid `array or []` — numpy arrays are ambiguous in boolean context 1264 time = getattr(eic_data, "time", None) 1265 n = self._sequence_len(time) 1266 if n == 0: 1267 continue 1268 1269 scans = getattr(eic_data, "scans", None) 1270 eic = getattr(eic_data, "eic", None) 1271 eic_smoothed = getattr(eic_data, "eic_smoothed", None) 1272 1273 frame = { 1274 "mz": [mz] * n, 1275 "scans": list(scans) if self._sequence_len(scans) == n else [None] * n, 1276 "time": list(time), 1277 "eic": list(eic) if self._sequence_len(eic) == n else [None] * n, 1278 } 1279 if self._sequence_len(eic_smoothed) == n: 1280 frame["eic_smoothed"] = list(eic_smoothed) 1281 frames.append(DataFrame(frame)) 1282 1283 if not frames: 1284 return DataFrame() 1285 return pd.concat(frames, ignore_index=True) 1286 1287 def _spectral_search_results_to_parquet_df(self): 1288 """Flatten spectral search results into a single DataFrame.""" 1289 frames = [] 1290 for scan_id, by_precursor in self.mass_spectra.spectral_search_results.items(): 1291 for precursor_key, result in by_precursor.items(): 1292 try: 1293 df = result.to_dataframe() 1294 except Exception: 1295 continue 1296 if df is None or df.empty: 1297 continue 1298 df = df.copy() 1299 # to_dataframe() already includes these fields from the result object; 1300 # only set them when missing so we do not insert duplicate columns. 1301 if "query_spectrum_id" not in df.columns: 1302 df.insert( 1303 0, 1304 "query_spectrum_id", 1305 getattr(result, "query_spectrum_id", scan_id), 1306 ) 1307 if "precursor_mz" not in df.columns: 1308 loc = 1 if "query_spectrum_id" in df.columns else 0 1309 df.insert( 1310 loc, 1311 "precursor_mz", 1312 getattr(result, "precursor_mz", precursor_key), 1313 ) 1314 frames.append(df) 1315 1316 if not frames: 1317 return DataFrame() 1318 return pd.concat(frames, ignore_index=True) 1319 1320 def to_parquet( 1321 self, 1322 overwrite=False, 1323 save_parameters=True, 1324 parameter_format="toml", 1325 export_spectra=True, 1326 ): 1327 """Export LC-MS datasets to Parquet files (tabular mirror of ``to_hdf``). 1328 1329 Writes under ``{out_file_path}.corems/``: 1330 1331 - ``scan_info.parquet`` 1332 - ``mass_features.parquet`` 1333 - ``induced_mass_features.parquet`` (if present) 1334 - ``mass_spectra.parquet`` (optional; all peak tables with scan metadata) 1335 - ``eics.parquet`` (if EICs are present and export is enabled) 1336 - ``spectral_search_results.parquet`` (if search results are present) 1337 - parameters as JSON or TOML when ``save_parameters`` is True 1338 1339 Parameters 1340 ---------- 1341 overwrite : bool, optional 1342 Whether to overwrite existing Parquet files. Default is False. 1343 save_parameters : bool, optional 1344 Whether to save processing parameters. Default is True. 1345 parameter_format : str, optional 1346 Parameter file format, ``'json'`` or ``'toml'``. Default is ``'toml'``. 1347 export_spectra : bool, optional 1348 Whether to export loaded mass-spectrum peak tables. Default is True. 1349 1350 Notes 1351 ----- 1352 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 1353 Raw profile arrays and nested HDF5-only blobs are not reproduced in full; 1354 use ``to_hdf`` for a complete self-contained archive. 1355 """ 1356 self.dir_loc.mkdir(exist_ok=True) 1357 1358 # Scan metadata (same source as the HDF5 scan_info group) 1359 scan_df = self.mass_spectra.scan_df 1360 if scan_df is not None and not scan_df.empty: 1361 self._write_parquet( 1362 scan_df.reset_index(drop=True), "scan_info", overwrite=overwrite 1363 ) 1364 1365 # Mass features 1366 if len(self.mass_spectra.mass_features) > 0: 1367 mf_df = self.mass_spectra.mass_features_to_df().reset_index(drop=False) 1368 self._write_parquet(mf_df, "mass_features", overwrite=overwrite) 1369 1370 if len(getattr(self.mass_spectra, "induced_mass_features", {}) or {}) > 0: 1371 imf_df = self.mass_spectra.mass_features_to_df( 1372 induced_features=True 1373 ).reset_index(drop=False) 1374 self._write_parquet(imf_df, "induced_mass_features", overwrite=overwrite) 1375 1376 # Peak tables for loaded spectra 1377 if export_spectra: 1378 spectra_df = self._mass_spectra_to_parquet_df() 1379 self._write_parquet(spectra_df, "mass_spectra", overwrite=overwrite) 1380 1381 # EICs 1382 export_eics = self.mass_spectra.parameters.lc_ms.export_eics 1383 if export_eics and len(self.mass_spectra.eics) > 0: 1384 eic_df = self._eics_to_parquet_df() 1385 self._write_parquet(eic_df, "eics", overwrite=overwrite) 1386 1387 # Spectral search results 1388 if len(self.mass_spectra.spectral_search_results) > 0: 1389 search_df = self._spectral_search_results_to_parquet_df() 1390 self._write_parquet( 1391 search_df, "spectral_search_results", overwrite=overwrite 1392 ) 1393 1394 if save_parameters: 1395 if parameter_format not in ["json", "toml"]: 1396 raise ValueError("parameter_format must be 'json' or 'toml'") 1397 if parameter_format == "json": 1398 dump_lcms_settings_json( 1399 filename=self.output_file.with_suffix(".json"), 1400 lcms_obj=self.mass_spectra, 1401 ) 1402 elif parameter_format == "toml": 1403 dump_lcms_settings_toml( 1404 filename=self.output_file.with_suffix(".toml"), 1405 lcms_obj=self.mass_spectra, 1406 ) 1407 1408 @staticmethod 1409 def _save_mass_features_dict_to_hdf5( 1410 mass_features_dict, 1411 mass_features_group, 1412 overwrite=False, 1413 compression=DEFAULT_HDF5_COMPRESSION, 1414 ): 1415 """Save a dictionary of mass features to an HDF5 group. 1416 1417 This is a helper method that can be reused by different export classes. 1418 1419 Parameters 1420 ---------- 1421 mass_features_dict : dict 1422 Dictionary of mass features to save, keyed by mass feature ID. 1423 mass_features_group : h5py.Group 1424 The HDF5 group to save the mass features to. 1425 overwrite : bool, optional 1426 Whether to overwrite existing mass features. Default is False. 1427 compression : int, str or None, optional 1428 Compression for the written datasets: a gzip level 0-9, 1429 ``"lzf"``, or ``None``/``"none"``. Default is 4. ``"lzf"`` is 1430 h5py's own filter, not a standard HDF5 one -- it needs h5py (or 1431 a tool with the lzf plugin) to read the result back. 1432 """ 1433 1434 # Create group for each mass feature, with key as the mass feature id 1435 for k, v in mass_features_dict.items(): 1436 if str(k) not in mass_features_group or overwrite: 1437 if str(k) in mass_features_group and overwrite: 1438 del mass_features_group[str(k)] 1439 mass_features_group.create_group(str(k)) 1440 # Loop through each of the mass feature attributes and add them as attributes (if single value) or datasets (if array) 1441 for k2, v2 in v.__dict__.items(): 1442 if v2 is not None: 1443 # Check if the attribute is an integer or float and set as an attribute in the mass feature group 1444 if k2 not in [ 1445 "chromatogram_parent", 1446 "ms2_mass_spectra", 1447 "mass_spectrum", 1448 "_eic_data", 1449 "ms2_similarity_results", 1450 ]: 1451 if k2 == "ms2_scan_numbers": 1452 array = np.array(v2) 1453 # Convert int64 to int32 1454 if array.dtype == np.int64: 1455 array = array.astype(np.int32) 1456 mass_features_group[str(k)].create_dataset( 1457 str(k2), 1458 data=array, 1459 **hdf5_dataset_kwargs( 1460 array.shape, compression=compression 1461 ), 1462 ) 1463 elif k2 == "_half_height_width": 1464 array = np.array(v2) 1465 # Convert float64 to float32 1466 if array.dtype == np.float64: 1467 array = array.astype(np.float32) 1468 mass_features_group[str(k)].create_dataset( 1469 str(k2), 1470 data=array, 1471 **hdf5_dataset_kwargs( 1472 array.shape, compression=compression 1473 ), 1474 ) 1475 elif k2 == "_ms_deconvoluted_idx": 1476 array = np.array(v2) 1477 # Convert int64 to int32 1478 if array.dtype == np.int64: 1479 array = array.astype(np.int32) 1480 mass_features_group[str(k)].create_dataset( 1481 str(k2), 1482 data=array, 1483 **hdf5_dataset_kwargs( 1484 array.shape, compression=compression 1485 ), 1486 ) 1487 elif k2 == "associated_mass_features_deconvoluted": 1488 array = np.array(v2) 1489 # Convert int64 to int32 1490 if array.dtype == np.int64: 1491 array = array.astype(np.int32) 1492 mass_features_group[str(k)].create_dataset( 1493 str(k2), 1494 data=array, 1495 **hdf5_dataset_kwargs( 1496 array.shape, compression=compression 1497 ), 1498 ) 1499 elif k2 == "_noise_score": 1500 array = np.array(v2) 1501 # Convert float64 to float32 1502 if array.dtype == np.float64: 1503 array = array.astype(np.float32) 1504 mass_features_group[str(k)].create_dataset( 1505 str(k2), 1506 data=array, 1507 **hdf5_dataset_kwargs( 1508 array.shape, compression=compression 1509 ), 1510 ) 1511 elif ( 1512 isinstance(v2, int) 1513 or isinstance(v2, float) 1514 or isinstance(v2, str) 1515 or isinstance(v2, np.integer) 1516 or isinstance(v2, np.float32) 1517 or isinstance(v2, np.float64) 1518 or isinstance(v2, np.bool_) 1519 ): 1520 # Convert numpy types to smaller precision for storage 1521 if isinstance(v2, np.int64): 1522 v2 = np.int32(v2) 1523 elif isinstance(v2, np.float64): 1524 v2 = np.float32(v2) 1525 mass_features_group[str(k)].attrs[str(k2)] = v2 1526 1527 @staticmethod 1528 def _save_eics_dict_to_hdf5( 1529 eics_dict, eics_group, overwrite=False, compression=DEFAULT_HDF5_COMPRESSION 1530 ): 1531 """Save a dictionary of EICs to an HDF5 group. 1532 1533 This is a static helper method that can be reused by different export classes 1534 to save EIC data in a consistent format. 1535 1536 Parameters 1537 ---------- 1538 eics_dict : dict 1539 Dictionary of EIC_Data objects, keyed by m/z value. 1540 eics_group : h5py.Group 1541 The HDF5 group to save the EICs to. 1542 overwrite : bool, optional 1543 Whether to overwrite existing EICs. Default is False. 1544 compression : int, str or None, optional 1545 Compression for the written datasets: a gzip level 0-9, 1546 ``"lzf"``, or ``None``/``"none"``. Default is 4. ``"lzf"`` is 1547 h5py's own filter, not a standard HDF5 one -- it needs h5py (or 1548 a tool with the lzf plugin) to read the result back. 1549 """ 1550 for mz, eic_data in eics_dict.items(): 1551 mz_str = str(mz) 1552 if mz_str not in eics_group or overwrite: 1553 if mz_str in eics_group and overwrite: 1554 del eics_group[mz_str] 1555 eic_grp = eics_group.create_group(mz_str) 1556 eic_grp.attrs["mz"] = mz 1557 1558 # Save all EIC_Data attributes as datasets 1559 for attr_name, attr_value in eic_data.__dict__.items(): 1560 if attr_value is not None: 1561 array = np.array(attr_value) 1562 # Apply data type optimization and compression 1563 if array.dtype == np.int64: 1564 array = array.astype(np.int32) 1565 elif array.dtype == np.float64: 1566 array = array.astype(np.float32) 1567 elif array.dtype.str[0:2] == "<U": 1568 # Convert Unicode strings to UTF-8 encoded strings 1569 string_data = [str(item) for item in array] 1570 string_dtype = h5py.string_dtype(encoding="utf-8") 1571 eic_grp.create_dataset( 1572 str(attr_name), 1573 data=string_data, 1574 dtype=string_dtype, 1575 **hdf5_dataset_kwargs( 1576 array.shape, compression=compression, vlen_str=True 1577 ), 1578 ) 1579 continue 1580 eic_grp.create_dataset( 1581 str(attr_name), 1582 data=array, 1583 **hdf5_dataset_kwargs(array.shape, compression=compression), 1584 ) 1585 1586 def _save_mass_features_to_hdf5( 1587 self, 1588 hdf_handle, 1589 group_name="mass_features", 1590 overwrite=False, 1591 compression=DEFAULT_HDF5_COMPRESSION, 1592 ): 1593 """Save the mass features to the HDF5 file. 1594 1595 Parameters 1596 ---------- 1597 hdf_handle : h5py.File 1598 The HDF5 file handle. 1599 group_name : str, optional 1600 The name of the group to save the mass features to. Default is 'mass_features'. 1601 overwrite : bool, optional 1602 Whether to overwrite the group if it exists. Default is False. 1603 compression : int, str or None, optional 1604 Compression for the written datasets: a gzip level 0-9, 1605 ``"lzf"``, or ``None``/``"none"``. Default is 4. ``"lzf"`` is 1606 h5py's own filter, not a standard HDF5 one -- it needs h5py (or 1607 a tool with the lzf plugin) to read the result back. 1608 """ 1609 # Determine which mass features to save based on group_name 1610 if group_name == "induced_mass_features": 1611 if len(self.mass_spectra.induced_mass_features) == 0: 1612 return # No induced mass features to save 1613 mass_features_dict = self.mass_spectra.induced_mass_features 1614 else: 1615 if len(self.mass_spectra.mass_features) == 0: 1616 return # No mass features to save 1617 mass_features_dict = self.mass_spectra.mass_features 1618 1619 # Add LCMS mass features to hdf5 file 1620 if group_name not in hdf_handle: 1621 mass_features_group = hdf_handle.create_group(group_name) 1622 else: 1623 mass_features_group = hdf_handle.get(group_name) 1624 1625 # Use the static helper method to save the mass features 1626 self._save_mass_features_dict_to_hdf5( 1627 mass_features_dict, mass_features_group, overwrite, compression=compression 1628 ) 1629 1630 def to_hdf( 1631 self, 1632 overwrite=False, 1633 save_parameters=True, 1634 parameter_format="toml", 1635 compression=DEFAULT_HDF5_COMPRESSION, 1636 profile_dtype="f4", 1637 ): 1638 """Export the data to an HDF5. 1639 1640 Parameters 1641 ---------- 1642 overwrite : bool, optional 1643 Whether to overwrite the output file. Default is False. 1644 save_parameters : bool, optional 1645 Whether to save the parameters as a separate json or toml file. Default is True. 1646 parameter_format : str, optional 1647 The format to save the parameters in. Default is 'toml'. 1648 compression : int, str or None, optional 1649 Compression for the written datasets: a gzip level 0-9, 1650 ``"lzf"``, or ``None``/``"none"``. Default is 4. ``"lzf"`` is 1651 h5py's own filter, not a standard HDF5 one -- it needs h5py (or 1652 a tool with the lzf plugin) to read the result back. 1653 profile_dtype : str, optional 1654 dtype used for the stored profile m/z and abundance arrays 1655 (``"f4"`` or ``"f8"``). Default is ``"f4"``. 1656 1657 Raises 1658 ------ 1659 ValueError 1660 If parameter_format is not 'json' or 'toml'. 1661 """ 1662 export_profile_spectra = ( 1663 self.mass_spectra.parameters.lc_ms.export_profile_spectra 1664 ) 1665 1666 # Write the mass spectra data to the hdf5 file 1667 super().to_hdf( 1668 overwrite=overwrite, 1669 export_raw=export_profile_spectra, 1670 compression=compression, 1671 profile_dtype=profile_dtype, 1672 ) 1673 1674 # Write scan info, ms_unprocessed, mass features, eics, and ms2_search results to the hdf5 file 1675 with h5py.File(self.output_file.with_suffix(".hdf5"), "a") as hdf_handle: 1676 # Add scan_info to hdf5 file 1677 if "scan_info" not in hdf_handle or overwrite: 1678 if "scan_info" in hdf_handle and overwrite: 1679 del hdf_handle["scan_info"] 1680 scan_info_group = hdf_handle.create_group("scan_info") 1681 for k, v in self.mass_spectra._scan_info.items(): 1682 array = np.array(list(v.values())) 1683 if array.dtype.str[0:2] == "<U": 1684 array = array.astype("S") 1685 scan_info_group.create_dataset( 1686 k, 1687 data=array, 1688 **hdf5_dataset_kwargs(array.shape, compression=compression), 1689 ) 1690 1691 # Add ms_unprocessed to hdf5 file 1692 export_unprocessed_ms1 = ( 1693 self.mass_spectra.parameters.lc_ms.export_unprocessed_ms1 1694 ) 1695 if self.mass_spectra._ms_unprocessed and export_unprocessed_ms1: 1696 if "ms_unprocessed" not in hdf_handle or overwrite: 1697 if "ms_unprocessed" in hdf_handle and overwrite: 1698 del hdf_handle["ms_unprocessed"] 1699 ms_unprocessed_group = hdf_handle.create_group("ms_unprocessed") 1700 else: 1701 ms_unprocessed_group = hdf_handle.get("ms_unprocessed") 1702 for k, v in self.mass_spectra._ms_unprocessed.items(): 1703 if str(k) not in ms_unprocessed_group or overwrite: 1704 if str(k) in ms_unprocessed_group and overwrite: 1705 del ms_unprocessed_group[str(k)] 1706 array = np.array(v) 1707 column_labels = ( 1708 list(v.columns) 1709 if hasattr(v, "columns") 1710 else ["scan", "mz", "intensity"] 1711 ) 1712 ms_dset = ms_unprocessed_group.create_dataset( 1713 str(k), 1714 data=array, 1715 **hdf5_dataset_kwargs(array.shape, compression=compression), 1716 ) 1717 ms_dset.attrs["column_labels"] = json.dumps(column_labels) 1718 1719 # Add LCMS mass features to hdf5 file 1720 self._save_mass_features_to_hdf5( 1721 hdf_handle, 1722 group_name="mass_features", 1723 overwrite=overwrite, 1724 compression=compression, 1725 ) 1726 self._save_mass_features_to_hdf5( 1727 hdf_handle, 1728 group_name="induced_mass_features", 1729 overwrite=overwrite, 1730 compression=compression, 1731 ) 1732 1733 # Add EIC data to hdf5 file 1734 export_eics = self.mass_spectra.parameters.lc_ms.export_eics 1735 if len(self.mass_spectra.eics) > 0 and export_eics: 1736 if "eics" not in hdf_handle or overwrite: 1737 if "eics" in hdf_handle and overwrite: 1738 del hdf_handle["eics"] 1739 eic_group = hdf_handle.create_group("eics") 1740 else: 1741 eic_group = hdf_handle.get("eics") 1742 1743 # Use the static helper method to save the EICs 1744 self._save_eics_dict_to_hdf5( 1745 self.mass_spectra.eics, 1746 eic_group, 1747 overwrite, 1748 compression=compression, 1749 ) 1750 1751 # Add ms2_search results to hdf5 file (parameterized) 1752 if len(self.mass_spectra.spectral_search_results) > 0: 1753 if "spectral_search_results" not in hdf_handle or overwrite: 1754 if "spectral_search_results" in hdf_handle and overwrite: 1755 del hdf_handle["spectral_search_results"] 1756 spectral_search_results = hdf_handle.create_group( 1757 "spectral_search_results" 1758 ) 1759 else: 1760 spectral_search_results = hdf_handle.get("spectral_search_results") 1761 # Create group for each search result by ms2_scan / precursor_mz 1762 for k, v in self.mass_spectra.spectral_search_results.items(): 1763 # TODO KRH: Fix to handle if export_only_relevant and k not in relevant_scan_numbers: continue! 1764 if str(k) not in spectral_search_results or overwrite: 1765 if str(k) in spectral_search_results and overwrite: 1766 del spectral_search_results[str(k)] 1767 spectral_search_results.create_group(str(k)) 1768 for k2, v2 in v.items(): 1769 spectral_search_results[str(k)].create_group(str(k2)) 1770 spectral_search_results[str(k)][str(k2)].attrs[ 1771 "precursor_mz" 1772 ] = v2.precursor_mz 1773 spectral_search_results[str(k)][str(k2)].attrs[ 1774 "query_spectrum_id" 1775 ] = v2.query_spectrum_id 1776 # Loop through each of the attributes and add them as datasets (if array) 1777 for k3, v3 in v2.__dict__.items(): 1778 if v3 is not None and k3 not in [ 1779 "query_spectrum", 1780 "precursor_mz", 1781 "query_spectrum_id", 1782 ]: 1783 if ( 1784 k3 == "query_frag_types" 1785 or k3 == "ref_frag_types" 1786 ): 1787 v3 = [", ".join(x) for x in v3] 1788 if all(v3 is not None for v3 in v3): 1789 array = np.array(v3) 1790 if array.dtype.str[0:2] == "<U": 1791 array = array.astype("S") 1792 spectral_search_results[str(k)][ 1793 str(k2) 1794 ].create_dataset( 1795 str(k3), 1796 data=array, 1797 **hdf5_dataset_kwargs( 1798 array.shape, compression=compression 1799 ), 1800 ) 1801 1802 # Save parameters as separate json 1803 if save_parameters: 1804 # Check if parameter_format is valid 1805 if parameter_format not in ["json", "toml"]: 1806 raise ValueError("parameter_format must be 'json' or 'toml'") 1807 1808 if parameter_format == "json": 1809 dump_lcms_settings_json( 1810 filename=self.output_file.with_suffix(".json"), 1811 lcms_obj=self.mass_spectra, 1812 ) 1813 elif parameter_format == "toml": 1814 dump_lcms_settings_toml( 1815 filename=self.output_file.with_suffix(".toml"), 1816 lcms_obj=self.mass_spectra, 1817 )
A class to export high resolution LC-MS data.
This class provides methods to export high resolution LC-MS data to HDF5 and Parquet (tabular datasets that mirror the main HDF5 groups).
Parameters
- out_file_path (str | Path): The output file path, do not include the file extension.
- lcms_object (LCMSBase): The high resolution lc-ms object.
LCMSExport(out_file_path, mass_spectra)
1110 def __init__(self, out_file_path, mass_spectra): 1111 super().__init__(out_file_path, mass_spectra, output_type="hdf5")
This constructor should always be called with keyword arguments. Arguments are:
group should be None; reserved for future extension when a ThreadGroup class is implemented.
target is the callable object to be invoked by the run() method. Defaults to None, meaning nothing is called.
name is the thread name. By default, a unique name is constructed of the form "Thread-N" where N is a small decimal number.
args is a list or tuple of arguments for the target invocation. Defaults to ().
kwargs is a dictionary of keyword arguments for the target invocation. Defaults to {}.
If a subclass overrides the constructor, it must make sure to invoke the base class constructor (Thread.__init__()) before doing anything else to the thread.
def
to_parquet( self, overwrite=False, save_parameters=True, parameter_format='toml', export_spectra=True):
1320 def to_parquet( 1321 self, 1322 overwrite=False, 1323 save_parameters=True, 1324 parameter_format="toml", 1325 export_spectra=True, 1326 ): 1327 """Export LC-MS datasets to Parquet files (tabular mirror of ``to_hdf``). 1328 1329 Writes under ``{out_file_path}.corems/``: 1330 1331 - ``scan_info.parquet`` 1332 - ``mass_features.parquet`` 1333 - ``induced_mass_features.parquet`` (if present) 1334 - ``mass_spectra.parquet`` (optional; all peak tables with scan metadata) 1335 - ``eics.parquet`` (if EICs are present and export is enabled) 1336 - ``spectral_search_results.parquet`` (if search results are present) 1337 - parameters as JSON or TOML when ``save_parameters`` is True 1338 1339 Parameters 1340 ---------- 1341 overwrite : bool, optional 1342 Whether to overwrite existing Parquet files. Default is False. 1343 save_parameters : bool, optional 1344 Whether to save processing parameters. Default is True. 1345 parameter_format : str, optional 1346 Parameter file format, ``'json'`` or ``'toml'``. Default is ``'toml'``. 1347 export_spectra : bool, optional 1348 Whether to export loaded mass-spectrum peak tables. Default is True. 1349 1350 Notes 1351 ----- 1352 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 1353 Raw profile arrays and nested HDF5-only blobs are not reproduced in full; 1354 use ``to_hdf`` for a complete self-contained archive. 1355 """ 1356 self.dir_loc.mkdir(exist_ok=True) 1357 1358 # Scan metadata (same source as the HDF5 scan_info group) 1359 scan_df = self.mass_spectra.scan_df 1360 if scan_df is not None and not scan_df.empty: 1361 self._write_parquet( 1362 scan_df.reset_index(drop=True), "scan_info", overwrite=overwrite 1363 ) 1364 1365 # Mass features 1366 if len(self.mass_spectra.mass_features) > 0: 1367 mf_df = self.mass_spectra.mass_features_to_df().reset_index(drop=False) 1368 self._write_parquet(mf_df, "mass_features", overwrite=overwrite) 1369 1370 if len(getattr(self.mass_spectra, "induced_mass_features", {}) or {}) > 0: 1371 imf_df = self.mass_spectra.mass_features_to_df( 1372 induced_features=True 1373 ).reset_index(drop=False) 1374 self._write_parquet(imf_df, "induced_mass_features", overwrite=overwrite) 1375 1376 # Peak tables for loaded spectra 1377 if export_spectra: 1378 spectra_df = self._mass_spectra_to_parquet_df() 1379 self._write_parquet(spectra_df, "mass_spectra", overwrite=overwrite) 1380 1381 # EICs 1382 export_eics = self.mass_spectra.parameters.lc_ms.export_eics 1383 if export_eics and len(self.mass_spectra.eics) > 0: 1384 eic_df = self._eics_to_parquet_df() 1385 self._write_parquet(eic_df, "eics", overwrite=overwrite) 1386 1387 # Spectral search results 1388 if len(self.mass_spectra.spectral_search_results) > 0: 1389 search_df = self._spectral_search_results_to_parquet_df() 1390 self._write_parquet( 1391 search_df, "spectral_search_results", overwrite=overwrite 1392 ) 1393 1394 if save_parameters: 1395 if parameter_format not in ["json", "toml"]: 1396 raise ValueError("parameter_format must be 'json' or 'toml'") 1397 if parameter_format == "json": 1398 dump_lcms_settings_json( 1399 filename=self.output_file.with_suffix(".json"), 1400 lcms_obj=self.mass_spectra, 1401 ) 1402 elif parameter_format == "toml": 1403 dump_lcms_settings_toml( 1404 filename=self.output_file.with_suffix(".toml"), 1405 lcms_obj=self.mass_spectra, 1406 )
Export LC-MS datasets to Parquet files (tabular mirror of to_hdf).
Writes under {out_file_path}.corems/:
scan_info.parquetmass_features.parquetinduced_mass_features.parquet(if present)mass_spectra.parquet(optional; all peak tables with scan metadata)eics.parquet(if EICs are present and export is enabled)spectral_search_results.parquet(if search results are present)- parameters as JSON or TOML when
save_parametersis True
Parameters
- overwrite (bool, optional): Whether to overwrite existing Parquet files. Default is False.
- save_parameters (bool, optional): Whether to save processing parameters. Default is True.
- parameter_format (str, optional):
Parameter file format,
'json'or'toml'. Default is'toml'. - export_spectra (bool, optional): Whether to export loaded mass-spectrum peak tables. Default is True.
Notes
Requires pyarrow (or fastparquet) for pandas Parquet I/O.
Raw profile arrays and nested HDF5-only blobs are not reproduced in full;
use to_hdf for a complete self-contained archive.
def
to_hdf( self, overwrite=False, save_parameters=True, parameter_format='toml', compression=4, profile_dtype='f4'):
1630 def to_hdf( 1631 self, 1632 overwrite=False, 1633 save_parameters=True, 1634 parameter_format="toml", 1635 compression=DEFAULT_HDF5_COMPRESSION, 1636 profile_dtype="f4", 1637 ): 1638 """Export the data to an HDF5. 1639 1640 Parameters 1641 ---------- 1642 overwrite : bool, optional 1643 Whether to overwrite the output file. Default is False. 1644 save_parameters : bool, optional 1645 Whether to save the parameters as a separate json or toml file. Default is True. 1646 parameter_format : str, optional 1647 The format to save the parameters in. Default is 'toml'. 1648 compression : int, str or None, optional 1649 Compression for the written datasets: a gzip level 0-9, 1650 ``"lzf"``, or ``None``/``"none"``. Default is 4. ``"lzf"`` is 1651 h5py's own filter, not a standard HDF5 one -- it needs h5py (or 1652 a tool with the lzf plugin) to read the result back. 1653 profile_dtype : str, optional 1654 dtype used for the stored profile m/z and abundance arrays 1655 (``"f4"`` or ``"f8"``). Default is ``"f4"``. 1656 1657 Raises 1658 ------ 1659 ValueError 1660 If parameter_format is not 'json' or 'toml'. 1661 """ 1662 export_profile_spectra = ( 1663 self.mass_spectra.parameters.lc_ms.export_profile_spectra 1664 ) 1665 1666 # Write the mass spectra data to the hdf5 file 1667 super().to_hdf( 1668 overwrite=overwrite, 1669 export_raw=export_profile_spectra, 1670 compression=compression, 1671 profile_dtype=profile_dtype, 1672 ) 1673 1674 # Write scan info, ms_unprocessed, mass features, eics, and ms2_search results to the hdf5 file 1675 with h5py.File(self.output_file.with_suffix(".hdf5"), "a") as hdf_handle: 1676 # Add scan_info to hdf5 file 1677 if "scan_info" not in hdf_handle or overwrite: 1678 if "scan_info" in hdf_handle and overwrite: 1679 del hdf_handle["scan_info"] 1680 scan_info_group = hdf_handle.create_group("scan_info") 1681 for k, v in self.mass_spectra._scan_info.items(): 1682 array = np.array(list(v.values())) 1683 if array.dtype.str[0:2] == "<U": 1684 array = array.astype("S") 1685 scan_info_group.create_dataset( 1686 k, 1687 data=array, 1688 **hdf5_dataset_kwargs(array.shape, compression=compression), 1689 ) 1690 1691 # Add ms_unprocessed to hdf5 file 1692 export_unprocessed_ms1 = ( 1693 self.mass_spectra.parameters.lc_ms.export_unprocessed_ms1 1694 ) 1695 if self.mass_spectra._ms_unprocessed and export_unprocessed_ms1: 1696 if "ms_unprocessed" not in hdf_handle or overwrite: 1697 if "ms_unprocessed" in hdf_handle and overwrite: 1698 del hdf_handle["ms_unprocessed"] 1699 ms_unprocessed_group = hdf_handle.create_group("ms_unprocessed") 1700 else: 1701 ms_unprocessed_group = hdf_handle.get("ms_unprocessed") 1702 for k, v in self.mass_spectra._ms_unprocessed.items(): 1703 if str(k) not in ms_unprocessed_group or overwrite: 1704 if str(k) in ms_unprocessed_group and overwrite: 1705 del ms_unprocessed_group[str(k)] 1706 array = np.array(v) 1707 column_labels = ( 1708 list(v.columns) 1709 if hasattr(v, "columns") 1710 else ["scan", "mz", "intensity"] 1711 ) 1712 ms_dset = ms_unprocessed_group.create_dataset( 1713 str(k), 1714 data=array, 1715 **hdf5_dataset_kwargs(array.shape, compression=compression), 1716 ) 1717 ms_dset.attrs["column_labels"] = json.dumps(column_labels) 1718 1719 # Add LCMS mass features to hdf5 file 1720 self._save_mass_features_to_hdf5( 1721 hdf_handle, 1722 group_name="mass_features", 1723 overwrite=overwrite, 1724 compression=compression, 1725 ) 1726 self._save_mass_features_to_hdf5( 1727 hdf_handle, 1728 group_name="induced_mass_features", 1729 overwrite=overwrite, 1730 compression=compression, 1731 ) 1732 1733 # Add EIC data to hdf5 file 1734 export_eics = self.mass_spectra.parameters.lc_ms.export_eics 1735 if len(self.mass_spectra.eics) > 0 and export_eics: 1736 if "eics" not in hdf_handle or overwrite: 1737 if "eics" in hdf_handle and overwrite: 1738 del hdf_handle["eics"] 1739 eic_group = hdf_handle.create_group("eics") 1740 else: 1741 eic_group = hdf_handle.get("eics") 1742 1743 # Use the static helper method to save the EICs 1744 self._save_eics_dict_to_hdf5( 1745 self.mass_spectra.eics, 1746 eic_group, 1747 overwrite, 1748 compression=compression, 1749 ) 1750 1751 # Add ms2_search results to hdf5 file (parameterized) 1752 if len(self.mass_spectra.spectral_search_results) > 0: 1753 if "spectral_search_results" not in hdf_handle or overwrite: 1754 if "spectral_search_results" in hdf_handle and overwrite: 1755 del hdf_handle["spectral_search_results"] 1756 spectral_search_results = hdf_handle.create_group( 1757 "spectral_search_results" 1758 ) 1759 else: 1760 spectral_search_results = hdf_handle.get("spectral_search_results") 1761 # Create group for each search result by ms2_scan / precursor_mz 1762 for k, v in self.mass_spectra.spectral_search_results.items(): 1763 # TODO KRH: Fix to handle if export_only_relevant and k not in relevant_scan_numbers: continue! 1764 if str(k) not in spectral_search_results or overwrite: 1765 if str(k) in spectral_search_results and overwrite: 1766 del spectral_search_results[str(k)] 1767 spectral_search_results.create_group(str(k)) 1768 for k2, v2 in v.items(): 1769 spectral_search_results[str(k)].create_group(str(k2)) 1770 spectral_search_results[str(k)][str(k2)].attrs[ 1771 "precursor_mz" 1772 ] = v2.precursor_mz 1773 spectral_search_results[str(k)][str(k2)].attrs[ 1774 "query_spectrum_id" 1775 ] = v2.query_spectrum_id 1776 # Loop through each of the attributes and add them as datasets (if array) 1777 for k3, v3 in v2.__dict__.items(): 1778 if v3 is not None and k3 not in [ 1779 "query_spectrum", 1780 "precursor_mz", 1781 "query_spectrum_id", 1782 ]: 1783 if ( 1784 k3 == "query_frag_types" 1785 or k3 == "ref_frag_types" 1786 ): 1787 v3 = [", ".join(x) for x in v3] 1788 if all(v3 is not None for v3 in v3): 1789 array = np.array(v3) 1790 if array.dtype.str[0:2] == "<U": 1791 array = array.astype("S") 1792 spectral_search_results[str(k)][ 1793 str(k2) 1794 ].create_dataset( 1795 str(k3), 1796 data=array, 1797 **hdf5_dataset_kwargs( 1798 array.shape, compression=compression 1799 ), 1800 ) 1801 1802 # Save parameters as separate json 1803 if save_parameters: 1804 # Check if parameter_format is valid 1805 if parameter_format not in ["json", "toml"]: 1806 raise ValueError("parameter_format must be 'json' or 'toml'") 1807 1808 if parameter_format == "json": 1809 dump_lcms_settings_json( 1810 filename=self.output_file.with_suffix(".json"), 1811 lcms_obj=self.mass_spectra, 1812 ) 1813 elif parameter_format == "toml": 1814 dump_lcms_settings_toml( 1815 filename=self.output_file.with_suffix(".toml"), 1816 lcms_obj=self.mass_spectra, 1817 )
Export the data to an HDF5.
Parameters
- overwrite (bool, optional): Whether to overwrite the output file. Default is False.
- save_parameters (bool, optional): Whether to save the parameters as a separate json or toml file. Default is True.
- parameter_format (str, optional): The format to save the parameters in. Default is 'toml'.
- compression (int, str or None, optional):
Compression for the written datasets: a gzip level 0-9,
"lzf", orNone/"none". Default is 4."lzf"is h5py's own filter, not a standard HDF5 one -- it needs h5py (or a tool with the lzf plugin) to read the result back. - profile_dtype (str, optional):
dtype used for the stored profile m/z and abundance arrays
(
"f4"or"f8"). Default is"f4".
Raises
- ValueError: If parameter_format is not 'json' or 'toml'.
Inherited Members
1820class LCMSMetabolomicsExport(LCMSExport): 1821 """A class to export LCMS metabolite data. 1822 1823 This class provides methods to export LCMS metabolite data to various formats 1824 (CSV/Parquet reports, HDF5) and summarize the metabolite report. 1825 1826 Parameters 1827 ---------- 1828 out_file_path : str | Path 1829 The output file path, do not include the file extension. 1830 mass_spectra : object 1831 The high resolution mass spectra object. 1832 """ 1833 1834 def __init__(self, out_file_path, mass_spectra): 1835 super().__init__(out_file_path, mass_spectra) 1836 self.ion_type_dict = MolecularFormulaCalc.ion_type_dict 1837 1838 @staticmethod 1839 def get_ion_formula(neutral_formula, ion_type): 1840 """From a neutral formula and an ion type, return the formula of the ion. 1841 1842 Notes 1843 ----- 1844 Delegates to 1845 :meth:`corems.molecular_formula.calc.MolecularFormulaCalc.MolecularFormulaCalc.get_ion_formula`. 1846 1847 Parameters 1848 ---------- 1849 neutral_formula : str 1850 The neutral formula, this should be a string form from the MolecularFormula class 1851 (e.g. 'C2 H4 O2', isotopes OK), or simple string (e.g. 'C2H4O2', no isotope handling in this case). 1852 In the case of a simple string, the atoms are parsed based on the presence of capital letters, 1853 e.g. MgCl2 is parsed as 'Mg Cl2. 1854 ion_type : str 1855 The ion type, e.g. 'protonated', '[M+H]+', '[M+Na]+', etc. 1856 See ``MolecularFormulaCalc.ion_type_dict`` for the available ion types. 1857 1858 Returns 1859 ------- 1860 str 1861 The formula of the ion as a string (like 'C2 H4 O2'); or None if the neutral_formula is not a string. 1862 """ 1863 return MolecularFormulaCalc.get_ion_formula(neutral_formula, ion_type) 1864 1865 @staticmethod 1866 def precursor_mz_from_formula(neutral_formula, ion_type, charge=None): 1867 """Calculated precursor m/z from neutral formula and adduct / ion type. 1868 1869 Notes 1870 ----- 1871 Delegates to 1872 :meth:`corems.molecular_formula.calc.MolecularFormulaCalc.MolecularFormulaCalc.precursor_mz_from_formula`. 1873 """ 1874 return MolecularFormulaCalc.precursor_mz_from_formula( 1875 neutral_formula, ion_type, charge=charge 1876 ) 1877 1878 @staticmethod 1879 def get_isotope_type(ion_formula): 1880 """From an ion formula, return the 13C isotope type of the ion. 1881 1882 Notes 1883 ----- 1884 This is a static method. 1885 If the ion_formula is not a string, this method will return None. 1886 This is currently only functional for 13C isotopes. 1887 1888 Parameters 1889 ---------- 1890 ion_formula : str 1891 The formula of the ion, expected to be a string like 'C2 H4 O2'. 1892 1893 Returns 1894 ------- 1895 str 1896 The isotope type of the ion, e.g. '13C1', '13C2', etc; or None if the ion_formula does not contain a 13C isotope. 1897 1898 Raises 1899 ------ 1900 ValueError 1901 If the ion_formula is not a string. 1902 """ 1903 if not isinstance(ion_formula, str): 1904 return None 1905 1906 if re.search(r"\s", ion_formula): 1907 ion_formula = MolecularFormula(ion_formula, ion_charge=0) 1908 else: 1909 raise ValueError('ion_formula should be a string like "C2 H4 O2"') 1910 ion_formula_dict = ion_formula.to_dict().copy() 1911 1912 try: 1913 iso_class = "13C" + str(ion_formula_dict.pop("13C")) 1914 except KeyError: 1915 iso_class = None 1916 1917 return iso_class 1918 1919 def report_to_csv(self, molecular_metadata=None): 1920 """Create a report of the mass features and their annotations and save it as a CSV file. 1921 1922 Parameters 1923 ---------- 1924 molecular_metadata : dict, optional 1925 The molecular metadata. Default is None. 1926 """ 1927 report = self.to_report(molecular_metadata=molecular_metadata) 1928 out_file = self.output_file.with_suffix(".csv") 1929 report.to_csv(out_file, index=False) 1930 1931 def report_to_parquet(self, molecular_metadata=None): 1932 """Create a report of the mass features and their annotations and save it as a Parquet file. 1933 1934 Parameters 1935 ---------- 1936 molecular_metadata : dict, optional 1937 The molecular metadata. Default is None. 1938 1939 Notes 1940 ----- 1941 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 1942 Shared by metabolomics and lipidomics exporters via inheritance. 1943 """ 1944 report = self.to_report(molecular_metadata=molecular_metadata) 1945 out_file = self.output_file.with_suffix(".parquet") 1946 report.to_parquet(out_file, index=False) 1947 1948 def clean_ms1_report(self, ms1_summary_full): 1949 """Clean the MS1 report. 1950 1951 Parameters 1952 ---------- 1953 ms1_summary_full : DataFrame 1954 The full MS1 summary DataFrame. 1955 1956 Returns 1957 ------- 1958 DataFrame 1959 The cleaned MS1 summary DataFrame. 1960 """ 1961 ms1_summary_full = ms1_summary_full.reset_index() 1962 cols_to_keep = [ 1963 "mf_id", 1964 "Molecular Formula", 1965 "Ion Type", 1966 "Calculated m/z", 1967 "m/z Error (ppm)", 1968 "m/z Error Score", 1969 "Is Isotopologue", 1970 "Isotopologue Similarity", 1971 "Confidence Score", 1972 ] 1973 ms1_summary = ms1_summary_full[cols_to_keep].copy() 1974 ms1_summary["ion_formula"] = [ 1975 self.get_ion_formula(f, a) 1976 for f, a in zip(ms1_summary["Molecular Formula"], ms1_summary["Ion Type"]) 1977 ] 1978 ms1_summary["isotopologue_type"] = [ 1979 self.get_isotope_type(f) for f in ms1_summary["ion_formula"].tolist() 1980 ] 1981 1982 # Keep MS1 neutral Molecular Formula so it is not confused with library formulas 1983 # from MS2 spectral metadata. 1984 ms1_summary = ms1_summary[ 1985 [ 1986 "mf_id", 1987 "ion_formula", 1988 "isotopologue_type", 1989 "Molecular Formula", 1990 "Calculated m/z", 1991 "m/z Error (ppm)", 1992 "m/z Error Score", 1993 "Isotopologue Similarity", 1994 "Confidence Score", 1995 ] 1996 ] 1997 1998 # Set the index to mf_id 1999 ms1_summary = ms1_summary.set_index("mf_id") 2000 2001 return ms1_summary 2002 2003 def summarize_ms2_report(self, ms2_annot_report): 2004 """ 2005 Summarize the MS2 report. 2006 2007 Parameters 2008 ---------- 2009 ms2_annot_report : DataFrame 2010 The MS2 annotation DataFrame with all annotations, output of mass_features_ms2_annot_to_df. 2011 2012 Returns 2013 ------- 2014 """ 2015 2016 def summarize_metabolomics_report(self, ms2_annot_report): 2017 """Summarize the MS2 hits for a metabolomics report 2018 2019 Parameters 2020 ---------- 2021 ms2_annot : DataFrame 2022 The MS2 annotation DataFrame with all annotations. 2023 2024 Returns 2025 ------- 2026 DataFrame 2027 The summarized metabolomics report. 2028 """ 2029 columns_to_drop = [ 2030 "precursor_mz", 2031 "precursor_mz_error_ppm", 2032 "cas", 2033 "data_id", 2034 "iupac_name", 2035 "traditional_name", 2036 "common_name", 2037 "casno", 2038 ] 2039 ms2_annot = ms2_annot_report.drop( 2040 columns=[col for col in columns_to_drop if col in ms2_annot_report.columns] 2041 ) 2042 2043 # Prepare information about the search results, pulling out the best hit for the single report 2044 # Group by mf_id,ref_mol_id grab row with highest entropy similarity 2045 ms2_annot = ms2_annot.reset_index() 2046 # Add column called "n_spectra_contributing" that is the number of unique values in query_spectrum_id per mf_id,ref_mol_id 2047 ms2_annot["n_spectra_contributing"] = ms2_annot.groupby( 2048 ["mf_id", "ref_mol_id"] 2049 )["query_spectrum_id"].transform("nunique") 2050 # Sort by entropy similarity 2051 ms2_annot = ms2_annot.sort_values( 2052 by=["mf_id", "ref_mol_id", "entropy_similarity"], 2053 ascending=[True, True, False], 2054 ) 2055 best_entropy = ms2_annot.drop_duplicates( 2056 subset=["mf_id", "ref_mol_id"], keep="first" 2057 ) 2058 2059 return best_entropy 2060 2061 def clean_ms2_report(self, metabolite_summary): 2062 """Clean the MS2 report. 2063 2064 Parameters 2065 ---------- 2066 metabolite_summary : DataFrame 2067 The full metabolomics summary DataFrame. 2068 2069 Returns 2070 ------- 2071 DataFrame 2072 The cleaned metabolomics summary DataFrame. 2073 """ 2074 metabolite_summary = metabolite_summary.reset_index() 2075 # Library (MS2 metadata) formulas — keep separate from MS1 search fields 2076 metabolite_summary["library_ion_formula"] = [ 2077 self.get_ion_formula(f, a) 2078 for f, a in zip( 2079 metabolite_summary["formula"], metabolite_summary["ref_ion_type"] 2080 ) 2081 ] 2082 metabolite_summary = metabolite_summary.rename( 2083 columns={"formula": "library_formula"} 2084 ) 2085 2086 col_order = [ 2087 "mf_id", 2088 "library_ion_formula", 2089 "ref_ion_type", 2090 "library_formula", 2091 "inchikey", 2092 "name", 2093 "inchi", 2094 "chebi", 2095 "smiles", 2096 "kegg", 2097 "cas", 2098 "database_name", 2099 "ref_ms_id", 2100 "entropy_similarity", 2101 "ref_mz_in_query_fract", 2102 "n_spectra_contributing", 2103 ] 2104 2105 # Reorder columns 2106 metabolite_summary = metabolite_summary[ 2107 [col for col in col_order if col in metabolite_summary.columns] 2108 ] 2109 2110 # Convert chebi (if present) to int: 2111 if "chebi" in metabolite_summary.columns: 2112 metabolite_summary["chebi"] = metabolite_summary["chebi"].astype( 2113 "Int64", errors="ignore" 2114 ) 2115 2116 # Set the index to mf_id 2117 metabolite_summary = metabolite_summary.set_index("mf_id") 2118 2119 return metabolite_summary 2120 2121 def combine_reports(self, mf_report, ms1_annot_report, ms2_annot_report): 2122 """Combine the mass feature report with the MS1 and MS2 reports. 2123 2124 Parameters 2125 ---------- 2126 mf_report : DataFrame 2127 The mass feature report DataFrame. 2128 ms1_annot_report : DataFrame 2129 The MS1 annotation report DataFrame. 2130 ms2_annot_report : DataFrame 2131 The MS2 annotation report DataFrame. 2132 2133 Notes 2134 ----- 2135 ``Molecular Formula`` / ``Ion Formula`` come only from MS1 molecular 2136 formula search. MS2 spectral library formulas are exposed as 2137 ``Library Molecular Formula`` / ``Library Ion Formula``. 2138 When MS1 and MS2 match (same ion formula), the library formula columns 2139 are cleared so they do not repeat the MS1 values on that row. 2140 """ 2141 # If there is an ms1_annot_report, merge it with the mf_report 2142 if ms1_annot_report is not None and not ms1_annot_report.empty: 2143 # MS1 has been run and has molecular formula information 2144 mf_report = pd.merge( 2145 mf_report, 2146 ms1_annot_report, 2147 how="left", 2148 on=["mf_id", "isotopologue_type"], 2149 ) 2150 if ms2_annot_report is not None: 2151 # Prefer joining MS2 hits to matching MS1 ion formula when both exist. 2152 # MS2 uses library_ion_formula so MS1 Ion Formula is not overwritten. 2153 if ( 2154 "ion_formula" in mf_report.columns 2155 and "library_ion_formula" in ms2_annot_report.columns 2156 ): 2157 # pull out the records without ion_formula and merge on mf_id only 2158 mf_no_ion_formula = mf_report[mf_report["ion_formula"].isna()] 2159 mf_no_ion_formula = ( 2160 mf_no_ion_formula.drop(columns=["ion_formula"]) 2161 if "ion_formula" in mf_no_ion_formula.columns 2162 else mf_no_ion_formula 2163 ) 2164 mf_no_ion_formula = pd.merge( 2165 mf_no_ion_formula, ms2_annot_report, how="left", on=["mf_id"] 2166 ) 2167 2168 # pull out the records with ion_formula and merge on mf_id + formula match 2169 mf_with_ion_formula = mf_report[~mf_report["ion_formula"].isna()] 2170 mf_with_ion_formula = pd.merge( 2171 mf_with_ion_formula, 2172 ms2_annot_report, 2173 how="left", 2174 left_on=["mf_id", "ion_formula"], 2175 right_on=["mf_id", "library_ion_formula"], 2176 ) 2177 2178 # put back together 2179 mf_report = pd.concat([mf_no_ion_formula, mf_with_ion_formula]) 2180 else: 2181 # Fall back to merging on mf_id only (robust when ion_formula missing) 2182 mf_report = pd.merge( 2183 mf_report, ms2_annot_report, how="left", on=["mf_id"] 2184 ) 2185 2186 # Rename colums 2187 rename_dict = { 2188 "mf_id": "Mass Feature ID", 2189 "scan_time": "Retention Time (min)", 2190 "mz": "m/z", 2191 "apex_scan": "Apex Scan Number", 2192 "intensity": "Intensity", 2193 "persistence": "Persistence", 2194 "area": "Area", 2195 "half_height_width": "Half Height Width (min)", 2196 "tailing_factor": "Tailing Factor", 2197 "dispersity_index": "Dispersity Index", 2198 "ms2_spectrum": "MS2 Spectrum", 2199 "monoisotopic_mf_id": "Monoisotopic Mass Feature ID", 2200 "isotopologue_type": "Isotopologue Type", 2201 "mass_spectrum_deconvoluted_parent": "Is Largest Ion after Deconvolution", 2202 "associated_mass_features": "Associated Mass Features after Deconvolution", 2203 "ion_formula": "Ion Formula", 2204 "library_ion_formula": "Library Ion Formula", 2205 "library_formula": "Library Molecular Formula", 2206 "ref_ion_type": "Library Ion Type", 2207 "annot_level": "Lipid Annotation Level", 2208 "lipid_molecular_species_id": "Lipid Molecular Species", 2209 "lipid_summed_name": "Lipid Species", 2210 "lipid_subclass": "Lipid Subclass", 2211 "lipid_class": "Lipid Class", 2212 "lipid_category": "Lipid Category", 2213 "entropy_similarity": "Entropy Similarity", 2214 "ref_mz_in_query_fract": "Library mzs in Query (fraction)", 2215 "n_spectra_contributing": "Spectra with Annotation (n)", 2216 } 2217 mf_report = mf_report.rename(columns=rename_dict) 2218 2219 # When MS1 and MS2 co-annotate the same ion/molecular formula, library formula 2220 # columns only repeat MS1 fields — clear them so the row is not redundant. 2221 # Keep Library * columns populated for MS2-only (or mismatched) hits. 2222 if ( 2223 "Ion Formula" in mf_report.columns 2224 and "Library Ion Formula" in mf_report.columns 2225 ): 2226 ion_match = ( 2227 mf_report["Ion Formula"].notna() 2228 & mf_report["Library Ion Formula"].notna() 2229 & (mf_report["Ion Formula"] == mf_report["Library Ion Formula"]) 2230 ) 2231 mf_report.loc[ion_match, "Library Ion Formula"] = pd.NA 2232 if "Library Molecular Formula" in mf_report.columns: 2233 mf_report.loc[ion_match, "Library Molecular Formula"] = pd.NA 2234 elif ( 2235 "Molecular Formula" in mf_report.columns 2236 and "Library Molecular Formula" in mf_report.columns 2237 ): 2238 mol_match = ( 2239 mf_report["Molecular Formula"].notna() 2240 & mf_report["Library Molecular Formula"].notna() 2241 & ( 2242 mf_report["Molecular Formula"] 2243 == mf_report["Library Molecular Formula"] 2244 ) 2245 ) 2246 mf_report.loc[mol_match, "Library Molecular Formula"] = pd.NA 2247 if "Library Ion Formula" in mf_report.columns: 2248 mf_report.loc[mol_match, "Library Ion Formula"] = pd.NA 2249 2250 mf_report["Sample Name"] = self.mass_spectra.sample_name 2251 mf_report["Polarity"] = self.mass_spectra.polarity 2252 mf_report = mf_report[ 2253 ["Mass Feature ID", "Sample Name", "Polarity"] 2254 + [ 2255 col 2256 for col in mf_report.columns 2257 if col not in ["Mass Feature ID", "Sample Name", "Polarity"] 2258 ] 2259 ] 2260 2261 # Reorder rows by "Mass Feature ID", then "Entropy Similarity" (descending), then "Confidence Score" (descending) 2262 if ( 2263 "Entropy Similarity" in mf_report.columns 2264 and "Confidence Score" in mf_report.columns 2265 ): 2266 mf_report = mf_report.sort_values( 2267 by=["Mass Feature ID", "Entropy Similarity", "Confidence Score"], 2268 ascending=[True, False, False], 2269 ) 2270 elif "Entropy Similarity" in mf_report.columns: 2271 mf_report = mf_report.sort_values( 2272 by=["Mass Feature ID", "Entropy Similarity"], 2273 ascending=[True, False], 2274 ) 2275 elif "Confidence Score" in mf_report.columns: 2276 mf_report = mf_report.sort_values( 2277 by=["Mass Feature ID", "Confidence Score"], 2278 ascending=[True, False], 2279 ) 2280 # If neither "Entropy Similarity" nor "Confidence Score" are in the columns, just sort by "Mass Feature ID" 2281 else: 2282 mf_report = mf_report.sort_values("Mass Feature ID") 2283 2284 # Reset index 2285 mf_report = mf_report.reset_index(drop=True) 2286 2287 return mf_report 2288 2289 def to_report(self, molecular_metadata=None, suppress_warnings=False): 2290 """Create a report of the mass features and their annotations. 2291 2292 Parameters 2293 ---------- 2294 molecular_metadata : dict, optional 2295 The molecular metadata. Default is None. 2296 suppress_warnings : bool, optional 2297 If True, suppresses warnings from mass_features_ms2_annot_to_df. 2298 Default is False. 2299 2300 Returns 2301 ------- 2302 DataFrame 2303 The report as a Pandas DataFrame. 2304 """ 2305 # Get mass feature dataframe 2306 mf_report = self.mass_spectra.mass_features_to_df() 2307 mf_report = mf_report.reset_index(drop=False) 2308 2309 # Get and clean ms1 annotation dataframe 2310 ms1_annot_report = self.mass_spectra.mass_features_ms1_annot_to_df( 2311 suppress_warnings=suppress_warnings 2312 ) 2313 if ms1_annot_report is not None: 2314 ms1_annot_report = ms1_annot_report.copy() 2315 ms1_annot_report = self.clean_ms1_report(ms1_annot_report) 2316 ms1_annot_report = ms1_annot_report.reset_index(drop=False) 2317 else: 2318 ms1_annot_report = None 2319 2320 # Get, summarize, and clean ms2 annotation dataframe 2321 ms2_annot_report = self.mass_spectra.mass_features_ms2_annot_to_df( 2322 molecular_metadata=molecular_metadata, suppress_warnings=suppress_warnings 2323 ) 2324 if ms2_annot_report is not None and molecular_metadata is not None: 2325 ms2_annot_report = self.summarize_metabolomics_report(ms2_annot_report) 2326 ms2_annot_report = self.clean_ms2_report(ms2_annot_report) 2327 ms2_annot_report = ms2_annot_report.dropna(axis=1, how="all") 2328 ms2_annot_report = ms2_annot_report.reset_index(drop=False) 2329 else: 2330 ms2_annot_report = None 2331 2332 report = self.combine_reports( 2333 mf_report=mf_report, 2334 ms1_annot_report=ms1_annot_report, 2335 ms2_annot_report=ms2_annot_report, 2336 ) 2337 2338 return report
A class to export LCMS metabolite data.
This class provides methods to export LCMS metabolite data to various formats (CSV/Parquet reports, HDF5) and summarize the metabolite report.
Parameters
- out_file_path (str | Path): The output file path, do not include the file extension.
- mass_spectra (object): The high resolution mass spectra object.
LCMSMetabolomicsExport(out_file_path, mass_spectra)
1834 def __init__(self, out_file_path, mass_spectra): 1835 super().__init__(out_file_path, mass_spectra) 1836 self.ion_type_dict = MolecularFormulaCalc.ion_type_dict
This constructor should always be called with keyword arguments. Arguments are:
group should be None; reserved for future extension when a ThreadGroup class is implemented.
target is the callable object to be invoked by the run() method. Defaults to None, meaning nothing is called.
name is the thread name. By default, a unique name is constructed of the form "Thread-N" where N is a small decimal number.
args is a list or tuple of arguments for the target invocation. Defaults to ().
kwargs is a dictionary of keyword arguments for the target invocation. Defaults to {}.
If a subclass overrides the constructor, it must make sure to invoke the base class constructor (Thread.__init__()) before doing anything else to the thread.
@staticmethod
def
get_ion_formula(neutral_formula, ion_type):
1838 @staticmethod 1839 def get_ion_formula(neutral_formula, ion_type): 1840 """From a neutral formula and an ion type, return the formula of the ion. 1841 1842 Notes 1843 ----- 1844 Delegates to 1845 :meth:`corems.molecular_formula.calc.MolecularFormulaCalc.MolecularFormulaCalc.get_ion_formula`. 1846 1847 Parameters 1848 ---------- 1849 neutral_formula : str 1850 The neutral formula, this should be a string form from the MolecularFormula class 1851 (e.g. 'C2 H4 O2', isotopes OK), or simple string (e.g. 'C2H4O2', no isotope handling in this case). 1852 In the case of a simple string, the atoms are parsed based on the presence of capital letters, 1853 e.g. MgCl2 is parsed as 'Mg Cl2. 1854 ion_type : str 1855 The ion type, e.g. 'protonated', '[M+H]+', '[M+Na]+', etc. 1856 See ``MolecularFormulaCalc.ion_type_dict`` for the available ion types. 1857 1858 Returns 1859 ------- 1860 str 1861 The formula of the ion as a string (like 'C2 H4 O2'); or None if the neutral_formula is not a string. 1862 """ 1863 return MolecularFormulaCalc.get_ion_formula(neutral_formula, ion_type)
From a neutral formula and an ion type, return the formula of the ion.
Notes
Delegates to
corems.molecular_formula.calc.MolecularFormulaCalc.MolecularFormulaCalc.get_ion_formula().
Parameters
- neutral_formula (str): The neutral formula, this should be a string form from the MolecularFormula class (e.g. 'C2 H4 O2', isotopes OK), or simple string (e.g. 'C2H4O2', no isotope handling in this case). In the case of a simple string, the atoms are parsed based on the presence of capital letters, e.g. MgCl2 is parsed as 'Mg Cl2.
- ion_type (str):
The ion type, e.g. 'protonated', '[M+H]+', '[M+Na]+', etc.
See
MolecularFormulaCalc.ion_type_dictfor the available ion types.
Returns
- str: The formula of the ion as a string (like 'C2 H4 O2'); or None if the neutral_formula is not a string.
@staticmethod
def
precursor_mz_from_formula(neutral_formula, ion_type, charge=None):
1865 @staticmethod 1866 def precursor_mz_from_formula(neutral_formula, ion_type, charge=None): 1867 """Calculated precursor m/z from neutral formula and adduct / ion type. 1868 1869 Notes 1870 ----- 1871 Delegates to 1872 :meth:`corems.molecular_formula.calc.MolecularFormulaCalc.MolecularFormulaCalc.precursor_mz_from_formula`. 1873 """ 1874 return MolecularFormulaCalc.precursor_mz_from_formula( 1875 neutral_formula, ion_type, charge=charge 1876 )
Calculated precursor m/z from neutral formula and adduct / ion type.
Notes
Delegates to
corems.molecular_formula.calc.MolecularFormulaCalc.MolecularFormulaCalc.precursor_mz_from_formula().
@staticmethod
def
get_isotope_type(ion_formula):
1878 @staticmethod 1879 def get_isotope_type(ion_formula): 1880 """From an ion formula, return the 13C isotope type of the ion. 1881 1882 Notes 1883 ----- 1884 This is a static method. 1885 If the ion_formula is not a string, this method will return None. 1886 This is currently only functional for 13C isotopes. 1887 1888 Parameters 1889 ---------- 1890 ion_formula : str 1891 The formula of the ion, expected to be a string like 'C2 H4 O2'. 1892 1893 Returns 1894 ------- 1895 str 1896 The isotope type of the ion, e.g. '13C1', '13C2', etc; or None if the ion_formula does not contain a 13C isotope. 1897 1898 Raises 1899 ------ 1900 ValueError 1901 If the ion_formula is not a string. 1902 """ 1903 if not isinstance(ion_formula, str): 1904 return None 1905 1906 if re.search(r"\s", ion_formula): 1907 ion_formula = MolecularFormula(ion_formula, ion_charge=0) 1908 else: 1909 raise ValueError('ion_formula should be a string like "C2 H4 O2"') 1910 ion_formula_dict = ion_formula.to_dict().copy() 1911 1912 try: 1913 iso_class = "13C" + str(ion_formula_dict.pop("13C")) 1914 except KeyError: 1915 iso_class = None 1916 1917 return iso_class
From an ion formula, return the 13C isotope type of the ion.
Notes
This is a static method. If the ion_formula is not a string, this method will return None. This is currently only functional for 13C isotopes.
Parameters
- ion_formula (str): The formula of the ion, expected to be a string like 'C2 H4 O2'.
Returns
- str: The isotope type of the ion, e.g. '13C1', '13C2', etc; or None if the ion_formula does not contain a 13C isotope.
Raises
- ValueError: If the ion_formula is not a string.
def
report_to_csv(self, molecular_metadata=None):
1919 def report_to_csv(self, molecular_metadata=None): 1920 """Create a report of the mass features and their annotations and save it as a CSV file. 1921 1922 Parameters 1923 ---------- 1924 molecular_metadata : dict, optional 1925 The molecular metadata. Default is None. 1926 """ 1927 report = self.to_report(molecular_metadata=molecular_metadata) 1928 out_file = self.output_file.with_suffix(".csv") 1929 report.to_csv(out_file, index=False)
Create a report of the mass features and their annotations and save it as a CSV file.
Parameters
- molecular_metadata (dict, optional): The molecular metadata. Default is None.
def
report_to_parquet(self, molecular_metadata=None):
1931 def report_to_parquet(self, molecular_metadata=None): 1932 """Create a report of the mass features and their annotations and save it as a Parquet file. 1933 1934 Parameters 1935 ---------- 1936 molecular_metadata : dict, optional 1937 The molecular metadata. Default is None. 1938 1939 Notes 1940 ----- 1941 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 1942 Shared by metabolomics and lipidomics exporters via inheritance. 1943 """ 1944 report = self.to_report(molecular_metadata=molecular_metadata) 1945 out_file = self.output_file.with_suffix(".parquet") 1946 report.to_parquet(out_file, index=False)
Create a report of the mass features and their annotations and save it as a Parquet file.
Parameters
- molecular_metadata (dict, optional): The molecular metadata. Default is None.
Notes
Requires pyarrow (or fastparquet) for pandas Parquet I/O.
Shared by metabolomics and lipidomics exporters via inheritance.
def
clean_ms1_report(self, ms1_summary_full):
1948 def clean_ms1_report(self, ms1_summary_full): 1949 """Clean the MS1 report. 1950 1951 Parameters 1952 ---------- 1953 ms1_summary_full : DataFrame 1954 The full MS1 summary DataFrame. 1955 1956 Returns 1957 ------- 1958 DataFrame 1959 The cleaned MS1 summary DataFrame. 1960 """ 1961 ms1_summary_full = ms1_summary_full.reset_index() 1962 cols_to_keep = [ 1963 "mf_id", 1964 "Molecular Formula", 1965 "Ion Type", 1966 "Calculated m/z", 1967 "m/z Error (ppm)", 1968 "m/z Error Score", 1969 "Is Isotopologue", 1970 "Isotopologue Similarity", 1971 "Confidence Score", 1972 ] 1973 ms1_summary = ms1_summary_full[cols_to_keep].copy() 1974 ms1_summary["ion_formula"] = [ 1975 self.get_ion_formula(f, a) 1976 for f, a in zip(ms1_summary["Molecular Formula"], ms1_summary["Ion Type"]) 1977 ] 1978 ms1_summary["isotopologue_type"] = [ 1979 self.get_isotope_type(f) for f in ms1_summary["ion_formula"].tolist() 1980 ] 1981 1982 # Keep MS1 neutral Molecular Formula so it is not confused with library formulas 1983 # from MS2 spectral metadata. 1984 ms1_summary = ms1_summary[ 1985 [ 1986 "mf_id", 1987 "ion_formula", 1988 "isotopologue_type", 1989 "Molecular Formula", 1990 "Calculated m/z", 1991 "m/z Error (ppm)", 1992 "m/z Error Score", 1993 "Isotopologue Similarity", 1994 "Confidence Score", 1995 ] 1996 ] 1997 1998 # Set the index to mf_id 1999 ms1_summary = ms1_summary.set_index("mf_id") 2000 2001 return ms1_summary
Clean the MS1 report.
Parameters
- ms1_summary_full (DataFrame): The full MS1 summary DataFrame.
Returns
- DataFrame: The cleaned MS1 summary DataFrame.
def
summarize_ms2_report(self, ms2_annot_report):
2003 def summarize_ms2_report(self, ms2_annot_report): 2004 """ 2005 Summarize the MS2 report. 2006 2007 Parameters 2008 ---------- 2009 ms2_annot_report : DataFrame 2010 The MS2 annotation DataFrame with all annotations, output of mass_features_ms2_annot_to_df. 2011 2012 Returns 2013 ------- 2014 """
Summarize the MS2 report.
Parameters
- ms2_annot_report (DataFrame): The MS2 annotation DataFrame with all annotations, output of mass_features_ms2_annot_to_df.
- Returns
- -------
def
summarize_metabolomics_report(self, ms2_annot_report):
2016 def summarize_metabolomics_report(self, ms2_annot_report): 2017 """Summarize the MS2 hits for a metabolomics report 2018 2019 Parameters 2020 ---------- 2021 ms2_annot : DataFrame 2022 The MS2 annotation DataFrame with all annotations. 2023 2024 Returns 2025 ------- 2026 DataFrame 2027 The summarized metabolomics report. 2028 """ 2029 columns_to_drop = [ 2030 "precursor_mz", 2031 "precursor_mz_error_ppm", 2032 "cas", 2033 "data_id", 2034 "iupac_name", 2035 "traditional_name", 2036 "common_name", 2037 "casno", 2038 ] 2039 ms2_annot = ms2_annot_report.drop( 2040 columns=[col for col in columns_to_drop if col in ms2_annot_report.columns] 2041 ) 2042 2043 # Prepare information about the search results, pulling out the best hit for the single report 2044 # Group by mf_id,ref_mol_id grab row with highest entropy similarity 2045 ms2_annot = ms2_annot.reset_index() 2046 # Add column called "n_spectra_contributing" that is the number of unique values in query_spectrum_id per mf_id,ref_mol_id 2047 ms2_annot["n_spectra_contributing"] = ms2_annot.groupby( 2048 ["mf_id", "ref_mol_id"] 2049 )["query_spectrum_id"].transform("nunique") 2050 # Sort by entropy similarity 2051 ms2_annot = ms2_annot.sort_values( 2052 by=["mf_id", "ref_mol_id", "entropy_similarity"], 2053 ascending=[True, True, False], 2054 ) 2055 best_entropy = ms2_annot.drop_duplicates( 2056 subset=["mf_id", "ref_mol_id"], keep="first" 2057 ) 2058 2059 return best_entropy
Summarize the MS2 hits for a metabolomics report
Parameters
- ms2_annot (DataFrame): The MS2 annotation DataFrame with all annotations.
Returns
- DataFrame: The summarized metabolomics report.
def
clean_ms2_report(self, metabolite_summary):
2061 def clean_ms2_report(self, metabolite_summary): 2062 """Clean the MS2 report. 2063 2064 Parameters 2065 ---------- 2066 metabolite_summary : DataFrame 2067 The full metabolomics summary DataFrame. 2068 2069 Returns 2070 ------- 2071 DataFrame 2072 The cleaned metabolomics summary DataFrame. 2073 """ 2074 metabolite_summary = metabolite_summary.reset_index() 2075 # Library (MS2 metadata) formulas — keep separate from MS1 search fields 2076 metabolite_summary["library_ion_formula"] = [ 2077 self.get_ion_formula(f, a) 2078 for f, a in zip( 2079 metabolite_summary["formula"], metabolite_summary["ref_ion_type"] 2080 ) 2081 ] 2082 metabolite_summary = metabolite_summary.rename( 2083 columns={"formula": "library_formula"} 2084 ) 2085 2086 col_order = [ 2087 "mf_id", 2088 "library_ion_formula", 2089 "ref_ion_type", 2090 "library_formula", 2091 "inchikey", 2092 "name", 2093 "inchi", 2094 "chebi", 2095 "smiles", 2096 "kegg", 2097 "cas", 2098 "database_name", 2099 "ref_ms_id", 2100 "entropy_similarity", 2101 "ref_mz_in_query_fract", 2102 "n_spectra_contributing", 2103 ] 2104 2105 # Reorder columns 2106 metabolite_summary = metabolite_summary[ 2107 [col for col in col_order if col in metabolite_summary.columns] 2108 ] 2109 2110 # Convert chebi (if present) to int: 2111 if "chebi" in metabolite_summary.columns: 2112 metabolite_summary["chebi"] = metabolite_summary["chebi"].astype( 2113 "Int64", errors="ignore" 2114 ) 2115 2116 # Set the index to mf_id 2117 metabolite_summary = metabolite_summary.set_index("mf_id") 2118 2119 return metabolite_summary
Clean the MS2 report.
Parameters
- metabolite_summary (DataFrame): The full metabolomics summary DataFrame.
Returns
- DataFrame: The cleaned metabolomics summary DataFrame.
def
combine_reports(self, mf_report, ms1_annot_report, ms2_annot_report):
2121 def combine_reports(self, mf_report, ms1_annot_report, ms2_annot_report): 2122 """Combine the mass feature report with the MS1 and MS2 reports. 2123 2124 Parameters 2125 ---------- 2126 mf_report : DataFrame 2127 The mass feature report DataFrame. 2128 ms1_annot_report : DataFrame 2129 The MS1 annotation report DataFrame. 2130 ms2_annot_report : DataFrame 2131 The MS2 annotation report DataFrame. 2132 2133 Notes 2134 ----- 2135 ``Molecular Formula`` / ``Ion Formula`` come only from MS1 molecular 2136 formula search. MS2 spectral library formulas are exposed as 2137 ``Library Molecular Formula`` / ``Library Ion Formula``. 2138 When MS1 and MS2 match (same ion formula), the library formula columns 2139 are cleared so they do not repeat the MS1 values on that row. 2140 """ 2141 # If there is an ms1_annot_report, merge it with the mf_report 2142 if ms1_annot_report is not None and not ms1_annot_report.empty: 2143 # MS1 has been run and has molecular formula information 2144 mf_report = pd.merge( 2145 mf_report, 2146 ms1_annot_report, 2147 how="left", 2148 on=["mf_id", "isotopologue_type"], 2149 ) 2150 if ms2_annot_report is not None: 2151 # Prefer joining MS2 hits to matching MS1 ion formula when both exist. 2152 # MS2 uses library_ion_formula so MS1 Ion Formula is not overwritten. 2153 if ( 2154 "ion_formula" in mf_report.columns 2155 and "library_ion_formula" in ms2_annot_report.columns 2156 ): 2157 # pull out the records without ion_formula and merge on mf_id only 2158 mf_no_ion_formula = mf_report[mf_report["ion_formula"].isna()] 2159 mf_no_ion_formula = ( 2160 mf_no_ion_formula.drop(columns=["ion_formula"]) 2161 if "ion_formula" in mf_no_ion_formula.columns 2162 else mf_no_ion_formula 2163 ) 2164 mf_no_ion_formula = pd.merge( 2165 mf_no_ion_formula, ms2_annot_report, how="left", on=["mf_id"] 2166 ) 2167 2168 # pull out the records with ion_formula and merge on mf_id + formula match 2169 mf_with_ion_formula = mf_report[~mf_report["ion_formula"].isna()] 2170 mf_with_ion_formula = pd.merge( 2171 mf_with_ion_formula, 2172 ms2_annot_report, 2173 how="left", 2174 left_on=["mf_id", "ion_formula"], 2175 right_on=["mf_id", "library_ion_formula"], 2176 ) 2177 2178 # put back together 2179 mf_report = pd.concat([mf_no_ion_formula, mf_with_ion_formula]) 2180 else: 2181 # Fall back to merging on mf_id only (robust when ion_formula missing) 2182 mf_report = pd.merge( 2183 mf_report, ms2_annot_report, how="left", on=["mf_id"] 2184 ) 2185 2186 # Rename colums 2187 rename_dict = { 2188 "mf_id": "Mass Feature ID", 2189 "scan_time": "Retention Time (min)", 2190 "mz": "m/z", 2191 "apex_scan": "Apex Scan Number", 2192 "intensity": "Intensity", 2193 "persistence": "Persistence", 2194 "area": "Area", 2195 "half_height_width": "Half Height Width (min)", 2196 "tailing_factor": "Tailing Factor", 2197 "dispersity_index": "Dispersity Index", 2198 "ms2_spectrum": "MS2 Spectrum", 2199 "monoisotopic_mf_id": "Monoisotopic Mass Feature ID", 2200 "isotopologue_type": "Isotopologue Type", 2201 "mass_spectrum_deconvoluted_parent": "Is Largest Ion after Deconvolution", 2202 "associated_mass_features": "Associated Mass Features after Deconvolution", 2203 "ion_formula": "Ion Formula", 2204 "library_ion_formula": "Library Ion Formula", 2205 "library_formula": "Library Molecular Formula", 2206 "ref_ion_type": "Library Ion Type", 2207 "annot_level": "Lipid Annotation Level", 2208 "lipid_molecular_species_id": "Lipid Molecular Species", 2209 "lipid_summed_name": "Lipid Species", 2210 "lipid_subclass": "Lipid Subclass", 2211 "lipid_class": "Lipid Class", 2212 "lipid_category": "Lipid Category", 2213 "entropy_similarity": "Entropy Similarity", 2214 "ref_mz_in_query_fract": "Library mzs in Query (fraction)", 2215 "n_spectra_contributing": "Spectra with Annotation (n)", 2216 } 2217 mf_report = mf_report.rename(columns=rename_dict) 2218 2219 # When MS1 and MS2 co-annotate the same ion/molecular formula, library formula 2220 # columns only repeat MS1 fields — clear them so the row is not redundant. 2221 # Keep Library * columns populated for MS2-only (or mismatched) hits. 2222 if ( 2223 "Ion Formula" in mf_report.columns 2224 and "Library Ion Formula" in mf_report.columns 2225 ): 2226 ion_match = ( 2227 mf_report["Ion Formula"].notna() 2228 & mf_report["Library Ion Formula"].notna() 2229 & (mf_report["Ion Formula"] == mf_report["Library Ion Formula"]) 2230 ) 2231 mf_report.loc[ion_match, "Library Ion Formula"] = pd.NA 2232 if "Library Molecular Formula" in mf_report.columns: 2233 mf_report.loc[ion_match, "Library Molecular Formula"] = pd.NA 2234 elif ( 2235 "Molecular Formula" in mf_report.columns 2236 and "Library Molecular Formula" in mf_report.columns 2237 ): 2238 mol_match = ( 2239 mf_report["Molecular Formula"].notna() 2240 & mf_report["Library Molecular Formula"].notna() 2241 & ( 2242 mf_report["Molecular Formula"] 2243 == mf_report["Library Molecular Formula"] 2244 ) 2245 ) 2246 mf_report.loc[mol_match, "Library Molecular Formula"] = pd.NA 2247 if "Library Ion Formula" in mf_report.columns: 2248 mf_report.loc[mol_match, "Library Ion Formula"] = pd.NA 2249 2250 mf_report["Sample Name"] = self.mass_spectra.sample_name 2251 mf_report["Polarity"] = self.mass_spectra.polarity 2252 mf_report = mf_report[ 2253 ["Mass Feature ID", "Sample Name", "Polarity"] 2254 + [ 2255 col 2256 for col in mf_report.columns 2257 if col not in ["Mass Feature ID", "Sample Name", "Polarity"] 2258 ] 2259 ] 2260 2261 # Reorder rows by "Mass Feature ID", then "Entropy Similarity" (descending), then "Confidence Score" (descending) 2262 if ( 2263 "Entropy Similarity" in mf_report.columns 2264 and "Confidence Score" in mf_report.columns 2265 ): 2266 mf_report = mf_report.sort_values( 2267 by=["Mass Feature ID", "Entropy Similarity", "Confidence Score"], 2268 ascending=[True, False, False], 2269 ) 2270 elif "Entropy Similarity" in mf_report.columns: 2271 mf_report = mf_report.sort_values( 2272 by=["Mass Feature ID", "Entropy Similarity"], 2273 ascending=[True, False], 2274 ) 2275 elif "Confidence Score" in mf_report.columns: 2276 mf_report = mf_report.sort_values( 2277 by=["Mass Feature ID", "Confidence Score"], 2278 ascending=[True, False], 2279 ) 2280 # If neither "Entropy Similarity" nor "Confidence Score" are in the columns, just sort by "Mass Feature ID" 2281 else: 2282 mf_report = mf_report.sort_values("Mass Feature ID") 2283 2284 # Reset index 2285 mf_report = mf_report.reset_index(drop=True) 2286 2287 return mf_report
Combine the mass feature report with the MS1 and MS2 reports.
Parameters
- mf_report (DataFrame): The mass feature report DataFrame.
- ms1_annot_report (DataFrame): The MS1 annotation report DataFrame.
- ms2_annot_report (DataFrame): The MS2 annotation report DataFrame.
Notes
Molecular Formula / Ion Formula come only from MS1 molecular
formula search. MS2 spectral library formulas are exposed as
Library Molecular Formula / Library Ion Formula.
When MS1 and MS2 match (same ion formula), the library formula columns
are cleared so they do not repeat the MS1 values on that row.
def
to_report(self, molecular_metadata=None, suppress_warnings=False):
2289 def to_report(self, molecular_metadata=None, suppress_warnings=False): 2290 """Create a report of the mass features and their annotations. 2291 2292 Parameters 2293 ---------- 2294 molecular_metadata : dict, optional 2295 The molecular metadata. Default is None. 2296 suppress_warnings : bool, optional 2297 If True, suppresses warnings from mass_features_ms2_annot_to_df. 2298 Default is False. 2299 2300 Returns 2301 ------- 2302 DataFrame 2303 The report as a Pandas DataFrame. 2304 """ 2305 # Get mass feature dataframe 2306 mf_report = self.mass_spectra.mass_features_to_df() 2307 mf_report = mf_report.reset_index(drop=False) 2308 2309 # Get and clean ms1 annotation dataframe 2310 ms1_annot_report = self.mass_spectra.mass_features_ms1_annot_to_df( 2311 suppress_warnings=suppress_warnings 2312 ) 2313 if ms1_annot_report is not None: 2314 ms1_annot_report = ms1_annot_report.copy() 2315 ms1_annot_report = self.clean_ms1_report(ms1_annot_report) 2316 ms1_annot_report = ms1_annot_report.reset_index(drop=False) 2317 else: 2318 ms1_annot_report = None 2319 2320 # Get, summarize, and clean ms2 annotation dataframe 2321 ms2_annot_report = self.mass_spectra.mass_features_ms2_annot_to_df( 2322 molecular_metadata=molecular_metadata, suppress_warnings=suppress_warnings 2323 ) 2324 if ms2_annot_report is not None and molecular_metadata is not None: 2325 ms2_annot_report = self.summarize_metabolomics_report(ms2_annot_report) 2326 ms2_annot_report = self.clean_ms2_report(ms2_annot_report) 2327 ms2_annot_report = ms2_annot_report.dropna(axis=1, how="all") 2328 ms2_annot_report = ms2_annot_report.reset_index(drop=False) 2329 else: 2330 ms2_annot_report = None 2331 2332 report = self.combine_reports( 2333 mf_report=mf_report, 2334 ms1_annot_report=ms1_annot_report, 2335 ms2_annot_report=ms2_annot_report, 2336 ) 2337 2338 return report
Create a report of the mass features and their annotations.
Parameters
- molecular_metadata (dict, optional): The molecular metadata. Default is None.
- suppress_warnings (bool, optional): If True, suppresses warnings from mass_features_ms2_annot_to_df. Default is False.
Returns
- DataFrame: The report as a Pandas DataFrame.
Inherited Members
2341class LipidomicsExport(LCMSMetabolomicsExport): 2342 """A class to export lipidomics data. 2343 2344 This class provides methods to export lipidomics data to various formats 2345 (CSV/Parquet reports, HDF5) and summarize the lipid report. 2346 2347 Parameters 2348 ---------- 2349 out_file_path : str | Path 2350 The output file path, do not include the file extension. 2351 mass_spectra : object 2352 The high resolution mass spectra object. 2353 """ 2354 2355 def __init__(self, out_file_path, mass_spectra): 2356 super().__init__(out_file_path, mass_spectra) 2357 2358 def summarize_lipid_report(self, ms2_annot): 2359 """Summarize the lipid report. 2360 2361 Parameters 2362 ---------- 2363 ms2_annot : DataFrame 2364 The MS2 annotation DataFrame with all annotations. 2365 2366 Returns 2367 ------- 2368 DataFrame 2369 The summarized lipid report. 2370 """ 2371 # Drop unnecessary columns for easier viewing 2372 columns_to_drop = [ 2373 "precursor_mz", 2374 "precursor_mz_error_ppm", 2375 "ref_mol_id", 2376 "ref_precursor_mz", 2377 "cas", 2378 "inchikey", 2379 "inchi", 2380 "chebi", 2381 "smiles", 2382 "kegg", 2383 "data_id", 2384 "iupac_name", 2385 "traditional_name", 2386 "common_name", 2387 "casno", 2388 ] 2389 ms2_annot = ms2_annot.drop( 2390 columns=[col for col in columns_to_drop if col in ms2_annot.columns] 2391 ) 2392 2393 # If ion_types_excluded is not empty, remove those ion types 2394 ion_types_excluded = self.mass_spectra.parameters.mass_spectrum[ 2395 "ms2" 2396 ].molecular_search.ion_types_excluded 2397 if len(ion_types_excluded) > 0: 2398 ms2_annot = ms2_annot[~ms2_annot["ref_ion_type"].isin(ion_types_excluded)] 2399 2400 # If mf_id is not present, check that the index name is mf_id and reset the index 2401 if "mf_id" not in ms2_annot.columns: 2402 if ms2_annot.index.name == "mf_id": 2403 ms2_annot = ms2_annot.reset_index() 2404 else: 2405 raise ValueError("mf_id is not present in the dataframe") 2406 2407 # Attempt to get consensus annotations to the MLF level 2408 mlf_results_all = [] 2409 for mf_id in ms2_annot["mf_id"].unique(): 2410 mlf_results_perid = [] 2411 ms2_annot_mf = ms2_annot[ms2_annot["mf_id"] == mf_id].copy() 2412 ms2_annot_mf["n_spectra_contributing"] = ( 2413 ms2_annot_mf.query_spectrum_id.nunique() 2414 ) 2415 2416 for query_scan in ms2_annot["query_spectrum_id"].unique(): 2417 ms2_annot_sub = ms2_annot_mf[ 2418 ms2_annot_mf["query_spectrum_id"] == query_scan 2419 ].copy() 2420 2421 if ms2_annot_sub["lipid_summed_name"].nunique() == 1: 2422 # If there is only one lipid_summed_name, let's try to get consensus molecular species annotation 2423 if ms2_annot_sub["lipid_summed_name"].nunique() == 1: 2424 ms2_annot_sub["entropy_max"] = ( 2425 ms2_annot_sub["entropy_similarity"] 2426 == ms2_annot_sub["entropy_similarity"].max() 2427 ) 2428 ms2_annot_sub["ref_match_fract_max"] = ( 2429 ms2_annot_sub["ref_mz_in_query_fract"] 2430 == ms2_annot_sub["ref_mz_in_query_fract"].max() 2431 ) 2432 ms2_annot_sub["frag_max"] = ms2_annot_sub[ 2433 "query_frag_types" 2434 ].apply(lambda x: True if "MLF" in x else False) 2435 2436 # New column that looks if there is a consensus between the ranks (one row that is highest in all ranks) 2437 ms2_annot_sub["consensus"] = ms2_annot_sub[ 2438 ["entropy_max", "ref_match_fract_max", "frag_max"] 2439 ].all(axis=1) 2440 2441 # If there is a consensus, take the row with the highest entropy_similarity 2442 if ms2_annot_sub["consensus"].any(): 2443 ms2_annot_sub = ms2_annot_sub[ 2444 ms2_annot_sub["entropy_similarity"] 2445 == ms2_annot_sub["entropy_similarity"].max() 2446 ].head(1) 2447 mlf_results_perid.append(ms2_annot_sub) 2448 if len(mlf_results_perid) == 0: 2449 mlf_results_perid = pd.DataFrame() 2450 else: 2451 mlf_results_perid = pd.concat(mlf_results_perid) 2452 if mlf_results_perid["name"].nunique() == 1: 2453 mlf_results_perid = mlf_results_perid[ 2454 mlf_results_perid["entropy_similarity"] 2455 == mlf_results_perid["entropy_similarity"].max() 2456 ].head(1) 2457 else: 2458 mlf_results_perid = pd.DataFrame() 2459 mlf_results_all.append(mlf_results_perid) 2460 2461 # These are the consensus annotations to the MLF level 2462 if len(mlf_results_all) > 0: 2463 mlf_results_all = pd.concat(mlf_results_all) 2464 mlf_results_all["annot_level"] = mlf_results_all["structure_level"] 2465 else: 2466 # Make an empty dataframe 2467 mlf_results_all = ms2_annot.head(0) 2468 2469 # For remaining mf_ids, try to get a consensus annotation to the species level 2470 species_results_all = [] 2471 # Remove mf_ids that have consensus annotations to the MLF level 2472 ms2_annot_spec = ms2_annot[ 2473 ~ms2_annot["mf_id"].isin(mlf_results_all["mf_id"].unique()) 2474 ] 2475 for mf_id in ms2_annot_spec["mf_id"].unique(): 2476 # Do all the hits have the same lipid_summed_name? 2477 ms2_annot_sub = ms2_annot_spec[ms2_annot_spec["mf_id"] == mf_id].copy() 2478 ms2_annot_sub["n_spectra_contributing"] = len(ms2_annot_sub) 2479 2480 if ms2_annot_sub["lipid_summed_name"].nunique() == 1: 2481 # Grab the highest entropy_similarity result 2482 ms2_annot_sub = ms2_annot_sub[ 2483 ms2_annot_sub["entropy_similarity"] 2484 == ms2_annot_sub["entropy_similarity"].max() 2485 ].head(1) 2486 species_results_all.append(ms2_annot_sub) 2487 2488 # These are the consensus annotations to the species level 2489 if len(species_results_all) > 0: 2490 species_results_all = pd.concat(species_results_all) 2491 species_results_all["annot_level"] = "species" 2492 else: 2493 # Make an empty dataframe 2494 species_results_all = ms2_annot.head(0) 2495 2496 # Deal with the remaining mf_ids that do not have consensus annotations to the species level or MLF level 2497 # Remove mf_ids that have consensus annotations to the species level 2498 ms2_annot_remaining = ms2_annot_spec[ 2499 ~ms2_annot_spec["mf_id"].isin(species_results_all["mf_id"].unique()) 2500 ] 2501 no_consensus = [] 2502 for mf_id in ms2_annot_remaining["mf_id"].unique(): 2503 id_sub = [] 2504 id_no_con = [] 2505 ms2_annot_sub_mf = ms2_annot_remaining[ 2506 ms2_annot_remaining["mf_id"] == mf_id 2507 ].copy() 2508 for query_scan in ms2_annot_sub_mf["query_spectrum_id"].unique(): 2509 ms2_annot_sub = ms2_annot_sub_mf[ 2510 ms2_annot_sub_mf["query_spectrum_id"] == query_scan 2511 ].copy() 2512 2513 # New columns for ranking [HIGHER RANK = BETTER] 2514 ms2_annot_sub["entropy_max"] = ( 2515 ms2_annot_sub["entropy_similarity"] 2516 == ms2_annot_sub["entropy_similarity"].max() 2517 ) 2518 ms2_annot_sub["ref_match_fract_max"] = ( 2519 ms2_annot_sub["ref_mz_in_query_fract"] 2520 == ms2_annot_sub["ref_mz_in_query_fract"].max() 2521 ) 2522 ms2_annot_sub["frag_max"] = ms2_annot_sub["query_frag_types"].apply( 2523 lambda x: True if "MLF" in x else False 2524 ) 2525 2526 # New column that looks if there is a consensus between the ranks (one row that is highest in all ranks) 2527 ms2_annot_sub["consensus"] = ms2_annot_sub[ 2528 ["entropy_max", "ref_match_fract_max", "frag_max"] 2529 ].all(axis=1) 2530 ms2_annot_sub_con = ms2_annot_sub[ms2_annot_sub["consensus"]] 2531 id_sub.append(ms2_annot_sub_con) 2532 id_no_con.append(ms2_annot_sub) 2533 id_sub = pd.concat(id_sub) 2534 id_no_con = pd.concat(id_no_con) 2535 2536 # Scenario 1: Multiple scans are being resolved to different MLFs [could be coelutions and should both be kept and annotated to MS level] 2537 if ( 2538 id_sub["query_frag_types"] 2539 .apply(lambda x: True if "MLF" in x else False) 2540 .all() 2541 and len(id_sub) > 0 2542 ): 2543 idx = id_sub.groupby("name")["entropy_similarity"].idxmax() 2544 id_sub = id_sub.loc[idx] 2545 # Reorder so highest entropy_similarity is first 2546 id_sub = id_sub.sort_values("entropy_similarity", ascending=False) 2547 id_sub["annot_level"] = id_sub["structure_level"] 2548 no_consensus.append(id_sub) 2549 2550 # Scenario 2: Multiple scans are being resolved to different species, keep both and annotate to appropriate level 2551 elif len(id_sub) == 0: 2552 for lipid_summed_name in id_no_con["lipid_summed_name"].unique(): 2553 summed_sub = id_no_con[ 2554 id_no_con["lipid_summed_name"] == lipid_summed_name 2555 ] 2556 # Any consensus to MLF? 2557 if summed_sub["consensus"].any(): 2558 summed_sub = summed_sub[summed_sub["consensus"]] 2559 summed_sub["annot_level"] = summed_sub["structure_level"] 2560 no_consensus.append(summed_sub) 2561 else: 2562 # Grab the highest entropy_similarity, if there are multiple, grab the first one 2563 summed_sub = summed_sub[ 2564 summed_sub["entropy_similarity"] 2565 == summed_sub["entropy_similarity"].max() 2566 ].head(1) 2567 # get first row 2568 summed_sub["annot_level"] = "species" 2569 summed_sub["name"] = "" 2570 no_consensus.append(summed_sub) 2571 else: 2572 raise ValueError("Unexpected scenario for summarizing mf_id: ", mf_id) 2573 2574 if len(no_consensus) > 0: 2575 no_consensus = pd.concat(no_consensus) 2576 else: 2577 no_consensus = ms2_annot.head(0) 2578 2579 # Combine all the consensus annotations and reformat the dataframe for output 2580 species_results_all = species_results_all.drop(columns=["name"]) 2581 species_results_all["lipid_molecular_species_id"] = "" 2582 mlf_results_all["lipid_molecular_species_id"] = mlf_results_all["name"] 2583 no_consensus["lipid_molecular_species_id"] = no_consensus["name"] 2584 consensus_annotations = pd.concat( 2585 [mlf_results_all, species_results_all, no_consensus] 2586 ) 2587 consensus_annotations = consensus_annotations.sort_values( 2588 "mf_id", ascending=True 2589 ) 2590 cols_to_keep = [ 2591 "mf_id", 2592 "ref_ion_type", 2593 "entropy_similarity", 2594 "ref_mz_in_query_fract", 2595 "lipid_molecular_species_id", 2596 "lipid_summed_name", 2597 "lipid_subclass", 2598 "lipid_class", 2599 "lipid_category", 2600 "formula", 2601 "annot_level", 2602 "n_spectra_contributing", 2603 ] 2604 consensus_annotations = consensus_annotations[cols_to_keep] 2605 consensus_annotations = consensus_annotations.set_index("mf_id") 2606 2607 return consensus_annotations 2608 2609 def clean_ms2_report(self, lipid_summary): 2610 """Clean the MS2 report. 2611 2612 Parameters 2613 ---------- 2614 lipid_summary : DataFrame 2615 The full lipid summary DataFrame. 2616 2617 Returns 2618 ------- 2619 DataFrame 2620 The cleaned lipid summary DataFrame. 2621 """ 2622 lipid_summary = lipid_summary.reset_index() 2623 # Library (MS2 metadata) formulas — keep separate from MS1 search fields 2624 lipid_summary["library_ion_formula"] = [ 2625 self.get_ion_formula(f, a) 2626 for f, a in zip(lipid_summary["formula"], lipid_summary["ref_ion_type"]) 2627 ] 2628 lipid_summary = lipid_summary.rename(columns={"formula": "library_formula"}) 2629 2630 # Reorder columns 2631 lipid_summary = lipid_summary[ 2632 [ 2633 "mf_id", 2634 "library_ion_formula", 2635 "ref_ion_type", 2636 "library_formula", 2637 "annot_level", 2638 "lipid_molecular_species_id", 2639 "lipid_summed_name", 2640 "lipid_subclass", 2641 "lipid_class", 2642 "lipid_category", 2643 "entropy_similarity", 2644 "ref_mz_in_query_fract", 2645 "n_spectra_contributing", 2646 ] 2647 ] 2648 2649 # Set the index to mf_id 2650 lipid_summary = lipid_summary.set_index("mf_id") 2651 2652 return lipid_summary 2653 2654 def to_report(self, molecular_metadata=None): 2655 """Create a report of the mass features and their annotations. 2656 2657 Parameters 2658 ---------- 2659 molecular_metadata : dict, optional 2660 The molecular metadata. Default is None. 2661 2662 Returns 2663 ------- 2664 DataFrame 2665 The report of the mass features and their annotations. 2666 2667 Notes 2668 ----- 2669 The report will contain the mass features and their annotations from MS1 and MS2 (if available). 2670 """ 2671 # Get mass feature dataframe 2672 mf_report = self.mass_spectra.mass_features_to_df() 2673 mf_report = mf_report.reset_index(drop=False) 2674 2675 # Get and clean ms1 annotation dataframe 2676 ms1_annot_report = self.mass_spectra.mass_features_ms1_annot_to_df().copy() 2677 ms1_annot_report = self.clean_ms1_report(ms1_annot_report) 2678 ms1_annot_report = ms1_annot_report.reset_index(drop=False) 2679 2680 # Get, summarize, and clean ms2 annotation dataframe 2681 ms2_annot_report = self.mass_spectra.mass_features_ms2_annot_to_df( 2682 molecular_metadata=molecular_metadata 2683 ) 2684 if ms2_annot_report is not None and molecular_metadata is not None: 2685 ms2_annot_report = self.summarize_lipid_report(ms2_annot_report) 2686 ms2_annot_report = self.clean_ms2_report(ms2_annot_report) 2687 ms2_annot_report = ms2_annot_report.dropna(axis=1, how="all") 2688 ms2_annot_report = ms2_annot_report.reset_index(drop=False) 2689 report = self.combine_reports( 2690 mf_report=mf_report, 2691 ms1_annot_report=ms1_annot_report, 2692 ms2_annot_report=ms2_annot_report, 2693 ) 2694 return report
A class to export lipidomics data.
This class provides methods to export lipidomics data to various formats (CSV/Parquet reports, HDF5) and summarize the lipid report.
Parameters
- out_file_path (str | Path): The output file path, do not include the file extension.
- mass_spectra (object): The high resolution mass spectra object.
LipidomicsExport(out_file_path, mass_spectra)
2355 def __init__(self, out_file_path, mass_spectra): 2356 super().__init__(out_file_path, mass_spectra)
This constructor should always be called with keyword arguments. Arguments are:
group should be None; reserved for future extension when a ThreadGroup class is implemented.
target is the callable object to be invoked by the run() method. Defaults to None, meaning nothing is called.
name is the thread name. By default, a unique name is constructed of the form "Thread-N" where N is a small decimal number.
args is a list or tuple of arguments for the target invocation. Defaults to ().
kwargs is a dictionary of keyword arguments for the target invocation. Defaults to {}.
If a subclass overrides the constructor, it must make sure to invoke the base class constructor (Thread.__init__()) before doing anything else to the thread.
def
summarize_lipid_report(self, ms2_annot):
2358 def summarize_lipid_report(self, ms2_annot): 2359 """Summarize the lipid report. 2360 2361 Parameters 2362 ---------- 2363 ms2_annot : DataFrame 2364 The MS2 annotation DataFrame with all annotations. 2365 2366 Returns 2367 ------- 2368 DataFrame 2369 The summarized lipid report. 2370 """ 2371 # Drop unnecessary columns for easier viewing 2372 columns_to_drop = [ 2373 "precursor_mz", 2374 "precursor_mz_error_ppm", 2375 "ref_mol_id", 2376 "ref_precursor_mz", 2377 "cas", 2378 "inchikey", 2379 "inchi", 2380 "chebi", 2381 "smiles", 2382 "kegg", 2383 "data_id", 2384 "iupac_name", 2385 "traditional_name", 2386 "common_name", 2387 "casno", 2388 ] 2389 ms2_annot = ms2_annot.drop( 2390 columns=[col for col in columns_to_drop if col in ms2_annot.columns] 2391 ) 2392 2393 # If ion_types_excluded is not empty, remove those ion types 2394 ion_types_excluded = self.mass_spectra.parameters.mass_spectrum[ 2395 "ms2" 2396 ].molecular_search.ion_types_excluded 2397 if len(ion_types_excluded) > 0: 2398 ms2_annot = ms2_annot[~ms2_annot["ref_ion_type"].isin(ion_types_excluded)] 2399 2400 # If mf_id is not present, check that the index name is mf_id and reset the index 2401 if "mf_id" not in ms2_annot.columns: 2402 if ms2_annot.index.name == "mf_id": 2403 ms2_annot = ms2_annot.reset_index() 2404 else: 2405 raise ValueError("mf_id is not present in the dataframe") 2406 2407 # Attempt to get consensus annotations to the MLF level 2408 mlf_results_all = [] 2409 for mf_id in ms2_annot["mf_id"].unique(): 2410 mlf_results_perid = [] 2411 ms2_annot_mf = ms2_annot[ms2_annot["mf_id"] == mf_id].copy() 2412 ms2_annot_mf["n_spectra_contributing"] = ( 2413 ms2_annot_mf.query_spectrum_id.nunique() 2414 ) 2415 2416 for query_scan in ms2_annot["query_spectrum_id"].unique(): 2417 ms2_annot_sub = ms2_annot_mf[ 2418 ms2_annot_mf["query_spectrum_id"] == query_scan 2419 ].copy() 2420 2421 if ms2_annot_sub["lipid_summed_name"].nunique() == 1: 2422 # If there is only one lipid_summed_name, let's try to get consensus molecular species annotation 2423 if ms2_annot_sub["lipid_summed_name"].nunique() == 1: 2424 ms2_annot_sub["entropy_max"] = ( 2425 ms2_annot_sub["entropy_similarity"] 2426 == ms2_annot_sub["entropy_similarity"].max() 2427 ) 2428 ms2_annot_sub["ref_match_fract_max"] = ( 2429 ms2_annot_sub["ref_mz_in_query_fract"] 2430 == ms2_annot_sub["ref_mz_in_query_fract"].max() 2431 ) 2432 ms2_annot_sub["frag_max"] = ms2_annot_sub[ 2433 "query_frag_types" 2434 ].apply(lambda x: True if "MLF" in x else False) 2435 2436 # New column that looks if there is a consensus between the ranks (one row that is highest in all ranks) 2437 ms2_annot_sub["consensus"] = ms2_annot_sub[ 2438 ["entropy_max", "ref_match_fract_max", "frag_max"] 2439 ].all(axis=1) 2440 2441 # If there is a consensus, take the row with the highest entropy_similarity 2442 if ms2_annot_sub["consensus"].any(): 2443 ms2_annot_sub = ms2_annot_sub[ 2444 ms2_annot_sub["entropy_similarity"] 2445 == ms2_annot_sub["entropy_similarity"].max() 2446 ].head(1) 2447 mlf_results_perid.append(ms2_annot_sub) 2448 if len(mlf_results_perid) == 0: 2449 mlf_results_perid = pd.DataFrame() 2450 else: 2451 mlf_results_perid = pd.concat(mlf_results_perid) 2452 if mlf_results_perid["name"].nunique() == 1: 2453 mlf_results_perid = mlf_results_perid[ 2454 mlf_results_perid["entropy_similarity"] 2455 == mlf_results_perid["entropy_similarity"].max() 2456 ].head(1) 2457 else: 2458 mlf_results_perid = pd.DataFrame() 2459 mlf_results_all.append(mlf_results_perid) 2460 2461 # These are the consensus annotations to the MLF level 2462 if len(mlf_results_all) > 0: 2463 mlf_results_all = pd.concat(mlf_results_all) 2464 mlf_results_all["annot_level"] = mlf_results_all["structure_level"] 2465 else: 2466 # Make an empty dataframe 2467 mlf_results_all = ms2_annot.head(0) 2468 2469 # For remaining mf_ids, try to get a consensus annotation to the species level 2470 species_results_all = [] 2471 # Remove mf_ids that have consensus annotations to the MLF level 2472 ms2_annot_spec = ms2_annot[ 2473 ~ms2_annot["mf_id"].isin(mlf_results_all["mf_id"].unique()) 2474 ] 2475 for mf_id in ms2_annot_spec["mf_id"].unique(): 2476 # Do all the hits have the same lipid_summed_name? 2477 ms2_annot_sub = ms2_annot_spec[ms2_annot_spec["mf_id"] == mf_id].copy() 2478 ms2_annot_sub["n_spectra_contributing"] = len(ms2_annot_sub) 2479 2480 if ms2_annot_sub["lipid_summed_name"].nunique() == 1: 2481 # Grab the highest entropy_similarity result 2482 ms2_annot_sub = ms2_annot_sub[ 2483 ms2_annot_sub["entropy_similarity"] 2484 == ms2_annot_sub["entropy_similarity"].max() 2485 ].head(1) 2486 species_results_all.append(ms2_annot_sub) 2487 2488 # These are the consensus annotations to the species level 2489 if len(species_results_all) > 0: 2490 species_results_all = pd.concat(species_results_all) 2491 species_results_all["annot_level"] = "species" 2492 else: 2493 # Make an empty dataframe 2494 species_results_all = ms2_annot.head(0) 2495 2496 # Deal with the remaining mf_ids that do not have consensus annotations to the species level or MLF level 2497 # Remove mf_ids that have consensus annotations to the species level 2498 ms2_annot_remaining = ms2_annot_spec[ 2499 ~ms2_annot_spec["mf_id"].isin(species_results_all["mf_id"].unique()) 2500 ] 2501 no_consensus = [] 2502 for mf_id in ms2_annot_remaining["mf_id"].unique(): 2503 id_sub = [] 2504 id_no_con = [] 2505 ms2_annot_sub_mf = ms2_annot_remaining[ 2506 ms2_annot_remaining["mf_id"] == mf_id 2507 ].copy() 2508 for query_scan in ms2_annot_sub_mf["query_spectrum_id"].unique(): 2509 ms2_annot_sub = ms2_annot_sub_mf[ 2510 ms2_annot_sub_mf["query_spectrum_id"] == query_scan 2511 ].copy() 2512 2513 # New columns for ranking [HIGHER RANK = BETTER] 2514 ms2_annot_sub["entropy_max"] = ( 2515 ms2_annot_sub["entropy_similarity"] 2516 == ms2_annot_sub["entropy_similarity"].max() 2517 ) 2518 ms2_annot_sub["ref_match_fract_max"] = ( 2519 ms2_annot_sub["ref_mz_in_query_fract"] 2520 == ms2_annot_sub["ref_mz_in_query_fract"].max() 2521 ) 2522 ms2_annot_sub["frag_max"] = ms2_annot_sub["query_frag_types"].apply( 2523 lambda x: True if "MLF" in x else False 2524 ) 2525 2526 # New column that looks if there is a consensus between the ranks (one row that is highest in all ranks) 2527 ms2_annot_sub["consensus"] = ms2_annot_sub[ 2528 ["entropy_max", "ref_match_fract_max", "frag_max"] 2529 ].all(axis=1) 2530 ms2_annot_sub_con = ms2_annot_sub[ms2_annot_sub["consensus"]] 2531 id_sub.append(ms2_annot_sub_con) 2532 id_no_con.append(ms2_annot_sub) 2533 id_sub = pd.concat(id_sub) 2534 id_no_con = pd.concat(id_no_con) 2535 2536 # Scenario 1: Multiple scans are being resolved to different MLFs [could be coelutions and should both be kept and annotated to MS level] 2537 if ( 2538 id_sub["query_frag_types"] 2539 .apply(lambda x: True if "MLF" in x else False) 2540 .all() 2541 and len(id_sub) > 0 2542 ): 2543 idx = id_sub.groupby("name")["entropy_similarity"].idxmax() 2544 id_sub = id_sub.loc[idx] 2545 # Reorder so highest entropy_similarity is first 2546 id_sub = id_sub.sort_values("entropy_similarity", ascending=False) 2547 id_sub["annot_level"] = id_sub["structure_level"] 2548 no_consensus.append(id_sub) 2549 2550 # Scenario 2: Multiple scans are being resolved to different species, keep both and annotate to appropriate level 2551 elif len(id_sub) == 0: 2552 for lipid_summed_name in id_no_con["lipid_summed_name"].unique(): 2553 summed_sub = id_no_con[ 2554 id_no_con["lipid_summed_name"] == lipid_summed_name 2555 ] 2556 # Any consensus to MLF? 2557 if summed_sub["consensus"].any(): 2558 summed_sub = summed_sub[summed_sub["consensus"]] 2559 summed_sub["annot_level"] = summed_sub["structure_level"] 2560 no_consensus.append(summed_sub) 2561 else: 2562 # Grab the highest entropy_similarity, if there are multiple, grab the first one 2563 summed_sub = summed_sub[ 2564 summed_sub["entropy_similarity"] 2565 == summed_sub["entropy_similarity"].max() 2566 ].head(1) 2567 # get first row 2568 summed_sub["annot_level"] = "species" 2569 summed_sub["name"] = "" 2570 no_consensus.append(summed_sub) 2571 else: 2572 raise ValueError("Unexpected scenario for summarizing mf_id: ", mf_id) 2573 2574 if len(no_consensus) > 0: 2575 no_consensus = pd.concat(no_consensus) 2576 else: 2577 no_consensus = ms2_annot.head(0) 2578 2579 # Combine all the consensus annotations and reformat the dataframe for output 2580 species_results_all = species_results_all.drop(columns=["name"]) 2581 species_results_all["lipid_molecular_species_id"] = "" 2582 mlf_results_all["lipid_molecular_species_id"] = mlf_results_all["name"] 2583 no_consensus["lipid_molecular_species_id"] = no_consensus["name"] 2584 consensus_annotations = pd.concat( 2585 [mlf_results_all, species_results_all, no_consensus] 2586 ) 2587 consensus_annotations = consensus_annotations.sort_values( 2588 "mf_id", ascending=True 2589 ) 2590 cols_to_keep = [ 2591 "mf_id", 2592 "ref_ion_type", 2593 "entropy_similarity", 2594 "ref_mz_in_query_fract", 2595 "lipid_molecular_species_id", 2596 "lipid_summed_name", 2597 "lipid_subclass", 2598 "lipid_class", 2599 "lipid_category", 2600 "formula", 2601 "annot_level", 2602 "n_spectra_contributing", 2603 ] 2604 consensus_annotations = consensus_annotations[cols_to_keep] 2605 consensus_annotations = consensus_annotations.set_index("mf_id") 2606 2607 return consensus_annotations
Summarize the lipid report.
Parameters
- ms2_annot (DataFrame): The MS2 annotation DataFrame with all annotations.
Returns
- DataFrame: The summarized lipid report.
def
clean_ms2_report(self, lipid_summary):
2609 def clean_ms2_report(self, lipid_summary): 2610 """Clean the MS2 report. 2611 2612 Parameters 2613 ---------- 2614 lipid_summary : DataFrame 2615 The full lipid summary DataFrame. 2616 2617 Returns 2618 ------- 2619 DataFrame 2620 The cleaned lipid summary DataFrame. 2621 """ 2622 lipid_summary = lipid_summary.reset_index() 2623 # Library (MS2 metadata) formulas — keep separate from MS1 search fields 2624 lipid_summary["library_ion_formula"] = [ 2625 self.get_ion_formula(f, a) 2626 for f, a in zip(lipid_summary["formula"], lipid_summary["ref_ion_type"]) 2627 ] 2628 lipid_summary = lipid_summary.rename(columns={"formula": "library_formula"}) 2629 2630 # Reorder columns 2631 lipid_summary = lipid_summary[ 2632 [ 2633 "mf_id", 2634 "library_ion_formula", 2635 "ref_ion_type", 2636 "library_formula", 2637 "annot_level", 2638 "lipid_molecular_species_id", 2639 "lipid_summed_name", 2640 "lipid_subclass", 2641 "lipid_class", 2642 "lipid_category", 2643 "entropy_similarity", 2644 "ref_mz_in_query_fract", 2645 "n_spectra_contributing", 2646 ] 2647 ] 2648 2649 # Set the index to mf_id 2650 lipid_summary = lipid_summary.set_index("mf_id") 2651 2652 return lipid_summary
Clean the MS2 report.
Parameters
- lipid_summary (DataFrame): The full lipid summary DataFrame.
Returns
- DataFrame: The cleaned lipid summary DataFrame.
def
to_report(self, molecular_metadata=None):
2654 def to_report(self, molecular_metadata=None): 2655 """Create a report of the mass features and their annotations. 2656 2657 Parameters 2658 ---------- 2659 molecular_metadata : dict, optional 2660 The molecular metadata. Default is None. 2661 2662 Returns 2663 ------- 2664 DataFrame 2665 The report of the mass features and their annotations. 2666 2667 Notes 2668 ----- 2669 The report will contain the mass features and their annotations from MS1 and MS2 (if available). 2670 """ 2671 # Get mass feature dataframe 2672 mf_report = self.mass_spectra.mass_features_to_df() 2673 mf_report = mf_report.reset_index(drop=False) 2674 2675 # Get and clean ms1 annotation dataframe 2676 ms1_annot_report = self.mass_spectra.mass_features_ms1_annot_to_df().copy() 2677 ms1_annot_report = self.clean_ms1_report(ms1_annot_report) 2678 ms1_annot_report = ms1_annot_report.reset_index(drop=False) 2679 2680 # Get, summarize, and clean ms2 annotation dataframe 2681 ms2_annot_report = self.mass_spectra.mass_features_ms2_annot_to_df( 2682 molecular_metadata=molecular_metadata 2683 ) 2684 if ms2_annot_report is not None and molecular_metadata is not None: 2685 ms2_annot_report = self.summarize_lipid_report(ms2_annot_report) 2686 ms2_annot_report = self.clean_ms2_report(ms2_annot_report) 2687 ms2_annot_report = ms2_annot_report.dropna(axis=1, how="all") 2688 ms2_annot_report = ms2_annot_report.reset_index(drop=False) 2689 report = self.combine_reports( 2690 mf_report=mf_report, 2691 ms1_annot_report=ms1_annot_report, 2692 ms2_annot_report=ms2_annot_report, 2693 ) 2694 return report
Create a report of the mass features and their annotations.
Parameters
- molecular_metadata (dict, optional): The molecular metadata. Default is None.
Returns
- DataFrame: The report of the mass features and their annotations.
Notes
The report will contain the mass features and their annotations from MS1 and MS2 (if available).
Inherited Members
class
LCMSCollectionExport:
2697class LCMSCollectionExport: 2698 """A class to export an LCMS collection to HDF5 or Parquet format. 2699 2700 This class provides methods to export collection-level data from multi-sample LC-MS 2701 experiments to HDF5 or Parquet files. It handles the export of metadata, retention time 2702 alignments, cluster assignments, and induced mass features (gap-filled features) across 2703 the collection. 2704 2705 The exporter is designed to work with LCMSCollection objects and complements the individual 2706 LCMSExport class by focusing on collection-wide data rather than individual sample data. 2707 2708 Parameters 2709 ---------- 2710 out_file_path : str | Path 2711 The output file path, do not include the file extension. The .hdf5 / .parquet 2712 extensions are added automatically as needed. 2713 mass_spectra_collection : LCMSCollection 2714 The LCMS collection object containing multiple LCMS samples with processed mass features, 2715 alignments, and clustering information. 2716 2717 Attributes 2718 ---------- 2719 out_file_path : Path 2720 The output file path as a Path object. 2721 mass_spectra_collection : LCMSCollection 2722 The LCMS collection object to be exported. 2723 2724 Methods 2725 ------- 2726 export_to_hdf5(overwrite=False) 2727 Export the LCMS collection to an HDF5 file with collection-level data. 2728 export_to_parquet(overwrite=False) 2729 Export collection-level tabular datasets to Parquet files. 2730 2731 Notes 2732 ----- 2733 This class exports collection-level data including: 2734 - Sample manifest (metadata about all samples in the collection) 2735 - Retention time alignment data (if RT alignment has been performed) 2736 - Cluster assignments (consensus mass feature groupings across samples) 2737 - Induced mass features (gap-filled features saved to individual LCMS object HDF5 files) 2738 2739 Individual sample data (mass spectra, mass features, EICs, etc.) should be exported 2740 separately using the LCMSExport class for each LCMS object in the collection. 2741 2742 Examples 2743 -------- 2744 Export a collection after clustering and gap-filling: 2745 2746 >>> from corems.mass_spectra.output.export import LCMSCollectionExporter 2747 >>> exporter = LCMSCollectionExporter("my_collection", lcms_collection) 2748 >>> exporter.export_to_hdf5(overwrite=True) 2749 2750 The resulting HDF5 file will contain collection-level metadata and can be used 2751 to reconstruct the collection state for further analysis. 2752 2753 See Also 2754 -------- 2755 LCMSExport : Export individual LCMS objects to HDF5 2756 LCMSCollection : The collection object being exported 2757 """ 2758 2759 def __init__(self, out_file_path, mass_spectra_collection): 2760 self.out_file_path = Path(out_file_path) 2761 self.mass_spectra_collection = mass_spectra_collection 2762 2763 def export_to_hdf5( 2764 self, 2765 overwrite=False, 2766 save_parameters=True, 2767 parameter_format="toml", 2768 update_lcms_objects=True, 2769 ): 2770 """Export the LCMS collection to an HDF5 file. 2771 2772 This method saves the collection-level data to an HDF5 file, including: 2773 - Basic metadata (date, folder location, gap-filling status) 2774 - Sample manifest 2775 - Retention time alignments (if available) 2776 - Cluster assignments (if available) 2777 - Induced mass features for each LCMS object (if gap-filling was performed) 2778 2779 Individual LCMS objects in the collection are not exported by this method. 2780 Use LCMSExport for exporting individual LCMS objects. 2781 2782 Parameters 2783 ---------- 2784 overwrite : bool, optional 2785 If True, overwrites the output file if it already exists and replaces 2786 existing groups within the HDF5 file. If False, appends new data to 2787 existing file without overwriting existing groups. Default is False. 2788 save_parameters : bool, optional 2789 If True, saves the collection-level parameters to a separate file in the specified format. 2790 Default is True. 2791 parameter_format : str, optional 2792 The format for saving parameters, either "json" or "toml". Default is "toml". 2793 update_lcms_objects : bool, optional 2794 If True, updates the individual LCMS object HDF5 files with new raw file locations and any additional 2795 information produced during the processing of the collection (e.g. cluster mass feature associations). Default is True. 2796 2797 Notes 2798 ----- 2799 The HDF5 file structure includes: 2800 - Attributes: date_utc, lcms_objects_folder, missing_mass_features_searched, manifest 2801 - Groups: rt_alignments, cluster_assignments (if available) 2802 2803 Induced mass features are saved to the individual LCMS object HDF5 files 2804 within the .corems folder structure, not in the collection-level HDF5 file. 2805 2806 Examples 2807 -------- 2808 >>> exporter = LCMSCollectionExporter("my_collection", lcms_collection) 2809 >>> exporter.export_to_hdf5(overwrite=True) 2810 """ 2811 if overwrite: 2812 if self.out_file_path.with_suffix(".hdf5").exists(): 2813 self.out_file_path.with_suffix(".hdf5").unlink() 2814 2815 with h5py.File(self.out_file_path.with_suffix(".hdf5"), "a") as hdf_handle: 2816 # Add basic attributes to the HDF5 file, always overwrite these 2817 timenow = str(datetime.now(timezone.utc).strftime("%d/%m/%Y %H:%M:%S %Z")) 2818 hdf_handle.attrs["date_utc"] = timenow 2819 hdf_handle.attrs["lcms_objects_folder"] = str( 2820 self.mass_spectra_collection.collection_parser.folder_location 2821 ) 2822 hdf_handle.attrs["missing_mass_features_searched"] = ( 2823 self.mass_spectra_collection.missing_mass_features_searched 2824 ) 2825 hdf_handle.attrs["rt_aligned"] = self.mass_spectra_collection.rt_aligned 2826 hdf_handle.attrs["rt_alignment_attempted"] = ( 2827 self.mass_spectra_collection.rt_alignment_attempted 2828 ) 2829 2830 # Add the manifest to the HDF5 file, always overwrite this 2831 hdf_handle.attrs["manifest"] = self._convert_manifest_to_json() 2832 2833 # Save retention time alignments if they exist, only overwrite if specified 2834 self._save_rt_alignments_to_hdf5(hdf_handle, overwrite) 2835 2836 # Save cluster assignments if they exist, only overwrite if specified 2837 self._save_cluster_assignments_to_hdf5(hdf_handle, overwrite) 2838 2839 # Save consensus feature-group labels if present 2840 self._save_feature_group_labels_to_hdf5(hdf_handle, overwrite) 2841 2842 # Save new raw file locations to each LCMS object's HDF5 file if needed 2843 if ( 2844 hasattr(self.mass_spectra_collection, "raw_files_relocated") 2845 and self.mass_spectra_collection.raw_files_relocated 2846 ): 2847 self._update_raw_file_locations_in_hdf5() 2848 2849 # Save induced mass features to the collection with associations to each individual, only if lcms_collection.missing_mass_features_searched is True 2850 if self.mass_spectra_collection.missing_mass_features_searched: 2851 self._save_induced_mass_features_to_hdf5(overwrite) 2852 # Save EICs for induced mass features at collection level 2853 self._save_induced_eics_to_hdf5(overwrite) 2854 2855 # Build cluster mass feature map to know which features to update 2856 # This uses the same logic as process_consensus_features to determine loaded features 2857 cluster_mf_map = self._build_cluster_mf_map() 2858 2859 # Save updated mass features for each LCMS object 2860 # This implements selective update: only loaded features are updated, non-cluster features are preserved 2861 if update_lcms_objects: 2862 self._save_lcms_objects_to_hdf5(cluster_mf_map, overwrite) 2863 2864 # Save collection-level parameters as separate file 2865 if save_parameters: 2866 # Check if parameter_format is valid 2867 if parameter_format not in ["json", "toml"]: 2868 raise ValueError("parameter_format must be 'json' or 'toml'") 2869 2870 if parameter_format == "json": 2871 dump_lcms_collection_settings_json( 2872 filename=self.out_file_path.with_suffix(".json"), 2873 lcms_collection=self.mass_spectra_collection, 2874 ) 2875 elif parameter_format == "toml": 2876 dump_lcms_collection_settings_toml( 2877 filename=self.out_file_path.with_suffix(".toml"), 2878 lcms_collection=self.mass_spectra_collection, 2879 ) 2880 2881 def export_to_parquet( 2882 self, overwrite=False, save_parameters=True, parameter_format="toml" 2883 ): 2884 """Export collection-level LC-MS datasets to Parquet files. 2885 2886 Writes next to ``out_file_path`` (same stem as the HDF5 export): 2887 2888 - ``{stem}_manifest.parquet`` — sample manifest 2889 - ``{stem}_cluster_assignments.parquet`` — cluster labels (if present) 2890 - ``{stem}_mass_features.parquet`` — collection mass-features table (if present) 2891 - ``{stem}_rt_alignments.parquet`` — RT alignment curves (if aligned) 2892 - parameters as JSON or TOML when ``save_parameters`` is True 2893 2894 Parameters 2895 ---------- 2896 overwrite : bool, optional 2897 Whether to overwrite existing Parquet files. Default is False. 2898 save_parameters : bool, optional 2899 Whether to save collection parameters. Default is True. 2900 parameter_format : str, optional 2901 Parameter file format, ``'json'`` or ``'toml'``. Default is ``'toml'``. 2902 2903 Notes 2904 ----- 2905 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 2906 This is a tabular companion to ``export_to_hdf5``; it does not replace 2907 per-sample HDF5 archives or gap-fill side effects on individual LCMS files. 2908 """ 2909 stem = self.out_file_path 2910 2911 def _write(df, suffix): 2912 if df is None or df.empty: 2913 return 2914 out_path = Path(f"{stem}_{suffix}").with_suffix(".parquet") 2915 if out_path.exists() and not overwrite: 2916 return 2917 LCMSExport._prepare_df_for_parquet(df).to_parquet(out_path, index=False) 2918 2919 # Manifest — prefer the tidy samples-as-rows dataframe 2920 manifest_df = getattr(self.mass_spectra_collection, "manifest_dataframe", None) 2921 if manifest_df is None or getattr(manifest_df, "empty", True): 2922 manifest = getattr( 2923 self.mass_spectra_collection, "manifest", None 2924 ) or getattr( 2925 getattr(self.mass_spectra_collection, "collection_parser", None), 2926 "manifest", 2927 None, 2928 ) 2929 if isinstance(manifest, DataFrame): 2930 manifest_df = manifest 2931 elif isinstance(manifest, dict) and manifest: 2932 try: 2933 # Collection manifests are sample -> attrs; transpose to rows 2934 manifest_df = pd.DataFrame(manifest).T 2935 except ValueError: 2936 manifest_df = DataFrame( 2937 [{"manifest_json": self._convert_manifest_to_json()}] 2938 ) 2939 else: 2940 manifest_df = None 2941 2942 if manifest_df is not None and not manifest_df.empty: 2943 _write(manifest_df.reset_index(drop=False), "manifest") 2944 2945 # Cluster assignments / mass features table 2946 mf_df = getattr(self.mass_spectra_collection, "mass_features_dataframe", None) 2947 if mf_df is not None and not mf_df.empty: 2948 _write(mf_df.reset_index(drop=False), "mass_features") 2949 if "cluster" in mf_df.columns: 2950 cluster_df = mf_df[["cluster"]].copy().reset_index(drop=False) 2951 _write(cluster_df, "cluster_assignments") 2952 2953 # RT alignments as long-format table 2954 if getattr(self.mass_spectra_collection, "rt_aligned", False) and getattr( 2955 self.mass_spectra_collection, "rt_alignments", None 2956 ): 2957 frames = [] 2958 for ( 2959 sample_idx, 2960 alignment_data, 2961 ) in self.mass_spectra_collection.rt_alignments.items(): 2962 arr = np.asarray(alignment_data) 2963 if arr.ndim == 1: 2964 frames.append( 2965 DataFrame( 2966 { 2967 "sample_idx": sample_idx, 2968 "point": np.arange(len(arr)), 2969 "value": arr, 2970 } 2971 ) 2972 ) 2973 elif arr.ndim == 2 and arr.shape[1] >= 2: 2974 frames.append( 2975 DataFrame( 2976 { 2977 "sample_idx": sample_idx, 2978 "rt_original": arr[:, 0], 2979 "rt_aligned": arr[:, 1], 2980 } 2981 ) 2982 ) 2983 else: 2984 frames.append( 2985 DataFrame( 2986 { 2987 "sample_idx": [sample_idx], 2988 "alignment_json": [json.dumps(arr.tolist())], 2989 } 2990 ) 2991 ) 2992 if frames: 2993 _write(pd.concat(frames, ignore_index=True), "rt_alignments") 2994 2995 if save_parameters: 2996 if parameter_format not in ["json", "toml"]: 2997 raise ValueError("parameter_format must be 'json' or 'toml'") 2998 if parameter_format == "json": 2999 dump_lcms_collection_settings_json( 3000 filename=self.out_file_path.with_suffix(".json"), 3001 lcms_collection=self.mass_spectra_collection, 3002 ) 3003 elif parameter_format == "toml": 3004 dump_lcms_collection_settings_toml( 3005 filename=self.out_file_path.with_suffix(".toml"), 3006 lcms_collection=self.mass_spectra_collection, 3007 ) 3008 3009 def _save_rt_alignments_to_hdf5(self, hdf_handle, overwrite): 3010 """Save retention time alignments to HDF5 file.""" 3011 # If no rt_alignments, return early 3012 if not self.mass_spectra_collection.rt_aligned: 3013 return 3014 3015 # If rt_alignments exist, save them 3016 if self.mass_spectra_collection.rt_aligned: 3017 group_name = "rt_alignments" 3018 # grab dictionary of rt_alignments 3019 rt_alignments = self.mass_spectra_collection.rt_alignments 3020 3021 if rt_alignments: 3022 # Check if group exists and handle overwrite logic 3023 if group_name in hdf_handle: 3024 if not overwrite: 3025 return 3026 del hdf_handle[group_name] 3027 3028 grp = hdf_handle.create_group(group_name) 3029 3030 # Save each alignment as a dataset 3031 for sample_idx, alignment_data in rt_alignments.items(): 3032 grp.create_dataset(str(sample_idx), data=alignment_data) 3033 3034 def _convert_manifest_to_json(self): 3035 """Clean the manifest for export to HDF5.""" 3036 manifest = self.mass_spectra_collection.collection_parser.manifest 3037 3038 # Process the manifest to convert numpy.bool_ or bool values for the 'use_rt_alignment' key 3039 def convert_bool_values(data): 3040 if isinstance(data, dict): 3041 # Process each key-value pair recursively 3042 return { 3043 k: ( 3044 int(v) 3045 if k == "use_rt_alignment" and isinstance(v, (bool, np.bool_)) 3046 else convert_bool_values(v) 3047 ) 3048 for k, v in data.items() 3049 } 3050 elif isinstance(data, list): 3051 # Recursively process lists 3052 return [convert_bool_values(item) for item in data] 3053 else: 3054 # Return non-dict/list types unchanged 3055 return data 3056 3057 # Clean the whole manifest 3058 cleaned_manifest = convert_bool_values(manifest) 3059 3060 # Serialize the cleaned manifest into JSON format 3061 json_manifest = json.dumps(cleaned_manifest) 3062 return json_manifest 3063 3064 def _save_cluster_assignments_to_hdf5(self, hdf_handle, overwrite): 3065 """Save cluster assignments to HDF5 file.""" 3066 # Check if column "cluster" is present in self.mass_features_dataframe 3067 if "cluster" in self.mass_spectra_collection.mass_features_dataframe.columns: 3068 group_name = "cluster_assignments" 3069 cluster_assignments = self.mass_spectra_collection.mass_features_dataframe[ 3070 ["cluster"] 3071 ].copy() 3072 3073 # Check if group exists and handle overwrite logic 3074 if group_name in hdf_handle: 3075 if not overwrite: 3076 return 3077 del hdf_handle[group_name] 3078 3079 grp = hdf_handle.create_group(group_name) 3080 3081 # Save the index, converting strings to bytes. 3082 # Avoid Index.astype(str).values under pandas 3 string/CoW semantics; 3083 # build a plain numpy bytes array instead. 3084 index_bytes = np.asarray( 3085 [str(x).encode("utf-8") for x in cluster_assignments.index], 3086 dtype="S", 3087 ) 3088 grp.create_dataset("index", data=index_bytes) 3089 3090 # Save the "cluster" column 3091 grp.create_dataset( 3092 "cluster", data=cluster_assignments["cluster"].to_numpy() 3093 ) 3094 3095 def _save_feature_group_labels_to_hdf5(self, hdf_handle, overwrite): 3096 """Save cluster-indexed feature-group labels to the collection HDF5.""" 3097 from corems.mass_spectra.calc.feature_grouping import GROUP_COLUMNS 3098 3099 labels = getattr(self.mass_spectra_collection, "feature_group_dataframe", None) 3100 if labels is None or len(labels) == 0: 3101 return 3102 3103 group_name = "feature_group_labels" 3104 if group_name in hdf_handle: 3105 if not overwrite: 3106 return 3107 del hdf_handle[group_name] 3108 3109 grp = hdf_handle.create_group(group_name) 3110 index_vals = np.asarray( 3111 [int(x) for x in labels.index], 3112 dtype=np.int64, 3113 ) 3114 grp.create_dataset("cluster", data=index_vals) 3115 3116 for col in GROUP_COLUMNS: 3117 if col not in labels.columns: 3118 continue 3119 series = labels[col] 3120 if col in ("feature_group_id", "mono_cluster_id"): 3121 # Nullable ints → float with NaN for missing 3122 data = series.astype("Float64").to_numpy(dtype=float) 3123 grp.create_dataset(col, data=data) 3124 else: 3125 # Object / string columns 3126 encoded = np.asarray( 3127 [ 3128 ( 3129 b"" 3130 if v is None 3131 or (isinstance(v, float) and np.isnan(v)) 3132 or (hasattr(pd, "isna") and pd.isna(v)) 3133 else str(v).encode("utf-8") 3134 ) 3135 for v in series.tolist() 3136 ], 3137 dtype="S", 3138 ) 3139 grp.create_dataset(col, data=encoded) 3140 3141 def _build_cluster_mf_map(self): 3142 """Build a mapping of which mass features should be saved for each sample. 3143 3144 This uses the same logic as process_consensus_features to determine which 3145 mass features were loaded and should be updated in HDF5 files. 3146 3147 Returns 3148 ------- 3149 dict 3150 Dictionary mapping sample_id to list of tuples (mf_id, cluster_id). 3151 Only includes samples that have loaded representative features. 3152 Returns empty dict if no clusters exist. 3153 3154 Notes 3155 ----- 3156 This follows the DRY principle by using the same get_sample_mf_map_for_representatives 3157 method used by process_consensus_features and ReadSavedLCMSCollection. 3158 """ 3159 # Check if clusters exist 3160 if ( 3161 "cluster" 3162 not in self.mass_spectra_collection.mass_features_dataframe.columns 3163 ): 3164 return {} 3165 3166 # Check if cluster_summary_dataframe exists (needed by get_sample_mf_map_for_representatives) 3167 if ( 3168 not hasattr(self.mass_spectra_collection, "cluster_summary_dataframe") 3169 or self.mass_spectra_collection.cluster_summary_dataframe is None 3170 ): 3171 return {} 3172 3173 # Use the same DRY helper method that process_consensus_features uses 3174 # This ensures consistency across the codebase 3175 cluster_mf_map = ( 3176 self.mass_spectra_collection.get_sample_mf_map_for_representatives( 3177 include_cluster_id=True 3178 ) 3179 ) 3180 3181 return cluster_mf_map 3182 3183 def _update_raw_file_locations_in_hdf5(self): 3184 """Update raw file locations in each LCMS object's HDF5 file. 3185 3186 This method updates the 'original_file_location' attribute in each LCMS object's 3187 HDF5 file to reflect the new raw file location after files have been relocated. 3188 """ 3189 for lcms_obj in self.mass_spectra_collection: 3190 # Get the HDF5 file path for this LCMS object 3191 hdf5_path = lcms_obj.file_location.with_suffix(".hdf5") 3192 3193 if hdf5_path.exists(): 3194 with h5py.File(hdf5_path, "a") as hdf_handle: 3195 # Update the original_file_location attribute 3196 if "original_file_location" in hdf_handle.attrs: 3197 hdf_handle.attrs["original_file_location"] = str( 3198 lcms_obj.raw_file_location 3199 ) 3200 # If the attribute does not exist, create it 3201 else: 3202 hdf_handle.attrs.create( 3203 "original_file_location", str(lcms_obj.raw_file_location) 3204 ) 3205 3206 def _save_induced_mass_features_to_hdf5(self, overwrite): 3207 """Save induced mass features to the collection HDF5 file. 3208 3209 Induced mass features are gap-filled features that only exist at the collection level. 3210 They are saved with full detail (all attributes and datasets) in the collection HDF5 file 3211 and distributed to individual LCMS objects when the collection is loaded. 3212 3213 The induced mass features are stored in the collection's induced_mass_features_dataframe 3214 and are regenerated as LCMSMassFeature objects for saving. 3215 3216 Parameters 3217 ---------- 3218 overwrite : bool 3219 If True, overwrites existing induced mass features group. If False, skips if group exists. 3220 """ 3221 # Check if we have any induced mass features to save 3222 if ( 3223 self.mass_spectra_collection.induced_mass_features_dataframe is None 3224 or self.mass_spectra_collection.induced_mass_features_dataframe.empty 3225 ): 3226 return 3227 3228 # Open the collection HDF5 file to save induced mass features 3229 with h5py.File(self.out_file_path.with_suffix(".hdf5"), "a") as hdf_handle: 3230 group_name = "induced_mass_features" 3231 3232 # Check if group exists and handle overwrite logic 3233 if group_name in hdf_handle: 3234 if not overwrite: 3235 return 3236 del hdf_handle[group_name] 3237 3238 # Create top-level group for induced mass features 3239 imf_group = hdf_handle.create_group(group_name) 3240 3241 # Get the induced mass features dataframe 3242 induced_df = self.mass_spectra_collection.induced_mass_features_dataframe 3243 3244 # Get unique sample IDs from the dataframe 3245 sample_ids = induced_df["sample_id"].unique() 3246 3247 # Iterate through each sample and save its induced mass features 3248 for sample_id in sample_ids: 3249 # Filter dataframe to this sample 3250 sample_df = induced_df[induced_df["sample_id"] == sample_id].copy() 3251 3252 if sample_df.empty: 3253 continue 3254 3255 # Regenerate mass features from the dataframe 3256 regenerated_features = self._regenerate_mass_features_from_sample_df( 3257 sample_df, sample_id 3258 ) 3259 3260 if not regenerated_features: 3261 continue 3262 3263 # Create a subgroup for this sample's induced mass features 3264 sample_group = imf_group.create_group(str(sample_id)) 3265 3266 # Use the static helper method from LCMSExport to save the mass features 3267 LCMSExport._save_mass_features_dict_to_hdf5( 3268 regenerated_features, sample_group, overwrite=overwrite 3269 ) 3270 3271 def _save_induced_eics_to_hdf5(self, overwrite): 3272 """Save EICs for induced mass features to the collection HDF5 file. 3273 3274 Induced mass features are gap-filled features created during process_consensus_features. 3275 Their associated EICs need to be saved at the collection level so they can be reloaded. 3276 3277 The induced mass features are identified from the collection's induced_mass_features_dataframe, 3278 and their EICs are retrieved from the individual LCMS objects. 3279 3280 Parameters 3281 ---------- 3282 overwrite : bool 3283 If True, overwrites existing induced EICs group. If False, skips if group exists. 3284 """ 3285 # Check if we have any induced mass features to save 3286 if ( 3287 self.mass_spectra_collection.induced_mass_features_dataframe is None 3288 or self.mass_spectra_collection.induced_mass_features_dataframe.empty 3289 ): 3290 return 3291 3292 # Open the collection HDF5 file to save induced EICs 3293 with h5py.File(self.out_file_path.with_suffix(".hdf5"), "a") as hdf_handle: 3294 group_name = "induced_eics" 3295 3296 # Check if group exists and handle overwrite logic 3297 if group_name in hdf_handle: 3298 if not overwrite: 3299 return 3300 del hdf_handle[group_name] 3301 3302 # Create top-level group for induced EICs 3303 induced_eics_group = hdf_handle.create_group(group_name) 3304 3305 # Get the induced mass features dataframe 3306 induced_df = self.mass_spectra_collection.induced_mass_features_dataframe 3307 3308 # Get unique sample IDs from the dataframe 3309 sample_ids = induced_df["sample_id"].unique() 3310 3311 # Iterate through each sample and save EICs for its induced mass features 3312 for sample_id in sample_ids: 3313 lcms_obj = self.mass_spectra_collection[sample_id] 3314 3315 # Filter dataframe to this sample 3316 sample_df = induced_df[induced_df["sample_id"] == sample_id].copy() 3317 3318 if sample_df.empty: 3319 continue 3320 3321 # Collect EICs for induced mass features using _eic_mz from dataframe 3322 induced_eics = {} 3323 for _, row in sample_df.iterrows(): 3324 # Get the EIC m/z from the dataframe 3325 eic_mz = row.get("_eic_mz") 3326 3327 if eic_mz is not None and pd.notna(eic_mz): 3328 # Try to get the EIC from the LCMS object 3329 if ( 3330 hasattr(lcms_obj, "eics") 3331 and lcms_obj.eics 3332 and eic_mz in lcms_obj.eics 3333 ): 3334 induced_eics[eic_mz] = lcms_obj.eics[eic_mz] 3335 3336 if not induced_eics: 3337 continue 3338 3339 # Create a subgroup for this sample's induced EICs 3340 sample_group = induced_eics_group.create_group(str(sample_id)) 3341 3342 # Use the static helper method from LCMSExport to save the EICs 3343 LCMSExport._save_eics_dict_to_hdf5( 3344 induced_eics, sample_group, overwrite 3345 ) 3346 3347 def _regenerate_mass_features_from_sample_df(self, sample_df, sample_id): 3348 """Regenerate induced mass features from a sample-specific dataframe. 3349 3350 This method creates LCMSMassFeature objects from rows in the induced_mass_features_dataframe 3351 for a specific sample. The regenerated features are used for saving to HDF5. 3352 3353 Parameters 3354 ---------- 3355 sample_df : pd.DataFrame 3356 DataFrame containing induced mass features for a specific sample. 3357 sample_id : int 3358 The sample ID (index in the collection). 3359 3360 Returns 3361 ------- 3362 dict 3363 Dictionary of regenerated LCMSMassFeature objects keyed by feature ID. 3364 """ 3365 from corems.chroma_peak.factory.chroma_peak_classes import LCMSMassFeature 3366 3367 if sample_df.empty: 3368 return {} 3369 3370 # Get the corresponding LCMS object for proper parent reference 3371 lcms_obj = self.mass_spectra_collection[sample_id] 3372 3373 # Regenerate mass features from the dataframe 3374 regenerated_features = {} 3375 3376 for _, row in sample_df.iterrows(): 3377 # Extract the original ID from mf_id (format: c{cluster}_{index}_i) 3378 # This is the ID used in lcms_obj.induced_mass_features dict 3379 original_id = row["mf_id"] 3380 3381 # Create a new LCMSMassFeature with proper parent reference 3382 # Note: dataframe uses 'scan_time' but __init__ parameter is 'retention_time' 3383 mass_feature = LCMSMassFeature( 3384 lcms_parent=lcms_obj, 3385 mz=row["mz"], 3386 retention_time=row["scan_time"], # Column is 'scan_time' in dataframe 3387 intensity=row["intensity"], 3388 apex_scan=int(row["apex_scan"]), 3389 persistence=row.get("persistence", None) 3390 if "persistence" in row 3391 else None, 3392 id=original_id, # Use the original string ID from gap-filling 3393 ) 3394 3395 # Set additional attributes dynamically from dataframe columns 3396 # Skip columns already handled in __init__ or structural metadata 3397 skip_cols = { 3398 "sample_id", 3399 "mf_id", 3400 "mz", 3401 "scan_time", 3402 "scan_time_aligned", 3403 "intensity", 3404 "apex_scan", 3405 "persistence", 3406 } 3407 3408 # Iterate through all columns and set via property setters 3409 for col_name in row.index: 3410 if col_name in skip_cols: 3411 continue 3412 value = row[col_name] 3413 try: 3414 if pd.isna(value): 3415 continue 3416 except (TypeError, ValueError): 3417 pass # value is array-like; not NA, proceed 3418 3419 # Set via property (public interface handles private attributes) 3420 # Don't save empty lists 3421 if isinstance(value, list) and len(value) == 0: 3422 continue 3423 try: 3424 setattr(mass_feature, col_name, value) 3425 except (AttributeError, TypeError): 3426 pass # Skip attributes that don't exist or can't be set 3427 3428 # Set cluster_index if present 3429 if "cluster" in row and pd.notna(row["cluster"]): 3430 mass_feature.cluster_index = int(row["cluster"]) 3431 3432 regenerated_features[mass_feature.id] = mass_feature 3433 3434 return regenerated_features 3435 3436 def _save_lcms_objects_to_hdf5(self, cluster_mf_map, overwrite): 3437 """Save updated mass features for each LCMS object. 3438 3439 This method implements a "selective update" strategy for mass features: 3440 - For mass features specified in cluster_mf_map (loaded representatives), we selectively 3441 update them by deleting their old entries and re-saving with new attributes. 3442 - Non-cluster features (not loaded) are never touched/overwritten. 3443 3444 Note: EICs are NOT saved here. Induced feature EICs are saved at the collection level. 3445 3446 Parameters 3447 ---------- 3448 cluster_mf_map : dict 3449 Dictionary mapping sample_id to list of tuples (mf_id, cluster_id). 3450 This explicitly defines which mass features should be updated. 3451 overwrite : bool 3452 If True, allows overwriting of existing data. If False, skips if data exists. 3453 """ 3454 for sample_id, lcms_obj in enumerate(self.mass_spectra_collection): 3455 hdf5_path = lcms_obj.file_location.with_suffix(".hdf5") 3456 3457 if not hdf5_path.exists(): 3458 # If HDF5 doesn't exist, we can't do selective update, raise error 3459 raise FileNotFoundError( 3460 f"HDF5 file for LCMS object {lcms_obj.sample_name} not found at {hdf5_path}" 3461 ) 3462 3463 # Check if this sample has any loaded features in the map 3464 if sample_id not in cluster_mf_map or not cluster_mf_map[sample_id]: 3465 # Nothing loaded for this sample, nothing to update 3466 continue 3467 3468 # Extract mf_ids from the map (cluster_mf_map contains tuples of (mf_id, cluster_id)) 3469 mf_ids_to_update = [ 3470 mf_id for mf_id, cluster_id in cluster_mf_map[sample_id] 3471 ] 3472 3473 # Perform selective update of mass features 3474 self._selective_update_mass_features( 3475 lcms_obj, hdf5_path, mf_ids_to_update, overwrite 3476 ) 3477 3478 # Save any new mass spectra that were added during processing 3479 self._save_new_mass_spectra(lcms_obj, hdf5_path, overwrite) 3480 3481 def _save_new_mass_spectra(self, lcms_obj, hdf5_path, overwrite): 3482 """Save new mass spectra that were added during processing. 3483 3484 This method checks what mass spectra are in lcms_obj._ms and saves any 3485 that aren't already in the HDF5 file's mass_spectra group. Uses the 3486 existing add_mass_spectrum_to_hdf5 method for consistency with original 3487 export logic. 3488 3489 Parameters 3490 ---------- 3491 lcms_obj : LCMSBase 3492 The LCMS object with potentially new mass spectra. 3493 hdf5_path : Path 3494 Path to the HDF5 file. 3495 overwrite : bool 3496 If True, allows overwriting existing spectra. 3497 """ 3498 # Check if there are any mass spectra to save 3499 if not hasattr(lcms_obj, "_ms") or not lcms_obj._ms: 3500 return 3501 3502 # Create an LCMS exporter instance for this LCMS object 3503 # This gives us access to add_mass_spectrum_to_hdf5 method inherited from HighResMassSpecExport 3504 # Turn hdf5_path into str without suffix for LCMSExport 3505 hdf5_path_str = str(hdf5_path.with_suffix("")) 3506 exporter = LCMSExport(out_file_path=hdf5_path_str, mass_spectra=lcms_obj) 3507 3508 # Open HDF5 file and check existing mass spectra 3509 with h5py.File(hdf5_path, "a") as hdf_handle: 3510 # Create mass_spectra group if it doesn't exist 3511 if "mass_spectra" not in hdf_handle: 3512 ms_group = hdf_handle.create_group("mass_spectra") 3513 existing_scan_numbers = set() 3514 else: 3515 ms_group = hdf_handle["mass_spectra"] 3516 existing_scan_numbers = set(int(k) for k in ms_group.keys()) 3517 3518 # Find new mass spectra (in _ms but not in HDF5) 3519 new_scan_numbers = set(lcms_obj._ms.keys()) - existing_scan_numbers 3520 3521 if not new_scan_numbers: 3522 return 3523 3524 # Save new mass spectra using existing add_mass_spectrum_to_hdf5 method 3525 export_profile = lcms_obj.parameters.lc_ms.export_profile_spectra 3526 for scan_number in new_scan_numbers: 3527 mass_spec = lcms_obj._ms[scan_number] 3528 scan_group_name = str(scan_number) 3529 3530 # Delete existing group if overwrite is True 3531 if scan_group_name in ms_group and overwrite: 3532 del ms_group[scan_group_name] 3533 elif scan_group_name in ms_group: 3534 continue 3535 3536 # Use the existing method from HighResMassSpecExport 3537 exporter.add_mass_spectrum_to_hdf5( 3538 hdf_handle=hdf_handle, 3539 mass_spectrum=mass_spec, 3540 group_key=scan_group_name, 3541 mass_spectra_group=ms_group, 3542 export_raw=export_profile, 3543 ) 3544 3545 def _selective_update_mass_features( 3546 self, lcms_obj, hdf5_path, mf_ids_to_update, overwrite 3547 ): 3548 """Selectively update mass features in HDF5 file. 3549 3550 This method deletes only the mass features specified in mf_ids_to_update, 3551 then re-saves them with their potentially updated attributes. Non-cluster features 3552 in the HDF5 file are left untouched. 3553 3554 Parameters 3555 ---------- 3556 lcms_obj : LCMSBase 3557 The LCMS object with mass features to update. 3558 hdf5_path : Path 3559 Path to the HDF5 file. 3560 mf_ids_to_update : list of int 3561 List of mass feature IDs that should be updated. This explicitly defines 3562 which features were loaded and should be saved. 3563 overwrite : bool 3564 If True, allows overwriting. If False, skips if group exists. 3565 """ 3566 if not mf_ids_to_update: 3567 return 3568 3569 # Open HDF5 file and delete specified feature IDs, then re-save 3570 with h5py.File(hdf5_path, "a") as hdf_handle: 3571 if "mass_features" not in hdf_handle: 3572 return 3573 3574 mf_group = hdf_handle["mass_features"] 3575 3576 # Delete features that are being updated 3577 for feature_id in mf_ids_to_update: 3578 feature_id_str = str(feature_id) 3579 if feature_id_str in mf_group: 3580 del mf_group[feature_id_str] 3581 3582 # Re-save updated features (only those that exist in mass_features dict) 3583 updated_features = { 3584 mf.id: mf 3585 for mf in lcms_obj.mass_features.values() 3586 if mf.id in mf_ids_to_update 3587 } 3588 3589 if updated_features: 3590 LCMSExport._save_mass_features_dict_to_hdf5( 3591 updated_features, mf_group, overwrite=overwrite 3592 )
A class to export an LCMS collection to HDF5 or Parquet format.
This class provides methods to export collection-level data from multi-sample LC-MS experiments to HDF5 or Parquet files. It handles the export of metadata, retention time alignments, cluster assignments, and induced mass features (gap-filled features) across the collection.
The exporter is designed to work with LCMSCollection objects and complements the individual LCMSExport class by focusing on collection-wide data rather than individual sample data.
Parameters
- out_file_path (str | Path): The output file path, do not include the file extension. The .hdf5 / .parquet extensions are added automatically as needed.
- mass_spectra_collection (LCMSCollection): The LCMS collection object containing multiple LCMS samples with processed mass features, alignments, and clustering information.
Attributes
- out_file_path (Path): The output file path as a Path object.
- mass_spectra_collection (LCMSCollection): The LCMS collection object to be exported.
Methods
export_to_hdf5(overwrite=False) Export the LCMS collection to an HDF5 file with collection-level data. export_to_parquet(overwrite=False) Export collection-level tabular datasets to Parquet files.
Notes
This class exports collection-level data including:
- Sample manifest (metadata about all samples in the collection)
- Retention time alignment data (if RT alignment has been performed)
- Cluster assignments (consensus mass feature groupings across samples)
- Induced mass features (gap-filled features saved to individual LCMS object HDF5 files)
Individual sample data (mass spectra, mass features, EICs, etc.) should be exported separately using the LCMSExport class for each LCMS object in the collection.
Examples
Export a collection after clustering and gap-filling:
>>> from corems.mass_spectra.output.export import LCMSCollectionExporter
>>> exporter = LCMSCollectionExporter("my_collection", lcms_collection)
>>> exporter.export_to_hdf5(overwrite=True)
The resulting HDF5 file will contain collection-level metadata and can be used to reconstruct the collection state for further analysis.
See Also
LCMSExport: Export individual LCMS objects to HDF5
LCMSCollection: The collection object being exported
def
export_to_hdf5( self, overwrite=False, save_parameters=True, parameter_format='toml', update_lcms_objects=True):
2763 def export_to_hdf5( 2764 self, 2765 overwrite=False, 2766 save_parameters=True, 2767 parameter_format="toml", 2768 update_lcms_objects=True, 2769 ): 2770 """Export the LCMS collection to an HDF5 file. 2771 2772 This method saves the collection-level data to an HDF5 file, including: 2773 - Basic metadata (date, folder location, gap-filling status) 2774 - Sample manifest 2775 - Retention time alignments (if available) 2776 - Cluster assignments (if available) 2777 - Induced mass features for each LCMS object (if gap-filling was performed) 2778 2779 Individual LCMS objects in the collection are not exported by this method. 2780 Use LCMSExport for exporting individual LCMS objects. 2781 2782 Parameters 2783 ---------- 2784 overwrite : bool, optional 2785 If True, overwrites the output file if it already exists and replaces 2786 existing groups within the HDF5 file. If False, appends new data to 2787 existing file without overwriting existing groups. Default is False. 2788 save_parameters : bool, optional 2789 If True, saves the collection-level parameters to a separate file in the specified format. 2790 Default is True. 2791 parameter_format : str, optional 2792 The format for saving parameters, either "json" or "toml". Default is "toml". 2793 update_lcms_objects : bool, optional 2794 If True, updates the individual LCMS object HDF5 files with new raw file locations and any additional 2795 information produced during the processing of the collection (e.g. cluster mass feature associations). Default is True. 2796 2797 Notes 2798 ----- 2799 The HDF5 file structure includes: 2800 - Attributes: date_utc, lcms_objects_folder, missing_mass_features_searched, manifest 2801 - Groups: rt_alignments, cluster_assignments (if available) 2802 2803 Induced mass features are saved to the individual LCMS object HDF5 files 2804 within the .corems folder structure, not in the collection-level HDF5 file. 2805 2806 Examples 2807 -------- 2808 >>> exporter = LCMSCollectionExporter("my_collection", lcms_collection) 2809 >>> exporter.export_to_hdf5(overwrite=True) 2810 """ 2811 if overwrite: 2812 if self.out_file_path.with_suffix(".hdf5").exists(): 2813 self.out_file_path.with_suffix(".hdf5").unlink() 2814 2815 with h5py.File(self.out_file_path.with_suffix(".hdf5"), "a") as hdf_handle: 2816 # Add basic attributes to the HDF5 file, always overwrite these 2817 timenow = str(datetime.now(timezone.utc).strftime("%d/%m/%Y %H:%M:%S %Z")) 2818 hdf_handle.attrs["date_utc"] = timenow 2819 hdf_handle.attrs["lcms_objects_folder"] = str( 2820 self.mass_spectra_collection.collection_parser.folder_location 2821 ) 2822 hdf_handle.attrs["missing_mass_features_searched"] = ( 2823 self.mass_spectra_collection.missing_mass_features_searched 2824 ) 2825 hdf_handle.attrs["rt_aligned"] = self.mass_spectra_collection.rt_aligned 2826 hdf_handle.attrs["rt_alignment_attempted"] = ( 2827 self.mass_spectra_collection.rt_alignment_attempted 2828 ) 2829 2830 # Add the manifest to the HDF5 file, always overwrite this 2831 hdf_handle.attrs["manifest"] = self._convert_manifest_to_json() 2832 2833 # Save retention time alignments if they exist, only overwrite if specified 2834 self._save_rt_alignments_to_hdf5(hdf_handle, overwrite) 2835 2836 # Save cluster assignments if they exist, only overwrite if specified 2837 self._save_cluster_assignments_to_hdf5(hdf_handle, overwrite) 2838 2839 # Save consensus feature-group labels if present 2840 self._save_feature_group_labels_to_hdf5(hdf_handle, overwrite) 2841 2842 # Save new raw file locations to each LCMS object's HDF5 file if needed 2843 if ( 2844 hasattr(self.mass_spectra_collection, "raw_files_relocated") 2845 and self.mass_spectra_collection.raw_files_relocated 2846 ): 2847 self._update_raw_file_locations_in_hdf5() 2848 2849 # Save induced mass features to the collection with associations to each individual, only if lcms_collection.missing_mass_features_searched is True 2850 if self.mass_spectra_collection.missing_mass_features_searched: 2851 self._save_induced_mass_features_to_hdf5(overwrite) 2852 # Save EICs for induced mass features at collection level 2853 self._save_induced_eics_to_hdf5(overwrite) 2854 2855 # Build cluster mass feature map to know which features to update 2856 # This uses the same logic as process_consensus_features to determine loaded features 2857 cluster_mf_map = self._build_cluster_mf_map() 2858 2859 # Save updated mass features for each LCMS object 2860 # This implements selective update: only loaded features are updated, non-cluster features are preserved 2861 if update_lcms_objects: 2862 self._save_lcms_objects_to_hdf5(cluster_mf_map, overwrite) 2863 2864 # Save collection-level parameters as separate file 2865 if save_parameters: 2866 # Check if parameter_format is valid 2867 if parameter_format not in ["json", "toml"]: 2868 raise ValueError("parameter_format must be 'json' or 'toml'") 2869 2870 if parameter_format == "json": 2871 dump_lcms_collection_settings_json( 2872 filename=self.out_file_path.with_suffix(".json"), 2873 lcms_collection=self.mass_spectra_collection, 2874 ) 2875 elif parameter_format == "toml": 2876 dump_lcms_collection_settings_toml( 2877 filename=self.out_file_path.with_suffix(".toml"), 2878 lcms_collection=self.mass_spectra_collection, 2879 )
Export the LCMS collection to an HDF5 file.
This method saves the collection-level data to an HDF5 file, including:
- Basic metadata (date, folder location, gap-filling status)
- Sample manifest
- Retention time alignments (if available)
- Cluster assignments (if available)
- Induced mass features for each LCMS object (if gap-filling was performed)
Individual LCMS objects in the collection are not exported by this method. Use LCMSExport for exporting individual LCMS objects.
Parameters
- overwrite (bool, optional): If True, overwrites the output file if it already exists and replaces existing groups within the HDF5 file. If False, appends new data to existing file without overwriting existing groups. Default is False.
- save_parameters (bool, optional): If True, saves the collection-level parameters to a separate file in the specified format. Default is True.
- parameter_format (str, optional): The format for saving parameters, either "json" or "toml". Default is "toml".
- update_lcms_objects (bool, optional): If True, updates the individual LCMS object HDF5 files with new raw file locations and any additional information produced during the processing of the collection (e.g. cluster mass feature associations). Default is True.
Notes
The HDF5 file structure includes:
- Attributes: date_utc, lcms_objects_folder, missing_mass_features_searched, manifest
- Groups: rt_alignments, cluster_assignments (if available)
Induced mass features are saved to the individual LCMS object HDF5 files within the .corems folder structure, not in the collection-level HDF5 file.
Examples
>>> exporter = LCMSCollectionExporter("my_collection", lcms_collection)
>>> exporter.export_to_hdf5(overwrite=True)
def
export_to_parquet(self, overwrite=False, save_parameters=True, parameter_format='toml'):
2881 def export_to_parquet( 2882 self, overwrite=False, save_parameters=True, parameter_format="toml" 2883 ): 2884 """Export collection-level LC-MS datasets to Parquet files. 2885 2886 Writes next to ``out_file_path`` (same stem as the HDF5 export): 2887 2888 - ``{stem}_manifest.parquet`` — sample manifest 2889 - ``{stem}_cluster_assignments.parquet`` — cluster labels (if present) 2890 - ``{stem}_mass_features.parquet`` — collection mass-features table (if present) 2891 - ``{stem}_rt_alignments.parquet`` — RT alignment curves (if aligned) 2892 - parameters as JSON or TOML when ``save_parameters`` is True 2893 2894 Parameters 2895 ---------- 2896 overwrite : bool, optional 2897 Whether to overwrite existing Parquet files. Default is False. 2898 save_parameters : bool, optional 2899 Whether to save collection parameters. Default is True. 2900 parameter_format : str, optional 2901 Parameter file format, ``'json'`` or ``'toml'``. Default is ``'toml'``. 2902 2903 Notes 2904 ----- 2905 Requires ``pyarrow`` (or ``fastparquet``) for pandas Parquet I/O. 2906 This is a tabular companion to ``export_to_hdf5``; it does not replace 2907 per-sample HDF5 archives or gap-fill side effects on individual LCMS files. 2908 """ 2909 stem = self.out_file_path 2910 2911 def _write(df, suffix): 2912 if df is None or df.empty: 2913 return 2914 out_path = Path(f"{stem}_{suffix}").with_suffix(".parquet") 2915 if out_path.exists() and not overwrite: 2916 return 2917 LCMSExport._prepare_df_for_parquet(df).to_parquet(out_path, index=False) 2918 2919 # Manifest — prefer the tidy samples-as-rows dataframe 2920 manifest_df = getattr(self.mass_spectra_collection, "manifest_dataframe", None) 2921 if manifest_df is None or getattr(manifest_df, "empty", True): 2922 manifest = getattr( 2923 self.mass_spectra_collection, "manifest", None 2924 ) or getattr( 2925 getattr(self.mass_spectra_collection, "collection_parser", None), 2926 "manifest", 2927 None, 2928 ) 2929 if isinstance(manifest, DataFrame): 2930 manifest_df = manifest 2931 elif isinstance(manifest, dict) and manifest: 2932 try: 2933 # Collection manifests are sample -> attrs; transpose to rows 2934 manifest_df = pd.DataFrame(manifest).T 2935 except ValueError: 2936 manifest_df = DataFrame( 2937 [{"manifest_json": self._convert_manifest_to_json()}] 2938 ) 2939 else: 2940 manifest_df = None 2941 2942 if manifest_df is not None and not manifest_df.empty: 2943 _write(manifest_df.reset_index(drop=False), "manifest") 2944 2945 # Cluster assignments / mass features table 2946 mf_df = getattr(self.mass_spectra_collection, "mass_features_dataframe", None) 2947 if mf_df is not None and not mf_df.empty: 2948 _write(mf_df.reset_index(drop=False), "mass_features") 2949 if "cluster" in mf_df.columns: 2950 cluster_df = mf_df[["cluster"]].copy().reset_index(drop=False) 2951 _write(cluster_df, "cluster_assignments") 2952 2953 # RT alignments as long-format table 2954 if getattr(self.mass_spectra_collection, "rt_aligned", False) and getattr( 2955 self.mass_spectra_collection, "rt_alignments", None 2956 ): 2957 frames = [] 2958 for ( 2959 sample_idx, 2960 alignment_data, 2961 ) in self.mass_spectra_collection.rt_alignments.items(): 2962 arr = np.asarray(alignment_data) 2963 if arr.ndim == 1: 2964 frames.append( 2965 DataFrame( 2966 { 2967 "sample_idx": sample_idx, 2968 "point": np.arange(len(arr)), 2969 "value": arr, 2970 } 2971 ) 2972 ) 2973 elif arr.ndim == 2 and arr.shape[1] >= 2: 2974 frames.append( 2975 DataFrame( 2976 { 2977 "sample_idx": sample_idx, 2978 "rt_original": arr[:, 0], 2979 "rt_aligned": arr[:, 1], 2980 } 2981 ) 2982 ) 2983 else: 2984 frames.append( 2985 DataFrame( 2986 { 2987 "sample_idx": [sample_idx], 2988 "alignment_json": [json.dumps(arr.tolist())], 2989 } 2990 ) 2991 ) 2992 if frames: 2993 _write(pd.concat(frames, ignore_index=True), "rt_alignments") 2994 2995 if save_parameters: 2996 if parameter_format not in ["json", "toml"]: 2997 raise ValueError("parameter_format must be 'json' or 'toml'") 2998 if parameter_format == "json": 2999 dump_lcms_collection_settings_json( 3000 filename=self.out_file_path.with_suffix(".json"), 3001 lcms_collection=self.mass_spectra_collection, 3002 ) 3003 elif parameter_format == "toml": 3004 dump_lcms_collection_settings_toml( 3005 filename=self.out_file_path.with_suffix(".toml"), 3006 lcms_collection=self.mass_spectra_collection, 3007 )
Export collection-level LC-MS datasets to Parquet files.
Writes next to out_file_path (same stem as the HDF5 export):
{stem}_manifest.parquet— sample manifest{stem}_cluster_assignments.parquet— cluster labels (if present){stem}_mass_features.parquet— collection mass-features table (if present){stem}_rt_alignments.parquet— RT alignment curves (if aligned)- parameters as JSON or TOML when
save_parametersis True
Parameters
- overwrite (bool, optional): Whether to overwrite existing Parquet files. Default is False.
- save_parameters (bool, optional): Whether to save collection parameters. Default is True.
- parameter_format (str, optional):
Parameter file format,
'json'or'toml'. Default is'toml'.
Notes
Requires pyarrow (or fastparquet) for pandas Parquet I/O.
This is a tabular companion to export_to_hdf5; it does not replace
per-sample HDF5 archives or gap-fill side effects on individual LCMS files.