corems.molecular_id.search.database_interfaces

   1import os
   2import re
   3import warnings
   4from abc import ABC
   5from io import StringIO
   6from pathlib import Path
   7import sqlite3
   8
   9import numpy as np
  10import requests
  11import pandas as pd
  12from ms_entropy import FlashEntropySearch
  13
  14from corems.molecular_id.factory.EI_SQL import (
  15    EI_LowRes_SQLite,
  16    Metadatar,
  17    MetaboliteMetadata,
  18)
  19from corems.molecular_id.factory.lipid_molecular_metadata import LipidMetadata
  20from corems.mass_spectra.calc.lc_calc import find_closest
  21from corems.molecular_id.search import spectral_library_cache
  22
  23
  24def _resolve_fe_kwargs(settings=None, fe_kwargs=None) -> dict:
  25    """Merge SpectralSimilaritySearchSettings.as_fe_kwargs() with optional fe_kwargs overrides."""
  26    base = settings.as_fe_kwargs() if settings is not None else {}
  27    if fe_kwargs:
  28        base = {**base, **dict(fe_kwargs)}
  29    return base
  30
  31
  32class SpectralDatabaseInterface(ABC):
  33    """
  34    Base class that facilitates connection to spectral reference databases,
  35    such as EMSL's Metabolomics Reference Database (MetabRef).
  36
  37    """
  38
  39    def __init__(self, key=None):
  40        """
  41        Initialize instance.
  42
  43        Parameters
  44        ----------
  45        key : str
  46            Token key.
  47
  48        """
  49
  50        self.key = key
  51
  52    def set_token(self, path):
  53        """
  54        Set environment variable for MetabRef database token.
  55
  56        Parameters
  57        ----------
  58        path : str
  59            Path to token.
  60
  61        """
  62
  63        # Read token from file
  64        with open(path, "r", encoding="utf-8") as f:
  65            token = f.readline().strip()
  66
  67        # Set environment variable
  68        os.environ[self.key] = token
  69
  70    def get_token(self):
  71        """
  72        Get environment variable for database token.
  73
  74        Returns
  75        -------
  76        str
  77            Token string.
  78
  79        """
  80
  81        # Check for token
  82        if self.key not in os.environ:
  83            raise ValueError("Must set {} environment variable.".format(self.key))
  84
  85        # Get token from environment variables
  86        return os.environ.get(self.key)
  87
  88    def get_header(self):
  89        """
  90        Access stored database token and prepare as header.
  91
  92        Returns
  93        -------
  94        str
  95            Header string.
  96
  97        """
  98
  99        # Get token
 100        token = self.get_token()
 101
 102        # Pad header information
 103        header = {"Authorization": f"Bearer {token}", "Content-Type": "text/plain"}
 104
 105        return header
 106
 107    def get_query(self, url, use_header=True):
 108        """
 109        Request payload from URL according to `get` protocol.
 110
 111        Parameters
 112        ----------
 113        url : str
 114            URL for request.
 115        use_header: bool
 116            Whether or not the query should include the header
 117
 118        Returns
 119        -------
 120        dict
 121            Response as JSON.
 122
 123        """
 124
 125        # Query URL via `get`
 126        if use_header:
 127            response = requests.get(url, headers=self.get_header())
 128        else:
 129            response = requests.get(url)
 130
 131        # Check response
 132        response.raise_for_status()
 133
 134        # Return as JSON
 135        return response.json()
 136
 137    def post_query(self, url, variable, values, tolerance):
 138        """
 139        Request payload from URL according to `post` protocol.
 140
 141        Parameters
 142        ----------
 143        url : str
 144            URL for request.
 145        variable : str
 146            Variable to query.
 147        values : str
 148            Specific values of `variable` to query.
 149        tolerance : str
 150            Query tolerance relative to `values`.
 151
 152        Returns
 153        -------
 154        dict
 155            Response as JSON.
 156
 157        """
 158
 159        # Coerce to string
 160        if not isinstance(variable, str):
 161            variable = str(variable).replace(" ", "")
 162
 163        if not isinstance(values, str):
 164            values = str(values).replace(" ", "")
 165
 166        if not isinstance(tolerance, str):
 167            tolerance = str(tolerance).replace(" ", "")
 168
 169        # Query URL via `post`
 170        response = requests.post(
 171            os.path.join(url, variable, tolerance),
 172            data=values,
 173            headers=self.get_header(),
 174        )
 175
 176        # Check response
 177        response.raise_for_status()
 178
 179        # Return as JSON
 180        return response.json()
 181
 182    def _check_flash_entropy_kwargs(self, fe_kwargs):
 183        """
 184        Check FlashEntropy keyword arguments.
 185
 186        Parameters
 187        ----------
 188        fe_kwargs : dict
 189            Keyword arguments for FlashEntropy search.
 190
 191
 192        Raises
 193        ------
 194        ValueError
 195            If "min_ms2_difference_in_da" or "max_ms2_tolerance_in_da" are present in `fe_kwargs` and they
 196            are not equal.
 197
 198        """
 199        # If "min_ms2_difference_in_da" in fe_kwargs, check that "max_ms2_tolerance_in_da" is also present and that min_ms2_difference_in_da = 2xmax_ms2_tolerance_in_da
 200        if (
 201            "min_ms2_difference_in_da" in fe_kwargs
 202            or "max_ms2_tolerance_in_da" in fe_kwargs
 203        ):
 204            if (
 205                "min_ms2_difference_in_da" not in fe_kwargs
 206                or "max_ms2_tolerance_in_da" not in fe_kwargs
 207            ):
 208                raise ValueError(
 209                    "Both 'min_ms2_difference_in_da' and 'max_ms2_tolerance_in_da' must be specified."
 210                )
 211            if (
 212                fe_kwargs["min_ms2_difference_in_da"]
 213                != 2 * fe_kwargs["max_ms2_tolerance_in_da"]
 214            ):
 215                raise ValueError(
 216                    "The values of 'min_ms2_difference_in_da' must be exactly 2x 'max_ms2_tolerance_in_da'."
 217                )
 218
 219    def _get_format_func(self, format):
 220        """
 221        Obtain format function by key.
 222
 223        Returns
 224        -------
 225        func
 226            Formatting function.
 227        """
 228
 229        if format.lower() in self.format_map.keys():
 230            return self.format_map[format.lower()]
 231
 232        raise ValueError(("{} not a supported format.").format(format))
 233
 234    def _dict_to_dataclass(self, source_dict, data_class):
 235        """
 236        Convert dictionary to dataclass.
 237
 238        Notes
 239        -----
 240        This function will pull the attributes a dataclass and its parent class
 241        and convert the dictionary to a dataclass instance with the appropriate
 242        attributes.
 243
 244        Parameters
 245        ----------
 246        data_class : :obj:`~dataclasses.dataclass`
 247            Dataclass to convert to.
 248        source_dict : dict
 249            Dictionary object to convert to dataclass.
 250
 251        Returns
 252        -------
 253        :obj:`~dataclasses.dataclass`
 254            Dataclass instance.
 255
 256        """
 257
 258        # Get list of expected attributes of data_class
 259        data_class_keys = list(data_class.__annotations__.keys())
 260
 261        # Does the data_class inherit from another class, if so, get the attributes of the parent class as well
 262        if len(data_class.__mro__) > 2:
 263            parent_class_keys = list(data_class.__bases__[0].__annotations__.keys())
 264            data_class_keys = list(set(data_class_keys + parent_class_keys))
 265
 266        # Remove keys that are not in the data_class from the input dictionary
 267        input_dict = {k: v for k, v in source_dict.items() if k in data_class_keys}
 268
 269        # Add keys that are in the data class but not in the input dictionary as None
 270        for key in data_class_keys:
 271            if key not in input_dict.keys():
 272                input_dict[key] = None
 273        return data_class(**input_dict)
 274
 275    def _spectrum_to_array(self, spectrum, normalize=True):
 276        """
 277        Convert a parenthesis-delimited spectrum string to array.
 278
 279        Parameters
 280        ----------
 281        spectrum : str
 282            Spectrum string, i.e. list of (m/z,abundance) pairs.
 283        normalize : bool
 284            Normalize the spectrum by its magnitude.
 285
 286        Returns
 287        -------
 288        :obj:`~numpy.array`
 289            Array of shape (N, 2), with m/z in the first column and abundance in
 290            the second.
 291        """
 292
 293        arr = np.array(
 294            re.findall(r"\(([^,]+),([^)]+)\)", spectrum), dtype=float
 295        ).reshape(-1, 2)
 296
 297        if normalize:
 298            arr = self.normalize_peaks(arr)
 299
 300        return arr
 301
 302    @staticmethod
 303    def normalize_peaks(arr):
 304        """
 305        Normalize peaks in an array.
 306
 307        Parameters
 308        ----------
 309        arr : :obj:`~numpy.array`
 310            Array of shape (N, 2), with m/z in the first column and abundance in
 311            the second.
 312
 313        Returns
 314        -------
 315        :obj:`~numpy.array`
 316            Normalized array of shape (N, 2), with m/z in the first column and
 317            normalized abundance in the second.
 318        """
 319        # Normalize the array
 320        arr[:, -1] = arr[:, -1] / arr[:, -1].sum()
 321
 322        return arr
 323
 324    @staticmethod
 325    def _build_flash_entropy_index(fe_lib, fe_kwargs={}, clean_spectra=True):
 326        """
 327        Build FlashEntropy index.
 328
 329        Parameters
 330        ----------
 331        fe_lib : list
 332            List of spectra to build index from. Can be a list of dictionaries or
 333            a FlashEntropy search instance.
 334        fe_kwargs : dict, optional
 335            Keyword arguments for FlashEntropy search.
 336        clean_spectra : bool, optional
 337            Clean spectra before building index. Default is True.
 338
 339        Returns
 340        -------
 341        :obj:`~ms_entropy.FlashEntropySearch`
 342            FlashEntropy search instance.
 343
 344        """
 345        # Initialize FlashEntropy
 346        fe_init_kws = [
 347            "max_ms2_tolerance_in_da",
 348            "mz_index_step",
 349            "low_memory",
 350            "path_data",
 351        ]
 352        fe_init_kws = {k: v for k, v in fe_kwargs.items() if k in fe_init_kws}
 353        fes = FlashEntropySearch(**fe_init_kws)
 354
 355        # Build FlashEntropy index
 356        fe_index_kws = [
 357            "max_indexed_mz",
 358            "precursor_ions_removal_da",
 359            "noise_threshold",
 360            "min_ms2_difference_in_da",
 361            "max_peak_num",
 362        ]
 363        fe_index_kws = {k: v for k, v in fe_kwargs.items() if k in fe_index_kws}
 364        fes.build_index(fe_lib, **fe_index_kws, clean_spectra=clean_spectra)
 365
 366        # Store build_index parameters as custom attributes for later retrieval
 367        # (FlashEntropy doesn't store these natively)
 368        for key, value in fe_index_kws.items():
 369            setattr(fes, f"_build_{key}", value)
 370
 371        return fes
 372
 373
 374class MetabRefInterface(SpectralDatabaseInterface):
 375    """
 376    DEPRECATED interface retained for backward compatibility only.
 377    """
 378
 379    def __init__(self):
 380        """
 381        Initialize instance with deprecation warning.
 382
 383        """
 384
 385        super().__init__(key=None)
 386
 387        if self.__class__ is MetabRefInterface:
 388            warnings.warn(
 389                "MetabRefInterface is deprecated. Instantiate a concrete interface "
 390                "such as GCMSLibraryInterface or LCLipidLibraryInterface instead.",
 391                DeprecationWarning,
 392                stacklevel=2,
 393            )
 394
 395
 396class GCMSLibraryInterface(SpectralDatabaseInterface):
 397    """
 398    Interface to bundled GCMS spectral libraries in MSP format.
 399
 400    Loads GCMS compound library and FAMES calibration library from local MSP files.
 401    Default files are bundled with CoreMS, but can be overridden via environment variables.
 402    """
 403
 404    def __init__(self):
 405        """
 406        Initialize instance.
 407        """
 408        super().__init__(key=None)
 409
 410        # Local data file paths
 411        from pathlib import Path
 412
 413        # Default to bundled data files
 414        data_dir = Path(__file__).parent.parent / "data"
 415        self.gcms_library_file = os.getenv(
 416            "GCMS_LIBRARY_PATH", str(data_dir / "PNNLMetV20191015.msp")
 417        )
 418        self.fames_library_file = os.getenv(
 419            "FAMES_LIBRARY_PATH", str(data_dir / "FAMES_REF.msp")
 420        )
 421
 422        self.__init_format_map__()
 423
 424    def __init_format_map__(self):
 425        """
 426        Initialize database format mapper, enabling multiple format requests.
 427
 428        """
 429
 430        # Define format workflows
 431        self.format_map = {
 432            "json": lambda x, normalize, fe_kwargs: x,
 433            "dict": lambda x, normalize, fe_kwargs: (
 434                self._to_LowResolutionEICompound_dict(x, normalize)
 435            ),
 436            "sql": lambda x, normalize, fe_kwargs: (
 437                self._LowResolutionEICompound_dict_to_sqlite(
 438                    self._to_LowResolutionEICompound_dict(x, normalize)
 439                )
 440            ),
 441        }
 442
 443        # Add aliases
 444        self.format_map["metabref"] = self.format_map["json"]
 445        self.format_map["datadict"] = self.format_map["dict"]
 446        self.format_map["data-dict"] = self.format_map["dict"]
 447        self.format_map["lowreseicompound"] = self.format_map["dict"]
 448        self.format_map["lowres"] = self.format_map["dict"]
 449        self.format_map["lowresgc"] = self.format_map["dict"]
 450        self.format_map["sqlite"] = self.format_map["sql"]
 451
 452    def available_formats(self):
 453        """
 454        View list of available formats.
 455
 456        Returns
 457        -------
 458        list
 459            Format map keys.
 460        """
 461
 462        return list(self.format_map.keys())
 463
 464    def get_library(self, format="json", normalize=False):
 465        """
 466        Load GC/MS library from local MSP file.
 467
 468        Parameters
 469        ----------
 470        format : str
 471            Format of requested library, i.e. "json", "sql", "dict".
 472            See `available_formats` method for aliases.
 473        normalize : bool
 474            Normalize the spectrum by its magnitude.
 475
 476        Returns
 477        -------
 478        Library in requested format.
 479
 480        """
 481        # Load from local MSP file
 482        library_data = self._load_msp_file(self.gcms_library_file, normalize)
 483
 484        # Init format function
 485        format_func = self._get_format_func(format)
 486
 487        # Apply format conversion
 488        return format_func(library_data, normalize, {})
 489
 490    def get_fames(self, format="json", normalize=False):
 491        """
 492        Load GC/MS FAMEs library from local MSP file.
 493
 494        Parameters
 495        ----------
 496        format : str
 497            Format of requested library, i.e. "json", "sql", "dict".
 498            See `available_formats` method for aliases.
 499        normalize : bool
 500            Normalize the spectrum by its magnitude.
 501
 502        Returns
 503        -------
 504        Library in requested format.
 505
 506        """
 507        # Load from local MSP file
 508        library_data = self._load_msp_file(self.fames_library_file, normalize)
 509
 510        # Init format function
 511        format_func = self._get_format_func(format)
 512
 513        # Apply format conversion
 514        return format_func(library_data, normalize, {})
 515
 516    def _load_msp_file(self, file_path, normalize=False):
 517        """
 518        Load and parse MSP file into format compatible with existing pipeline.
 519
 520        Parameters
 521        ----------
 522        file_path : str
 523            Path to MSP file
 524        normalize : bool
 525            Normalize spectra
 526
 527        Returns
 528        -------
 529        list of dict
 530            Library data in format compatible with _to_LowResolutionEICompound_dict
 531        """
 532        from pathlib import Path
 533
 534        file_path = Path(file_path)
 535        if not file_path.exists():
 536            raise FileNotFoundError(
 537                f"Library file not found: {file_path}. "
 538                f"Set GCMS_LIBRARY_PATH or FAMES_LIBRARY_PATH environment variable to specify location."
 539            )
 540
 541        # Parse MSP file
 542        spectra = []
 543        spectrum = {}
 544        peaks = []
 545
 546        with open(file_path, "r") as f:
 547            for line in f:
 548                line = line.strip()
 549
 550                # Empty line marks end of spectrum
 551                if not line:
 552                    if spectrum and peaks:
 553                        # Convert peaks to the format expected by downstream code
 554                        # Format: "(mz,abundance)(mz,abundance)..."
 555                        peak_str = "".join(
 556                            [f"({int(mz)},{int(abun)})" for mz, abun in peaks]
 557                        )
 558                        spectrum["mz"] = peak_str
 559                        spectra.append(spectrum)
 560                    spectrum = {}
 561                    peaks = []
 562                    continue
 563
 564                # Check if line contains peak data (starts with digit)
 565                if line and line[0].isdigit():
 566                    parts = line.split()
 567                    if len(parts) >= 2:
 568                        peaks.append((float(parts[0]), float(parts[1])))
 569                    continue
 570
 571                # Handle metadata fields
 572                if ":" in line:
 573                    key, value = line.split(":", 1)
 574                    key = key.strip().lower()
 575                    value = value.strip()
 576
 577                    # Map MSP fields to expected format
 578                    field_mapping = {
 579                        "name": "molecule_name",
 580                        "formula": "formula",
 581                        "cas": "casno",
 582                        "retentiontime": "retention_time",
 583                        "ri": "ri",
 584                        "comment": "comments",
 585                        "num peaks": "peak_count",
 586                        "derivative": "derivative",
 587                    }
 588
 589                    # Metadata fields that go into the metadata dict
 590                    metadata_fields = {
 591                        "inchikey": "inchikey",
 592                        "inchi": "inchi",
 593                        "smiles": "smiles",
 594                        "pubchem": "pubchem",
 595                        "chebi": "chebi",
 596                        "kegg": "kegg",
 597                        "refmet": "refmet",
 598                        "iupac_name": "iupac_name",
 599                    }
 600
 601                    if key in field_mapping:
 602                        mapped_key = field_mapping[key]
 603                        # Convert numeric fields
 604                        if key in ["retentiontime", "ri"]:
 605                            try:
 606                                value = float(value)
 607                            except:
 608                                pass
 609                        elif key == "num peaks":
 610                            try:
 611                                value = int(value)
 612                            except:
 613                                pass
 614                        spectrum[mapped_key] = value
 615                    elif key in metadata_fields:
 616                        # Store in nested metadata dict
 617                        if "metadata" not in spectrum:
 618                            spectrum["metadata"] = {}
 619                        spectrum["metadata"][metadata_fields[key]] = value
 620                    else:
 621                        # Keep unmapped fields
 622                        spectrum[key] = value
 623
 624        # Add last spectrum if file doesn't end with blank line
 625        if spectrum and peaks:
 626            peak_str = "".join([f"({int(mz)},{int(abun)})" for mz, abun in peaks])
 627            spectrum["mz"] = peak_str
 628            spectra.append(spectrum)
 629
 630        return spectra
 631
 632    def _to_LowResolutionEICompound_dict(self, metabref_lib, normalize=False):
 633        """
 634        Convert MetabRef-formatted library to CoreMS LowResolutionEICompound-formatted
 635        dictionary for local ingestion.
 636
 637        Parameters
 638        ----------
 639        metabref_lib : dict
 640            MetabRef GC-MS library in JSON format.
 641        normalize : bool
 642            Normalize each spectrum by its magnitude.
 643
 644        Returns
 645        -------
 646        list of dict
 647            List of each spectrum contained in dictionary.
 648
 649        """
 650
 651        # All below key:value lookups are based on CoreMS class definitions
 652        # NOT MetabRef content. For example, MetabRef has keys for PubChem,
 653        # USI, etc. that are not considered below.
 654
 655        # Dictionary to map metabref keys to corems keys
 656        metadatar_cols = {
 657            "casno": "cas",
 658            "inchikey": "inchikey",
 659            "inchi": "inchi",
 660            "chebi": "chebi",
 661            "smiles": "smiles",
 662            "kegg": "kegg",
 663            "iupac_name": "iupac_name",
 664            "traditional_name": "traditional_name",  # Not present in metabref
 665            "common_name": "common_name",  # Not present in metabref
 666        }
 667
 668        # Dictionary to map metabref keys to corems keys
 669        lowres_ei_compound_cols = {
 670            "id": "metabref_id",
 671            "molecule_name": "name",  # Is this correct?
 672            "classify": "classify",  # Not present in metabref
 673            "formula": "formula",
 674            "ri": "ri",
 675            "rt": "retention_time",
 676            "source": "source",  # Not present in metabref
 677            "casno": "casno",
 678            "comments": "comment",
 679            "source_temp_c": "source_temp_c",  # Not present in metabref
 680            "ev": "ev",  # Not present in metabref
 681            "peak_count": "peaks_count",
 682            "mz": "mz",
 683            "abundance": "abundance",
 684        }
 685
 686        # Local result container
 687        corems_lib = []
 688
 689        # Enumerate spectra
 690        for i, source_ in enumerate(metabref_lib):
 691            # Copy source to prevent modification
 692            source = source_.copy()
 693
 694            # Parse target data
 695            target = {
 696                lowres_ei_compound_cols[k]: v
 697                for k, v in source.items()
 698                if k in lowres_ei_compound_cols
 699            }
 700
 701            # Explicitly add this to connect with LowResCompoundRef later
 702            if "retention_time" in source:
 703                target["rt"] = source["retention_time"]
 704            elif "rt" in source:
 705                target["rt"] = source["rt"]
 706
 707            # Parse (mz, abundance)
 708            arr = self._spectrum_to_array(target["mz"], normalize=normalize)
 709            target["mz"] = arr[:, 0]
 710            target["abundance"] = arr[:, 1]
 711
 712            # Parse meta data
 713            target["metadata"] = {
 714                metadatar_cols[k]: v for k, v in source.items() if k in metadatar_cols
 715            }
 716
 717            # Add anything else
 718            for k in source:
 719                if k not in lowres_ei_compound_cols:
 720                    target[k] = source[k]
 721
 722            # Add to CoreMS list
 723            corems_lib.append(target)
 724
 725        return corems_lib
 726
 727    def _LowResolutionEICompound_dict_to_sqlite(
 728        self, lowres_ei_compound_dict, url="sqlite://"
 729    ):
 730        """
 731        Convert CoreMS LowResolutionEICompound-formatted dictionary to SQLite
 732        database for local ingestion.
 733
 734        Parameters
 735        ----------
 736        lowres_ei_compound_dict : dict
 737            CoreMS GC-MS library formatted for LowResolutionEICompound.
 738        url : str
 739            URL to SQLite prefix.
 740
 741        Returns
 742        -------
 743        sqlite database
 744            Spectra contained in SQLite database.
 745
 746        """
 747
 748        # Dictionary to map corems keys to all-caps keys
 749        capped_cols = {
 750            "name": "NAME",
 751            "formula": "FORM",
 752            "ri": "RI",
 753            "retention_time": "RT",
 754            "source": "SOURCE",
 755            "casno": "CASNO",
 756            "comment": "COMMENT",
 757            "peaks_count": "NUM PEAKS",
 758        }
 759
 760        # Initialize SQLite object
 761        sqlite_obj = EI_LowRes_SQLite(url=url)
 762
 763        # Iterate spectra
 764        for _data_dict in lowres_ei_compound_dict:
 765            # Copy source to prevent modification
 766            data_dict = _data_dict.copy()
 767
 768            # Add missing capped values
 769            for k, v in capped_cols.items():
 770                # Key exists
 771                if k in data_dict:
 772                    # # This will replace the key
 773                    # data_dict[v] = data_dict.pop(k)
 774
 775                    # This will keep both keys
 776                    data_dict[v] = data_dict[k]
 777
 778            # Parse number of peaks
 779            if not data_dict.get("NUM PEAKS"):
 780                data_dict["NUM PEAKS"] = len(data_dict.get("mz"))
 781
 782            # Parse CAS number
 783            if not data_dict.get("CASNO"):
 784                data_dict["CASNO"] = data_dict.get("CAS")
 785
 786            if not data_dict["CASNO"]:
 787                data_dict["CASNO"] = 0
 788
 789            # Build linked metadata table
 790            if "metadata" in data_dict:
 791                metadata = data_dict.pop("metadata")
 792                # Only create metadata entry if we have required fields and valid data
 793                # Filter to only include fields that Metadatar model supports
 794                supported_metadata_fields = [
 795                    "cas",
 796                    "inchikey",
 797                    "inchi",
 798                    "chebi",
 799                    "smiles",
 800                    "kegg",
 801                    "iupac_name",
 802                    "traditional_name",
 803                    "common_name",
 804                ]
 805                filtered_metadata = {
 806                    k: v
 807                    for k, v in metadata.items()
 808                    if k in supported_metadata_fields and v
 809                }
 810                # Inchikey is required by the database model
 811                if filtered_metadata and filtered_metadata.get("inchikey"):
 812                    data_dict["metadatar"] = Metadatar(**filtered_metadata)
 813
 814            # Attempt addition to sqlite
 815            try:
 816                sqlite_obj.add_compound(data_dict)
 817            except:
 818                print(data_dict["NAME"])
 819
 820        return sqlite_obj
 821
 822
 823class MetabRefGCInterface(GCMSLibraryInterface):
 824    """
 825    DEPRECATED: Use GCMSLibraryInterface instead.
 826
 827    This interface is maintained for backward compatibility only.
 828    MetabRef API has been discontinued as of 2026.
 829    """
 830
 831    def __init__(self):
 832        """
 833        Initialize instance with deprecation warning.
 834        """
 835        warnings.warn(
 836            "MetabRefGCInterface is deprecated. Use GCMSLibraryInterface instead. "
 837            "MetabRef API has been discontinued; all data now loads from bundled local MSP files.",
 838            DeprecationWarning,
 839            stacklevel=2,
 840        )
 841        super().__init__()
 842
 843
 844class LCLipidLibraryInterface(SpectralDatabaseInterface):
 845    """
 846    Interface to a local sqlite lipid library for LC-MS spectral searches.
 847    """
 848
 849    DEFAULT_DOWNLOAD_URL = (
 850        "https://nmdcdemo.emsl.pnnl.gov/minio/lipidomics/parameter_files/"
 851        "202412_lipid_ref.sqlite"
 852    )
 853
 854    def __init__(self, db_location=None):
 855        """
 856        Initialize instance.
 857
 858        Parameters
 859        ----------
 860        db_location : str | Path, optional
 861            Local path to the sqlite lipid library. If omitted, the
 862            COREMS_LIPIDOMICS_SQLITE_PATH environment variable is used.
 863        """
 864
 865        super().__init__(key=None)
 866        self.db_location = db_location
 867        self.__init_format_map__()
 868
 869    def _to_flashentropy(self, spectral_library, normalize=True, fe_kwargs={}):
 870        """
 871        Convert a spectral library to FlashEntropy format.
 872
 873        Parameters
 874        ----------
 875        spectral_library : dict
 876            MS2 library in JSON format or FlashEntropy search instance
 877            (for reformatting at different MS2 separation).
 878        normalize : bool
 879            Normalize each spectrum by its magnitude.
 880        fe_kwargs : dict, optional
 881            Keyword arguments for instantiation of FlashEntropy search and building index for FlashEntropy search;
 882            any keys not recognized will be ignored. By default, all parameters set to defaults.
 883
 884        Returns
 885        -------
 886        :obj:`~ms_entropy.FlashEntropySearch`
 887            MS2 library as FlashEntropy search instance.
 888
 889        Raises
 890        ------
 891        ValueError
 892            If "min_ms2_difference_in_da" or "max_ms2_tolerance_in_da" are present in `fe_kwargs` and they are not equal.
 893
 894        """
 895        self._check_flash_entropy_kwargs(fe_kwargs)
 896
 897        # Initialize empty library
 898        fe_lib = []
 899
 900        # Enumerate spectra
 901        for i, source in enumerate(spectral_library):
 902            if "spectrum_data" in source.keys():
 903                spectrum = source["spectrum_data"]
 904            else:
 905                spectrum = source
 906
 907            if "precursor_mz" not in spectrum.keys():
 908                spectrum["precursor_mz"] = spectrum.pop("precursor_ion")
 909
 910            spectrum["peaks"] = self._spectrum_to_array(
 911                spectrum["mz"], normalize=normalize
 912            )
 913            fe_lib.append(spectrum)
 914
 915        fe_search = self._build_flash_entropy_index(fe_lib, fe_kwargs=fe_kwargs)
 916
 917        return fe_search
 918
 919    def __init_format_map__(self):
 920        """
 921        Initialize database format mapper, enabling multiple format requests.
 922        """
 923
 924        self.format_map = {
 925            "json": lambda x, normalize, fe_kwargs: x,
 926            "flashentropy": lambda x, normalize, fe_kwargs: self._to_flashentropy(
 927                x, normalize, fe_kwargs
 928            ),
 929            "dataframe": lambda x, normalize, fe_kwargs: pd.DataFrame(x),
 930        }
 931
 932        self.format_map["fe"] = self.format_map["flashentropy"]
 933        self.format_map["flash-entropy"] = self.format_map["flashentropy"]
 934        self.format_map["df"] = self.format_map["dataframe"]
 935
 936    def available_formats(self):
 937        """
 938        View list of available formats.
 939
 940        Returns
 941        -------
 942        list
 943            Format map keys.
 944        """
 945
 946        return list(self.format_map.keys())
 947
 948    def _resolve_db_location(self):
 949        """
 950        Resolve and validate sqlite database location.
 951
 952        Returns
 953        -------
 954        Path
 955            Existing sqlite database file path.
 956        """
 957
 958        db_location = self.db_location or os.getenv("COREMS_LIPIDOMICS_SQLITE_PATH")
 959        if not db_location:
 960            raise ValueError(
 961                "A local lipid sqlite library path is required. "
 962                "Set COREMS_LIPIDOMICS_SQLITE_PATH or pass db_location."
 963            )
 964
 965        db_path = Path(db_location).expanduser()
 966        if not db_path.exists():
 967            raise FileNotFoundError(
 968                f"Lipid sqlite library not found at {db_path}. "
 969                f"Download it from {self.DEFAULT_DOWNLOAD_URL} "
 970                "and set COREMS_LIPIDOMICS_SQLITE_PATH."
 971            )
 972
 973        return db_path
 974
 975    def _get_candidate_spectra(self, connection, mz_list, polarity, mz_tol_ppm):
 976        """
 977        Fetch candidate spectra rows by precursor m/z and polarity.
 978
 979        Returns
 980        -------
 981        pandas.DataFrame
 982            Filtered rows from lipidMassSpectrumObject.
 983        """
 984
 985        mz_observed = np.sort(np.asarray(mz_list, dtype=float))
 986        if mz_observed.size == 0:
 987            return pd.DataFrame()
 988
 989        mz_all = pd.read_sql_query(
 990            "SELECT id, polarity, precursor_mz FROM lipidMassSpectrumObject", connection
 991        )
 992        mz_all = mz_all[mz_all["polarity"] == polarity].copy()
 993        if mz_all.empty:
 994            return pd.DataFrame()
 995
 996        mz_all = mz_all.sort_values(by="precursor_mz").reset_index(drop=True)
 997
 998        if mz_observed.size == 1:
 999            mz_all["closest_mz_obs"] = mz_observed[0]
1000        else:
1001            mz_all["closest_mz_obs"] = mz_observed[
1002                find_closest(mz_observed, mz_all.precursor_mz.values)
1003            ]
1004
1005        mz_all["ppm_error"] = (
1006            (mz_all["precursor_mz"] - mz_all["closest_mz_obs"])
1007            / mz_all["precursor_mz"]
1008            * 1e6
1009        )
1010
1011        mz_all = mz_all[np.abs(mz_all["ppm_error"]) <= mz_tol_ppm]
1012        if mz_all.empty:
1013            return pd.DataFrame()
1014
1015        mz_ids = tuple(mz_all["id"].tolist())
1016        return pd.read_sql_query(
1017            f"SELECT * FROM lipidMassSpectrumObject WHERE id IN {mz_ids}",
1018            connection,
1019        )
1020
1021    def get_lipid_library(
1022        self,
1023        mz_list,
1024        polarity,
1025        mz_tol_ppm,
1026        mz_tol_da_api=None,
1027        format="json",
1028        normalize=True,
1029        fe_kwargs=None,
1030        settings=None,
1031        api_delay=5,
1032        api_attempts=10,
1033    ):
1034        """
1035        Retrieve lipid spectra and metadata from a local sqlite library.
1036
1037        FlashEntropy build knobs should come from
1038        :class:`~corems.encapsulation.factory.processingSetting.SpectralSimilaritySearchSettings`
1039        so library generation matches later
1040        :meth:`~corems.molecular_id.search.lcms_spectral_search.LCMSSpectralSearch.fe_search`
1041        / molecular networking on the same LC-MS sample or collection.
1042
1043        Parameters
1044        ----------
1045        mz_list : list
1046            List of precursor m/z values.
1047        polarity : str
1048            Ionization polarity, either "positive" or "negative".
1049        mz_tol_ppm : float
1050            Tolerance in ppm for precursor matching.
1051        mz_tol_da_api : float, optional
1052            Unused, kept for backward compatibility.
1053        format : str, optional
1054            Format of requested library, e.g. "json" or "flashentropy".
1055        normalize : bool, optional
1056            Normalize spectrum intensities (default True; always recommended).
1057        fe_kwargs : dict, optional
1058            Keyword arguments for FlashEntropy search. Override keys from
1059            *settings* when both are provided. Prefer *settings* for new code.
1060        settings : SpectralSimilaritySearchSettings, optional
1061            Encapsulated FE build settings. For an :class:`~corems.mass_spectra.factory.lc_class.LCMSBase`
1062            object, pass the profile bag used for MS2 association/search, typically::
1063
1064                from corems.encapsulation.factory.parameters import settings_from_lcms
1065                settings=settings_from_lcms(lcms_obj)  # mass_spectrum["ms2"]
1066
1067            or explicitly
1068            ``lcms_obj.parameters.mass_spectrum["ms2"].spectral_similarity_search``.
1069            For an :class:`~corems.mass_spectra.factory.lc_class.LCMSCollection`,
1070            configure settings once, broadcast to every sample, then pass
1071            collection settings into the library build::
1072
1073                from corems.encapsulation.factory.parameters import (
1074                    apply_spectral_similarity_search_to_collection,
1075                    settings_from_lcms_collection,
1076                )
1077                from corems.encapsulation.factory.processingSetting import (
1078                    SpectralSimilaritySearchSettings,
1079                )
1080
1081                ss = SpectralSimilaritySearchSettings()
1082                ss.max_ms2_tolerance_in_da = 0.01
1083                ss.ms2_min_fe_score = 0.3
1084                apply_spectral_similarity_search_to_collection(
1085                    lcms_collection, ss, profile="ms2"
1086                )
1087                settings=settings_from_lcms_collection(lcms_collection)
1088
1089            Mutating ``settings_from_lcms_collection(...)`` in place only changes
1090            the first sample. Multi-MS2 profiles use ``profile="ms2_cid"`` (etc.).
1091        api_delay : int, optional
1092            Unused, kept for backward compatibility.
1093        api_attempts : int, optional
1094            Unused, kept for backward compatibility.
1095
1096        Returns
1097        -------
1098        tuple
1099            Library in requested format and lipid metadata dictionary.
1100
1101        Notes
1102        -----
1103        *settings* only drives FlashEntropy index construction. Annotation
1104        thresholds such as ``ms2_min_fe_score`` are applied later by
1105        ``fe_search`` on the LCMS object (from the same
1106        ``spectral_similarity_search`` instance when configured consistently).
1107
1108        Examples
1109        --------
1110        Build a FlashEntropy library from an LCMSBase object's MS2 profile, then search::
1111
1112            from corems.encapsulation.factory.parameters import settings_from_lcms
1113
1114            settings = settings_from_lcms(lcms_obj, profile="ms2")
1115            fe_lib, lipid_meta = lipid_interface.get_lipid_library(
1116                mz_list=precursor_mzs,
1117                polarity=lcms_obj.polarity,
1118                mz_tol_ppm=5,
1119                format="flashentropy",
1120                settings=settings,
1121            )
1122            lcms_obj.fe_search(scan_list=ms2_scans, fe_lib=fe_lib)
1123
1124        Collection — configure, broadcast to all samples, then build library::
1125
1126            from corems.encapsulation.factory.parameters import (
1127                apply_spectral_similarity_search_to_collection,
1128                settings_from_lcms_collection,
1129            )
1130            from corems.encapsulation.factory.processingSetting import (
1131                SpectralSimilaritySearchSettings,
1132            )
1133
1134            ss = SpectralSimilaritySearchSettings()
1135            ss.max_ms2_tolerance_in_da = 0.01
1136            apply_spectral_similarity_search_to_collection(lcms_collection, ss)
1137            fe_lib, lipid_meta = lipid_interface.get_lipid_library(
1138                mz_list=precursor_mzs,
1139                polarity="positive",
1140                mz_tol_ppm=5,
1141                format="flashentropy",
1142                settings=settings_from_lcms_collection(lcms_collection),
1143            )
1144        """
1145        fe_kwargs = _resolve_fe_kwargs(settings=settings, fe_kwargs=fe_kwargs)
1146
1147        if not isinstance(mz_list, (list, np.ndarray)):
1148            raise ValueError("mz_list must be a list or numpy array")
1149        if not all(isinstance(mz, (float, int)) for mz in mz_list):
1150            raise ValueError("All elements in mz_list must be float or int")
1151        if polarity not in {"positive", "negative"}:
1152            raise ValueError("polarity must be either 'positive' or 'negative'")
1153        if not isinstance(mz_tol_ppm, (float, int)):
1154            raise ValueError("mz_tol_ppm must be a float or int")
1155
1156        db_path = self._resolve_db_location()
1157        connection = sqlite3.connect(str(db_path))
1158        try:
1159            # Step 1: Get candidate spectra records based on m/z and polarity
1160            spectra_df = self._get_candidate_spectra(
1161                connection=connection,
1162                mz_list=mz_list,
1163                polarity=polarity,
1164                mz_tol_ppm=float(mz_tol_ppm),
1165            )
1166
1167            if spectra_df.empty:
1168                format_func = self._get_format_func(format)
1169                return format_func([], normalize=normalize, fe_kwargs=fe_kwargs), {}
1170
1171            # Step 2: Get corresponding lipid metadata for candidate spectra from lipidTree view
1172            mol_ids = tuple(spectra_df["molecular_data_id"].tolist())
1173            mol_df = pd.read_sql_query(
1174                f"SELECT * FROM lipidTree WHERE id IN {mol_ids}",
1175                connection,
1176            )
1177        finally:
1178            connection.close()
1179
1180        mol_df["id_index"] = mol_df["id"]
1181        mol_df = mol_df.set_index("id_index")
1182        mol_records = mol_df.to_dict(orient="index")
1183        lipid_metadata = {
1184            int(k): self._dict_to_dataclass(v, LipidMetadata)
1185            for k, v in mol_records.items()
1186        }
1187
1188        spectra_records = spectra_df.to_dict(orient="records")
1189        format_func = self._get_format_func(format)
1190        library = format_func(spectra_records, normalize=normalize, fe_kwargs=fe_kwargs)
1191        return library, lipid_metadata
1192
1193
1194class MSPInterface(SpectralDatabaseInterface):
1195    """
1196    Interface to parse NIST MSP files
1197    """
1198
1199    def __init__(self, file_path, cache=True, cache_path=None, rebuild_cache=False):
1200        """
1201        Initialize instance.
1202
1203        Parameters
1204        ----------
1205        file_path : str
1206            Path to a local MSP file.
1207        cache : bool or str, optional
1208            If True, use a parquet sidecar when valid, otherwise parse and write
1209            it. If False, always parse. If ``"read"``, require a valid cache.
1210            Default is True. Sidecar path defaults to
1211            ``<file_path>.corems-lib.parquet``.
1212        cache_path : str or Path, optional
1213            Override the default cache path.
1214        rebuild_cache : bool, optional
1215            Force re-parse and rewrite the cache. Default is False.
1216
1217        Attributes
1218        ----------
1219        file_path : str
1220            Path to the MSP file.
1221        _file_content : str
1222            Content of the MSP file (only set when parsing text).
1223        _data_frame : :obj:`~pandas.DataFrame`
1224            DataFrame of spectra from the MSP file with unaltered content.
1225        """
1226        super().__init__(key=None)
1227
1228        self.file_path = file_path
1229        if not os.path.exists(self.file_path):
1230            raise FileNotFoundError(
1231                f"File {self.file_path} does not exist. Please check the file path."
1232            )
1233        self._file_content = None
1234        self._cache_path = (
1235            Path(cache_path)
1236            if cache_path is not None
1237            else spectral_library_cache.default_cache_path(self.file_path)
1238        )
1239        self._data_frame = self._load_msp_dataframe(
1240            cache=cache, rebuild_cache=rebuild_cache
1241        )
1242        self.__init_format_map__()
1243
1244    def _load_msp_dataframe(self, cache=True, rebuild_cache=False):
1245        """Load from parquet cache or parse the MSP file."""
1246        use_cache = cache not in (False, None)
1247        require_cache = cache == "read"
1248        extra = {"source_kind": "msp"}
1249
1250        if (
1251            use_cache
1252            and not rebuild_cache
1253            and spectral_library_cache.is_valid(
1254                self._cache_path, self.file_path, extra_meta=extra
1255            )
1256        ):
1257            return spectral_library_cache.read(self._cache_path)
1258
1259        if require_cache:
1260            raise FileNotFoundError(
1261                f"Valid library cache required at {self._cache_path}"
1262            )
1263
1264        with open(self.file_path, "r") as f:
1265            self._file_content = f.read()
1266        df = self._read_msp_file()
1267        if use_cache:
1268            spectral_library_cache.write(
1269                self._cache_path, df, self.file_path, extra_meta=extra
1270            )
1271        return df
1272
1273    def __init_format_map__(self):
1274        """
1275        Initialize database format mapper, enabling multiple format requests.
1276
1277        """
1278
1279        # x is a pandas dataframe similar to self._data_frame format
1280        # Define format workflows
1281        self.format_map = {
1282            "msp": lambda x, normalize, fe_kwargs: self._to_msp(x, normalize),
1283            "flashentropy": lambda x, normalize, fe_kwargs: self._to_flashentropy(
1284                x, normalize, fe_kwargs
1285            ),
1286            "df": lambda x, normalize, fe_kwargs: self._to_df(x, normalize),
1287        }
1288
1289        # Add aliases
1290        self.format_map["fe"] = self.format_map["flashentropy"]
1291        self.format_map["flash-entropy"] = self.format_map["flashentropy"]
1292        self.format_map["dataframe"] = self.format_map["df"]
1293        self.format_map["data-frame"] = self.format_map["df"]
1294
1295    def _read_msp_file(self):
1296        """
1297        Reads the MSP files into the pandas dataframe, and sort/remove zero intensity ions in MS/MS spectra.
1298
1299        Returns
1300        -------
1301        :obj:`~pandas.DataFrame`
1302            DataFrame of spectra from the MSP file, exacly as it is in the file (no sorting, filtering etc)
1303        """
1304        # If input_dataframe is provided, return it it
1305        spectra = []
1306        spectrum = {}
1307
1308        f = StringIO(self._file_content)
1309        for line in f:
1310            line = line.strip()
1311            if not line:
1312                continue  # Skip empty lines
1313
1314            # Handle metadata
1315            if ":" in line:
1316                key, value = line.split(":", 1)
1317                key = key.strip().lower()
1318                value = value.strip()
1319
1320                if key == "name":
1321                    # Save current spectrum and start a new one
1322                    if spectrum:
1323                        spectra.append(spectrum)
1324                    spectrum = {"name": value, "peaks": []}
1325                else:
1326                    spectrum[key] = value
1327
1328            # Handle peak data (assumed to start with a number)
1329            elif line[0].isdigit():
1330                peaks = line.split()
1331                m_z = float(peaks[0])
1332                intensity = float(peaks[1])
1333                spectrum["peaks"].append(([m_z, intensity]))
1334        # Save the last spectrum
1335        if spectrum:
1336            spectra.append(spectrum)
1337
1338        df = pd.DataFrame(spectra)
1339        for column in df.columns:
1340            if column != "peaks":  # Skip 'peaks' column
1341                try:
1342                    df[column] = pd.to_numeric(df[column], errors="raise")
1343                except:
1344                    pass
1345
1346        # Standardize spectra ID column name
1347        # Check for common variations and create a standard 'spectra_id' column
1348        spectra_id_variants = ["spectrum_id", "gnps_spectra_id"]
1349        for variant in spectra_id_variants:
1350            if variant in df.columns and "spectra_id" not in df.columns:
1351                df["spectra_id"] = df[variant]
1352                break
1353
1354        # If no spectra_id column exists after checking variants, create one with sequential IDs
1355        if "spectra_id" not in df.columns:
1356            df["spectra_id"] = [f"spectrum_{i:06d}" for i in range(len(df))]
1357
1358        # Standardize compound name column
1359        # Ensure 'compound_name' column exists, using 'name' field
1360        if "name" in df.columns and "compound_name" not in df.columns:
1361            df["compound_name"] = df["name"]
1362        elif "compound_name" not in df.columns:
1363            warnings.warn(
1364                "MSP file does not contain 'name' or 'compound_name' field. "
1365                "Compound names will be set to 'Unknown'. This may affect plot labels and annotations.",
1366                UserWarning,
1367            )
1368            df["compound_name"] = "Unknown"
1369
1370        return df
1371
1372    def _to_df(self, input_dataframe, normalize=True):
1373        """
1374        Convert MSP-derived library to FlashEntropy library.
1375
1376        Parameters
1377        ----------
1378        input_dataframe : :obj:`~pandas.DataFrame`
1379            Input DataFrame containing MSP-formatted spectra.
1380        normalize : bool, optional
1381            Normalize each spectrum by its magnitude.
1382            Default is True.
1383
1384        Returns
1385        -------
1386        :obj:`~pandas.DataFrame`
1387            DataFrame of with desired normalization
1388        """
1389        if not normalize:
1390            return input_dataframe
1391        else:
1392            # Convert to dictionary
1393            db_dict = input_dataframe.to_dict(orient="records")
1394
1395            # Initialize empty library
1396            lib = []
1397
1398            # Enumerate spectra
1399            for i, source in enumerate(db_dict):
1400                spectrum = source
1401                # Check that spectrum["peaks"] exists
1402                if "peaks" not in spectrum.keys():
1403                    raise KeyError(
1404                        "MSP not interpretted correctly, 'peaks' key not found in spectrum, check _dataframe attribute."
1405                    )
1406
1407                # Convert spectrum["peaks"] to numpy array
1408                if not isinstance(spectrum["peaks"], np.ndarray):
1409                    spectrum["peaks"] = np.array(spectrum["peaks"])
1410
1411                # Normalize peaks, if requested
1412                if normalize:
1413                    spectrum["peaks"] = self.normalize_peaks(spectrum["peaks"])
1414                    spectrum["num peaks"] = len(spectrum["peaks"])
1415
1416                # Add spectrum to library
1417                lib.append(spectrum)
1418
1419            # Convert to DataFrame
1420            df = pd.DataFrame(lib)
1421            return df
1422
1423    def _to_flashentropy(self, input_dataframe, normalize=True, fe_kwargs={}):
1424        """
1425        Convert MSP-derived library to FlashEntropy library.
1426
1427        Parameters
1428        ----------
1429        input_dataframe : :obj:`~pandas.DataFrame`
1430            Input DataFrame containing MSP-formatted spectra.
1431        normalize : bool
1432            Normalize each spectrum by its magnitude.
1433        fe_kwargs : dict, optional
1434            Keyword arguments for instantiation of FlashEntropy search and building index for FlashEntropy search;
1435            any keys not recognized will be ignored. By default, all parameters set to defaults.
1436
1437        Returns
1438        -------
1439        :obj:`~ms_entropy.FlashEntropySearch`
1440            MS2 library as FlashEntropy search instance.
1441
1442        Raises
1443        ------
1444        ValueError
1445            If "min_ms2_difference_in_da" or "max_ms2_tolerance_in_da" are present in `fe_kwargs` and they
1446        """
1447        self._check_flash_entropy_kwargs(fe_kwargs)
1448
1449        db_df = input_dataframe
1450
1451        # Convert to dictionary
1452        db_dict = db_df.to_dict(orient="records")
1453
1454        # Initialize empty library
1455        fe_lib = []
1456
1457        # Enumerate spectra
1458        for i, source in enumerate(db_dict):
1459            # Reorganize source dict, if necessary
1460            if "spectrum_data" in source.keys():
1461                spectrum = source["spectrum_data"]
1462            else:
1463                spectrum = source
1464
1465            # Rename precursor_mz key for FlashEntropy
1466            if "precursor_mz" not in spectrum.keys():
1467                if "precursormz" in spectrum:
1468                    spectrum["precursor_mz"] = spectrum.pop("precursormz")
1469                elif "precursor_ion" in spectrum:
1470                    spectrum["precursor_mz"] = spectrum.pop("precursor_ion")
1471                else:
1472                    raise KeyError(
1473                        "MSP must have either 'precursormz' or 'precursor_ion' key to be converted to FlashEntropy format."
1474                    )
1475
1476            # Check that spectrum["peaks"] exists
1477            if "peaks" not in spectrum.keys():
1478                raise KeyError(
1479                    "MSP not interpretted correctly, 'peaks' key not found in spectrum, check _dataframe attribute."
1480                )
1481
1482            # Convert spectrum["peaks"] to numpy array
1483            if not isinstance(spectrum["peaks"], np.ndarray):
1484                spectrum["peaks"] = np.array(spectrum["peaks"])
1485
1486            # Normalize peaks, if requested
1487            if normalize:
1488                spectrum["peaks"] = self.normalize_peaks(spectrum["peaks"])
1489
1490            # Add spectrum to library
1491            fe_lib.append(spectrum)
1492
1493        # Build FlashEntropy index
1494        fe_search = self._build_flash_entropy_index(fe_lib, fe_kwargs=fe_kwargs)
1495
1496        return fe_search
1497
1498    def _check_msp_compatibility(self):
1499        """
1500        Check if the MSP file is compatible with the get_metabolomics_spectra_library method and provide feedback if it is not.
1501        """
1502        # Check polarity
1503        if (
1504            "polarity" not in self._data_frame.columns
1505            and "ionmode" not in self._data_frame.columns
1506        ):
1507            raise ValueError(
1508                "Neither 'polarity' nor 'ionmode' columns found in the input MSP metadata. Please check the file."
1509            )
1510        polarity_column = (
1511            "polarity" if "polarity" in self._data_frame.columns else "ionmode"
1512        )
1513
1514        # Check if polarity_column contents is either "positive" or "negative"
1515        if not all(self._data_frame[polarity_column].isin(["positive", "negative"])):
1516            raise ValueError(
1517                f"Input field on MSP '{polarity_column}' must contain only 'positive' or 'negative' values."
1518            )
1519
1520        # Check if the MSP file contains the required columns for metabolite metadata
1521        # either formula or molecular_formula, not null
1522
1523        if (
1524            "formula" not in self._data_frame.columns
1525            and "molecular_formula" not in self._data_frame.columns
1526        ):
1527            raise ValueError(
1528                "Input field on MSP must contain either 'formula' or 'molecular_formula' columns."
1529            )
1530        molecular_formula_column = (
1531            "formula" if "formula" in self._data_frame.columns else "molecular_formula"
1532        )
1533        if not all(self._data_frame[molecular_formula_column].notnull()):
1534            raise ValueError(
1535                f"Input field on MSP '{molecular_formula_column}' must contain only non-null values."
1536            )
1537
1538    def get_metabolomics_spectra_library(
1539        self,
1540        polarity,
1541        metabolite_metadata_mapping={},
1542        format="fe",
1543        normalize=True,
1544        fe_kwargs=None,
1545        settings=None,
1546        molecular_id_field="inchikey",
1547    ):
1548        """
1549        Prepare metabolomics spectra library and associated metabolite metadata.
1550
1551        FlashEntropy build knobs should come from
1552        :class:`~corems.encapsulation.factory.processingSetting.SpectralSimilaritySearchSettings`
1553        so library generation matches later
1554        :meth:`~corems.molecular_id.search.lcms_spectral_search.LCMSSpectralSearch.fe_search`
1555        / molecular networking on the same LC-MS sample or collection.
1556
1557        Parameters
1558        ----------
1559        polarity : str
1560            Polarity of the spectra to extract. Must be 'positive' or 'negative'.
1561        metabolite_metadata_mapping : dict, optional
1562            Mapping of MSP field names to MetaboliteMetadata attribute names.
1563            Default uses common mappings (e.g., 'molecular_formula' -> 'formula').
1564        format : str, optional
1565            Output format for the spectral library. Options: 'fe', 'flashentropy', 'msp', 'df'.
1566            Default is 'fe' (FlashEntropy).
1567        normalize : bool, optional
1568            Whether to normalize spectra. Default is True (always recommended).
1569        fe_kwargs : dict, optional
1570            Additional keyword arguments for FlashEntropy library creation.
1571            Override keys from *settings* when both are provided. Prefer *settings*
1572            for new code.
1573        settings : SpectralSimilaritySearchSettings, optional
1574            Encapsulated FE build settings. For an
1575            :class:`~corems.mass_spectra.factory.lc_class.LCMSBase` object, pass
1576            the MS2 parameter profile used for association/search, typically::
1577
1578                from corems.encapsulation.factory.parameters import settings_from_lcms
1579                settings=settings_from_lcms(lcms_obj)  # mass_spectrum["ms2"]
1580
1581            or explicitly
1582            ``lcms_obj.parameters.mass_spectrum["ms2"].spectral_similarity_search``.
1583            For an :class:`~corems.mass_spectra.factory.lc_class.LCMSCollection`,
1584            configure settings once, broadcast to every sample, then pass the
1585            collection's (first-sample) settings into the library build::
1586
1587                from corems.encapsulation.factory.parameters import (
1588                    apply_spectral_similarity_search_to_collection,
1589                    settings_from_lcms_collection,
1590                )
1591                from corems.encapsulation.factory.processingSetting import (
1592                    SpectralSimilaritySearchSettings,
1593                )
1594
1595                ss = SpectralSimilaritySearchSettings()
1596                ss.max_ms2_tolerance_in_da = 0.01
1597                apply_spectral_similarity_search_to_collection(
1598                    lcms_collection, ss, profile="ms2"
1599                )
1600                settings=settings_from_lcms_collection(lcms_collection)
1601
1602            Mutating ``settings_from_lcms_collection(...)`` in place only changes
1603            the first sample — use :func:`apply_spectral_similarity_search_to_collection`
1604            to keep all members equal. Multi-MS2 profiles use ``profile="ms2_cid"``
1605            (or the matching ``mass_spectrum`` key).
1606        molecular_id_field : str, optional
1607            Field name to use as the unique molecular identifier for linking spectra to metadata.
1608            Default is 'inchikey'. The specified field must exist in the MSP file and contain
1609            non-null values for all entries.
1610
1611        Returns
1612        -------
1613        tuple
1614            (spectral_library, metabolite_metadata_dict) where spectral_library is in the
1615            requested format and metabolite_metadata_dict maps molecular IDs to MetaboliteMetadata objects.
1616
1617        Notes
1618        -----
1619        The molecular_id_field parameter allows flexibility for different MSP file formats:
1620
1621        - Use 'inchikey' for standard metabolite databases (default)
1622        - Use 'name' or 'spectra_id' for custom or isotope-labeled standards
1623        - The specified field must exist and contain non-null values for all entries
1624
1625        *settings* only drives FlashEntropy index construction. Annotation
1626        thresholds such as ``ms2_min_fe_score`` are applied later by
1627        ``fe_search`` on the LCMS object when it reads the same
1628        ``spectral_similarity_search`` instance.
1629
1630        Examples
1631        --------
1632        Single LCMSBase file::
1633
1634            from corems.encapsulation.factory.parameters import settings_from_lcms
1635
1636            settings = settings_from_lcms(lcms_obj, profile="ms2")
1637            fe_lib, mol_meta = msp_interface.get_metabolomics_spectra_library(
1638                polarity="positive",
1639                format="flashentropy",
1640                settings=settings,
1641            )
1642            lcms_obj.fe_search(scan_list=ms2_scans, fe_lib=fe_lib)
1643
1644        Collection — configure, broadcast to all samples, then build library::
1645
1646            from corems.encapsulation.factory.parameters import (
1647                apply_spectral_similarity_search_to_collection,
1648                settings_from_lcms_collection,
1649            )
1650            from corems.encapsulation.factory.processingSetting import (
1651                SpectralSimilaritySearchSettings,
1652            )
1653
1654            ss = SpectralSimilaritySearchSettings()
1655            ss.max_ms2_tolerance_in_da = 0.01
1656            apply_spectral_similarity_search_to_collection(lcms_collection, ss)
1657            fe_lib, mol_meta = msp_interface.get_metabolomics_spectra_library(
1658                polarity="negative",
1659                format="flashentropy",
1660                settings=settings_from_lcms_collection(lcms_collection),
1661            )
1662        """
1663        # Check if the MSP file is compatible with the get_metabolomics_spectra_library method
1664        self._check_msp_compatibility()
1665
1666        # Check if the polarity parameter is valid and if a polarity column exists in the dataframe
1667        if polarity not in ["positive", "negative"]:
1668            raise ValueError("Polarity must be 'positive' or 'negative'")
1669        polarity_column = (
1670            "polarity" if "polarity" in self._data_frame.columns else "ionmode"
1671        )
1672
1673        # Get a subset of the initial dataframea by polarity
1674        db_df = self._data_frame[self._data_frame[polarity_column] == polarity].copy()
1675
1676        # Rename the columns of the db_df to match the MetaboliteMetadata dataclass using the metabolite_metadata_mapping
1677        # If the mapping is not provided, use the default mapping
1678        if not metabolite_metadata_mapping:
1679            metabolite_metadata_mapping = {
1680                "chebi_id": "chebi",
1681                "kegg_id": "kegg",
1682                "refmet_name": "common_name",
1683                "molecular_formula": "formula",
1684                "gnps_spectra_id": "id",
1685                "precursormz": "precursor_mz",
1686                "precursortype": "ion_type",
1687            }
1688        db_df.rename(columns=metabolite_metadata_mapping, inplace=True)
1689
1690        # Create molecular_data_id from the specified field
1691        if molecular_id_field not in db_df.columns:
1692            raise ValueError(
1693                f"Specified molecular_id_field '{molecular_id_field}' not found in MSP data. "
1694                f"Available columns: {', '.join(db_df.columns)}"
1695            )
1696
1697        fe_kwargs = _resolve_fe_kwargs(settings=settings, fe_kwargs=fe_kwargs)
1698
1699        if not db_df[molecular_id_field].notnull().all():
1700            raise ValueError(
1701                f"Specified molecular_id_field '{molecular_id_field}' contains null values. "
1702                f"All entries must have non-null values for the molecular ID field."
1703            )
1704
1705        # Use the specified field as the molecular ID
1706        db_df["molecular_data_id"] = db_df[molecular_id_field].astype(str)
1707
1708        # Ensure 'id' field exists for spectra identification
1709        # If not present, create from spectra_id or use a sequential index
1710        if "id" not in db_df.columns:
1711            if "spectra_id" in db_df.columns:
1712                db_df["id"] = db_df["spectra_id"].astype(str)
1713            else:
1714                # Generate sequential IDs
1715                db_df["id"] = [f"spectrum_{i:06d}" for i in range(len(db_df))]
1716
1717        # Check if the resulting dataframe has the required columns for the flash entropy search
1718        required_columns = ["molecular_data_id", "precursor_mz", "ion_type", "id"]
1719        for col in required_columns:
1720            if col not in db_df.columns:
1721                raise ValueError(
1722                    f"Input field on MSP must contain '{col}' column for FlashEntropy search."
1723                )
1724
1725        # Pull out the metabolite metadata from the dataframe and put it into a different dataframe
1726        # First get a list of the possible attributes of the MetaboliteMetadata dataclass
1727        metabolite_metadata_keys = list(MetaboliteMetadata.__annotations__.keys())
1728        # Replace id with molecular_data_id in metabolite_metadata_keys
1729        metabolite_metadata_keys = [
1730            "molecular_data_id" if x == "id" else x for x in metabolite_metadata_keys
1731        ]
1732        metabolite_metadata_df = db_df[
1733            db_df.columns[db_df.columns.isin(metabolite_metadata_keys)]
1734        ].copy()
1735
1736        # Make unique and recast the id column for metabolite metadata
1737        metabolite_metadata_df.drop_duplicates(
1738            subset=["molecular_data_id"], inplace=True
1739        )
1740        metabolite_metadata_df["id"] = metabolite_metadata_df["molecular_data_id"]
1741
1742        # Convert to a dictionary using the molecular_data_id as the key
1743        metabolite_metadata_dict = metabolite_metadata_df.to_dict(orient="records")
1744        metabolite_metadata_dict = {
1745            v["id"]: self._dict_to_dataclass(v, MetaboliteMetadata)
1746            for v in metabolite_metadata_dict
1747        }
1748
1749        # Remove the metabolite metadata columns from the original dataframe
1750        for key in metabolite_metadata_keys:
1751            if key != "molecular_data_id":
1752                if key in db_df.columns:
1753                    db_df.drop(columns=key, inplace=True)
1754
1755        # Format the spectral library
1756        format_func = self._get_format_func(format)
1757        lib = format_func(db_df, normalize=normalize, fe_kwargs=fe_kwargs)
1758        return (lib, metabolite_metadata_dict)
1759
1760
1761class SpectraverseMS2Interface(SpectralDatabaseInterface):
1762    """Interface to Spectraverse MS/MS (MS2) libraries (MGF).
1763
1764    Loads MS2 spectra only for metabolomics spectral search (Flash Entropy or
1765    DataFrame). Precursor m/z used for search is calculated from formula + adduct
1766    via CoreMS ion-type machinery, not the library experimental PRECURSOR_MZ.
1767
1768    Download the release MGF from Zenodo, e.g.
1769    https://zenodo.org/records/19927403 (``spectraverse-1.0.2.mgf``).
1770
1771    References
1772    ----------
1773    Gupta, V., Qiang, H., Chung, H.-H., Herbst, E., & Skinnider, M. (2026).
1774    Comprehensive curation and harmonization of small molecule MS/MS libraries
1775    in Spectraverse [Data set]. Zenodo.
1776    https://doi.org/10.5281/zenodo.17252772
1777
1778    See also: https://github.com/skinniderlab/spectraverse-analysis
1779    """
1780
1781    def __init__(self, file_path, cache=True, cache_path=None, rebuild_cache=False):
1782        """
1783        Parameters
1784        ----------
1785        file_path : str or Path
1786            Path to a Spectraverse MGF file.
1787        cache : bool or str, optional
1788            If True, use a parquet sidecar when valid, otherwise parse and write
1789            it. If False, always parse. If ``"read"``, require a valid cache.
1790            Default is True.
1791        cache_path : str or Path, optional
1792            Override default ``<file_path>.corems-lib.parquet``.
1793        rebuild_cache : bool, optional
1794            Force re-parse and rewrite the cache. Default is False.
1795        """
1796        super().__init__(key=None)
1797        self.file_path = str(file_path)
1798        if not os.path.exists(self.file_path):
1799            raise FileNotFoundError(
1800                f"File {self.file_path} does not exist. Please check the file path."
1801            )
1802        self._cache_path = (
1803            Path(cache_path)
1804            if cache_path is not None
1805            else spectral_library_cache.default_cache_path(self.file_path)
1806        )
1807        self._precursor_cache = {}
1808        self._data_frame = self._load_dataframe(
1809            cache=cache, rebuild_cache=rebuild_cache
1810        )
1811        self.__init_format_map__()
1812
1813    def __init_format_map__(self):
1814        self.format_map = {
1815            "flashentropy": lambda x, normalize, fe_kwargs: self._to_flashentropy(
1816                x, normalize, fe_kwargs
1817            ),
1818            "df": lambda x, normalize, fe_kwargs: self._to_df(x, normalize),
1819        }
1820        self.format_map["fe"] = self.format_map["flashentropy"]
1821        self.format_map["flash-entropy"] = self.format_map["flashentropy"]
1822        self.format_map["dataframe"] = self.format_map["df"]
1823        self.format_map["data-frame"] = self.format_map["df"]
1824
1825    def _load_dataframe(self, cache=True, rebuild_cache=False):
1826        use_cache = cache not in (False, None)
1827        require_cache = cache == "read"
1828        extra = {
1829            "source_kind": "spectraverse_mgf",
1830            "precursor_policy": "formula+adduct",
1831        }
1832
1833        if (
1834            use_cache
1835            and not rebuild_cache
1836            and spectral_library_cache.is_valid(
1837                self._cache_path, self.file_path, extra_meta=extra
1838            )
1839        ):
1840            return spectral_library_cache.read(self._cache_path)
1841
1842        if require_cache:
1843            raise FileNotFoundError(
1844                f"Valid library cache required at {self._cache_path}"
1845            )
1846
1847        df = self._read_mgf_file()
1848        if use_cache:
1849            spectral_library_cache.write(
1850                self._cache_path, df, self.file_path, extra_meta=extra
1851            )
1852        return df
1853
1854    def _precursor_mz_from_formula(self, formula: str, adduct: str) -> float:
1855        """Calculate precursor m/z from neutral formula and adduct."""
1856        from corems.molecular_formula.calc.MolecularFormulaCalc import (
1857            MolecularFormulaCalc,
1858        )
1859
1860        key = (formula, adduct)
1861        if key in self._precursor_cache:
1862            return self._precursor_cache[key]
1863        mz = MolecularFormulaCalc.precursor_mz_from_formula(formula, adduct)
1864        self._precursor_cache[key] = mz
1865        return mz
1866
1867    def _read_mgf_file(self) -> pd.DataFrame:
1868        """Stream-parse Spectraverse MGF; keep MS2 spectra only."""
1869        spectra = []
1870        headers = {}
1871        peaks = []
1872        in_block = False
1873        n_skipped = 0
1874
1875        with open(self.file_path, "r", encoding="utf-8", errors="replace") as f:
1876            for line in f:
1877                line = line.strip()
1878                if not line:
1879                    continue
1880                if line == "BEGIN IONS":
1881                    in_block = True
1882                    headers = {}
1883                    peaks = []
1884                    continue
1885                if line == "END IONS":
1886                    if in_block:
1887                        row = self._finalize_mgf_block(headers, peaks)
1888                        if row is None:
1889                            n_skipped += 1
1890                        else:
1891                            spectra.append(row)
1892                    in_block = False
1893                    headers = {}
1894                    peaks = []
1895                    continue
1896                if not in_block:
1897                    continue
1898                if "=" in line and not line[0].isdigit():
1899                    key, value = line.split("=", 1)
1900                    headers[key.strip().upper()] = value.strip()
1901                elif line[0].isdigit() or (
1902                    line[0] == "-" and len(line) > 1 and line[1].isdigit()
1903                ):
1904                    parts = line.split()
1905                    if len(parts) >= 2:
1906                        peaks.append([float(parts[0]), float(parts[1])])
1907
1908        if not spectra:
1909            raise ValueError(
1910                f"No MS2 spectra loaded from {self.file_path}"
1911                + (f" ({n_skipped} blocks skipped)" if n_skipped else "")
1912            )
1913        if n_skipped:
1914            warnings.warn(
1915                f"Skipped {n_skipped} Spectraverse MGF blocks (non-MS2, empty, or "
1916                f"unsupported formula/adduct).",
1917                UserWarning,
1918            )
1919        return pd.DataFrame(spectra)
1920
1921    def _finalize_mgf_block(self, headers: dict, peaks: list):
1922        ms_level = str(headers.get("MS_LEVEL", "")).strip().upper()
1923        if ms_level not in ("MS2", "2"):
1924            return None
1925        if not peaks:
1926            return None
1927
1928        formula = headers.get("FORMULA", "").strip()
1929        adduct = headers.get("ADDUCT", "").strip()
1930        if not formula or not adduct:
1931            return None
1932
1933        try:
1934            precursor_mz = self._precursor_mz_from_formula(formula, adduct)
1935        except Exception:
1936            return None
1937
1938        ionmode = headers.get("IONMODE", "").strip().lower()
1939        title = headers.get("TITLE", "").strip()
1940        compound_name = headers.get("COMPOUND_NAME", "").strip() or "Unknown"
1941        peaks_arr = np.asarray(peaks, dtype=np.float64)
1942
1943        lib_precursor = headers.get("PRECURSOR_MZ", "").strip()
1944        try:
1945            precursor_mz_library = float(lib_precursor) if lib_precursor else None
1946        except ValueError:
1947            precursor_mz_library = None
1948
1949        return {
1950            "id": title or None,
1951            "spectra_id": title or None,
1952            "name": compound_name,
1953            "compound_name": compound_name,
1954            "formula": formula,
1955            "inchikey": headers.get("INCHIKEY", "").strip() or None,
1956            "inchi": headers.get("INCHI", "").strip() or None,
1957            "smiles": headers.get("SMILES", "").strip() or None,
1958            "ionmode": ionmode,
1959            "polarity": ionmode,
1960            "ion_type": adduct,
1961            "adduct": adduct,
1962            "precursor_mz": precursor_mz,
1963            "precursor_mz_library": precursor_mz_library,
1964            "charge": headers.get("CHARGE", "").strip() or None,
1965            "source": headers.get("SOURCE", "").strip() or None,
1966            "instrument_type": headers.get("INSTRUMENT_TYPE", "").strip() or None,
1967            "ms_level": ms_level,
1968            "peaks": peaks_arr,
1969            "num peaks": len(peaks_arr),
1970        }
1971
1972    def _to_df(self, input_dataframe, normalize=True):
1973        if not normalize:
1974            return input_dataframe
1975        db_dict = input_dataframe.to_dict(orient="records")
1976        lib = []
1977        for spectrum in db_dict:
1978            if "peaks" not in spectrum:
1979                raise KeyError("peaks key not found in spectrum")
1980            if not isinstance(spectrum["peaks"], np.ndarray):
1981                spectrum["peaks"] = np.array(spectrum["peaks"])
1982            spectrum["peaks"] = self.normalize_peaks(spectrum["peaks"])
1983            spectrum["num peaks"] = len(spectrum["peaks"])
1984            lib.append(spectrum)
1985        return pd.DataFrame(lib)
1986
1987    def _to_flashentropy(self, input_dataframe, normalize=True, fe_kwargs={}):
1988        self._check_flash_entropy_kwargs(fe_kwargs)
1989        fe_lib = []
1990        for source in input_dataframe.to_dict(orient="records"):
1991            spectrum = source
1992            if "precursor_mz" not in spectrum:
1993                raise KeyError("precursor_mz required for FlashEntropy format")
1994            if "peaks" not in spectrum:
1995                raise KeyError("peaks key not found in spectrum")
1996            if not isinstance(spectrum["peaks"], np.ndarray):
1997                spectrum["peaks"] = np.array(spectrum["peaks"])
1998            if normalize:
1999                spectrum["peaks"] = self.normalize_peaks(spectrum["peaks"])
2000            fe_lib.append(spectrum)
2001        return self._build_flash_entropy_index(fe_lib, fe_kwargs=fe_kwargs)
2002
2003    def _check_spectraverse_compatibility(self):
2004        if (
2005            "polarity" not in self._data_frame.columns
2006            and "ionmode" not in self._data_frame.columns
2007        ):
2008            raise ValueError(
2009                "Neither 'polarity' nor 'ionmode' columns found in Spectraverse data."
2010            )
2011        polarity_column = (
2012            "polarity" if "polarity" in self._data_frame.columns else "ionmode"
2013        )
2014        if not all(self._data_frame[polarity_column].isin(["positive", "negative"])):
2015            raise ValueError(
2016                f"'{polarity_column}' must contain only 'positive' or 'negative' values."
2017            )
2018        if "formula" not in self._data_frame.columns:
2019            raise ValueError("Spectraverse data must contain a 'formula' column.")
2020        if not all(self._data_frame["formula"].notnull()):
2021            raise ValueError("'formula' must contain only non-null values.")
2022
2023    def get_metabolomics_spectra_library(
2024        self,
2025        polarity,
2026        metabolite_metadata_mapping=None,
2027        format="fe",
2028        normalize=True,
2029        fe_kwargs=None,
2030        settings=None,
2031        molecular_id_field="inchikey",
2032    ):
2033        """
2034        Prepare metabolomics MS2 spectra library and metabolite metadata.
2035
2036        Parameters
2037        ----------
2038        polarity : str
2039            ``'positive'`` or ``'negative'``.
2040        metabolite_metadata_mapping : dict, optional
2041            Optional column renames into MetaboliteMetadata fields.
2042        format : str, optional
2043            ``'fe'`` / ``'flashentropy'`` or ``'df'`` / ``'dataframe'``.
2044        normalize : bool, optional
2045            Normalize peak intensities. Default True.
2046        fe_kwargs : dict, optional
2047            FlashEntropy build options. Override keys from *settings* when both
2048            are provided. Prefer *settings* for new code.
2049        settings : SpectralSimilaritySearchSettings, optional
2050            Encapsulated FE build settings (same object used by
2051            ``MSPInterface.get_metabolomics_spectra_library``). Constructor
2052            kwargs ``cache`` / ``cache_path`` / ``rebuild_cache`` stay on
2053            :meth:`__init__`; do not pass *settings* there.
2054        molecular_id_field : str, optional
2055            Field used as molecular metadata key. Default ``'inchikey'``.
2056
2057        Returns
2058        -------
2059        tuple
2060            ``(spectral_library, metabolite_metadata_dict)``.
2061        """
2062        if metabolite_metadata_mapping is None:
2063            metabolite_metadata_mapping = {}
2064        fe_kwargs = _resolve_fe_kwargs(settings=settings, fe_kwargs=fe_kwargs)
2065
2066        self._check_spectraverse_compatibility()
2067
2068        if polarity not in ["positive", "negative"]:
2069            raise ValueError("Polarity must be 'positive' or 'negative'")
2070        polarity_column = (
2071            "polarity" if "polarity" in self._data_frame.columns else "ionmode"
2072        )
2073        db_df = self._data_frame[self._data_frame[polarity_column] == polarity].copy()
2074        if db_df.empty:
2075            raise ValueError(f"No Spectraverse spectra for polarity={polarity!r}")
2076
2077        if metabolite_metadata_mapping:
2078            db_df.rename(columns=metabolite_metadata_mapping, inplace=True)
2079
2080        if molecular_id_field not in db_df.columns:
2081            raise ValueError(
2082                f"Specified molecular_id_field '{molecular_id_field}' not found. "
2083                f"Available columns: {', '.join(map(str, db_df.columns))}"
2084            )
2085        if not db_df[molecular_id_field].notnull().all():
2086            raise ValueError(
2087                f"Specified molecular_id_field '{molecular_id_field}' contains null values."
2088            )
2089
2090        db_df["molecular_data_id"] = db_df[molecular_id_field].astype(str)
2091        if "id" not in db_df.columns or db_df["id"].isnull().any():
2092            if "spectra_id" in db_df.columns:
2093                db_df["id"] = db_df["spectra_id"].astype(str)
2094            else:
2095                db_df["id"] = [f"spectrum_{i:06d}" for i in range(len(db_df))]
2096
2097        required_columns = ["molecular_data_id", "precursor_mz", "ion_type", "id"]
2098        for col in required_columns:
2099            if col not in db_df.columns:
2100                raise ValueError(
2101                    f"Spectraverse data must contain '{col}' for FlashEntropy search."
2102                )
2103
2104        metabolite_metadata_keys = list(MetaboliteMetadata.__annotations__.keys())
2105        metabolite_metadata_keys = [
2106            "molecular_data_id" if x == "id" else x for x in metabolite_metadata_keys
2107        ]
2108        metabolite_metadata_df = db_df[
2109            db_df.columns[db_df.columns.isin(metabolite_metadata_keys)]
2110        ].copy()
2111        metabolite_metadata_df.drop_duplicates(
2112            subset=["molecular_data_id"], inplace=True
2113        )
2114        metabolite_metadata_df["id"] = metabolite_metadata_df["molecular_data_id"]
2115        metabolite_metadata_dict = {
2116            v["id"]: self._dict_to_dataclass(v, MetaboliteMetadata)
2117            for v in metabolite_metadata_df.to_dict(orient="records")
2118        }
2119
2120        for key in metabolite_metadata_keys:
2121            if key != "molecular_data_id" and key in db_df.columns:
2122                db_df.drop(columns=key, inplace=True)
2123
2124        format_func = self._get_format_func(format)
2125        lib = format_func(db_df, normalize=normalize, fe_kwargs=fe_kwargs)
2126        return (lib, metabolite_metadata_dict)