In [1]:
# ====================================================================
# Developer Utility: Module Auto-Reloading
# --------------------------------------------------------------------
# Uncomment the lines below if you are actively modifying the underlying
# pi-metaboqc source code. It ensures that changes in .py files are
# dynamically reloaded without restarting the Jupyter kernel.
# ====================================================================
%load_ext autoreload
%autoreload 2
${\pi}$-metaboqc: Interactive Analytical Workflow¶
This notebook provides an interactive, step-by-step execution orchestration of the pi-metaboqc metabolomics data quality control (QC) pipeline.
By executing each cell sequentially, you can trace the data provenance, inspect intermediate matrices, and visualize quality assessment (QA) diagnostics at each stage of the computational framework.
Step 00: Environment Initialization¶
This phase initializes the pi-metaboqc computational environment and performs baseline hardware diagnostics. It ensures that the output directory structure is securely mounted before commencing heavy matrix operations.
In [2]:
import os
from importlib.resources import files
import pandas as pd
from loguru import logger
import pimqc
import pimqc.io_utils as iu
import pimqc.report_utils as ru
from pimqc import (
build_dataset,
MetaboIntAssessor,
MetaboIntFilter,
MetaboIntCorrector,
MetaboIntImputer,
MetaboIntNormalizer,
)
logger.info(f"pimqc.__version__: {pimqc.__version__}")
pimqc.init(check_hardware=False, log_level="INFO", show_progress=True)
# Define standard directories
DATA_DIR = str(files("pimqc") / "data")
OUTPUT_DIR = os.path.join(".", "tutorial_output")
iu._check_dir_exists(dir_path=OUTPUT_DIR, handle="makedirs")
# Load pipeline parameters
PARAMS_PATH = os.path.join(DATA_DIR, "pipeline_parameters.toml")
params = iu.load_pipeline_config(str(PARAMS_PATH))
# Load raw matrices
meta_df = pd.read_csv(os.path.join(DATA_DIR, "project_meta.csv"), header=[0])
int_df = pd.read_csv(
os.path.join(DATA_DIR, "project_intensity.csv"), index_col=[0], header=[0]
)
2026-07-23 21:03:04.907 | INFO | __main__:<module>:18 - pimqc.__version__: 1.1.5 2026-07-23 21:03:04.908 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output. 2026-07-23 21:03:04.915 | SUCCESS | io_utils:load_pipeline_config:412 - Pipeline configuration successfully loaded and validated via Pydantic. 2026-07-23 21:03:04.908 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output. 2026-07-23 21:03:04.915 | SUCCESS | io_utils:load_pipeline_config:412 - Pipeline configuration successfully loaded and validated via Pydantic.
Step 01: Raw Dataset Construction¶
The workflow begins by transforming fragmented raw peak tables and metadata into a standardized MetaboInt object. This phase ensures precise coordinate alignment between sample identifiers and feature intensities, establishing a robust structural foundation.
In [3]:
logger.info("Step 01: Dataset Construction...")
step1_dir = os.path.join(OUTPUT_DIR, "01_Raw_Data")
raw_data = build_dataset(
meta_info=meta_df, int_df=int_df, pipeline_params=params, output_dir=step1_dir
)
is_multi_batch_flag = raw_data.attrs["is_multi_batch"]
2026-07-23 21:03:05.037 | INFO | __main__:<module>:1 - Step 01: Dataset Construction... 2026-07-23 21:03:05.048 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output\01_Raw_Data. 2026-07-23 21:03:05.128 | INFO | dataset_builder:execute_build:385 - MetaboInt raw dataset saved as: .\tutorial_output\01_Raw_Data\Raw_Data_Intensity.csv 2026-07-23 21:03:05.129 | INFO | dataset_builder:execute_build:402 - MetaboInt object built: 376 metabolites, 466 samples. 2026-07-23 21:03:05.129 | INFO | dataset_builder:_audit_dataset_health:252 - Executing dataset health audit... 2026-07-23 21:03:05.131 | INFO | dataset_builder:_audit_dataset_health:277 - [Audit] No Outlier Reference Features (ORF) detected. This is normal for untargeted datasets; ORF diagnostics will be skipped.
2026-07-23 21:03:06.847 | INFO | dataset_builder:execute_build:419 - Global acquisition overview plot saved as: .\tutorial_output\01_Raw_Data\Global_Acquisition_Overview.svg 2026-07-23 21:03:06.849 | SUCCESS | io_utils:time_wrap:479 - Execution time of "build_dataset": 00:00:01.810. 2026-07-23 21:03:06.849 | SUCCESS | io_utils:time_wrap:479 - Execution time of "build_dataset": 00:00:01.810.
QA-Step 01: Quality Assessment of Raw Data¶
Evaluates the baseline data distribution, acquisition sequences, and pooled QC clustering before any computational manipulation occurs.
In [4]:
logger.info("QA-Step 01: Quality Assessment of Raw Data...")
qa_step1_dir = os.path.join(OUTPUT_DIR, "QA_01_Raw_Data")
qa_raw_engine = MetaboIntAssessor(data=raw_data, pipeline_params=params)
qa_raw_engine.execute_assessment(output_dir=qa_step1_dir)
2026-07-23 21:03:06.895 | INFO | __main__:<module>:1 - QA-Step 01: Quality Assessment of Raw Data... 2026-07-23 21:03:06.896 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output\QA_01_Raw_Data.
2026-07-23 21:03:18.374 | INFO | assessment:execute_assessment:542 - Assessor summary dashboard saved as: .\tutorial_output\QA_01_Raw_Data\QA_Summary_Dashboard.svg 2026-07-23 21:03:18.375 | SUCCESS | assessment:execute_assessment:543 - Data quality assessment completed. 2026-07-23 21:03:18.375 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_assessment": 00:00:11.478. 2026-07-23 21:03:18.375 | SUCCESS | assessment:execute_assessment:543 - Data quality assessment completed. 2026-07-23 21:03:18.375 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_assessment": 00:00:11.478.
Step 02: Missing Value Classification & Filtering¶
Implements a topological classification algorithm to segregate missing values into Missing at Random (MAR) and Missing Not at Random (MNAR) based on biological groupings and QC thresholds, strictly filtering out unsalvageable features.
In [5]:
logger.info("Step 02: High Missing Value Feature Filter...")
step2_dir = os.path.join(OUTPUT_DIR, "02_MV_Filtered")
fltr_mv_engine = MetaboIntFilter(data=raw_data, pipeline_params=params)
mv_filter_data = fltr_mv_engine.execute_mv_filtering(output_dir=step2_dir)
2026-07-23 21:03:18.404 | INFO | __main__:<module>:1 - Step 02: High Missing Value Feature Filter... 2026-07-23 21:03:18.414 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output\02_MV_Filtered.
2026-07-23 21:03:20.915 | INFO | filtering:_execute_s1_visualization:527 - High-MV Filter summary dashboard saved as: .\tutorial_output\02_MV_Filtered\MV_Classification_Dashboard.svg 2026-07-23 21:03:20.916 | SUCCESS | filtering:execute_mv_filtering:423 - High-missing value feature filtering completed. 2026-07-23 21:03:20.917 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_mv_filtering": 00:00:02.511. 2026-07-23 21:03:20.916 | SUCCESS | filtering:execute_mv_filtering:423 - High-missing value feature filtering completed. 2026-07-23 21:03:20.917 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_mv_filtering": 00:00:02.511.
QA-Step 02: Quality Assessment of High-MV Filtered Data¶
Evaluates whether global data distributions remain undisturbed after the removal of high-missing-rate features, ensuring no artificial bias is introduced during topological pruning.
In [6]:
logger.info("QA-Step 02: Quality Assessment of High-MV Filtered Data...")
qa_step2_dir = os.path.join(OUTPUT_DIR, "QA_02_MV_Filtered")
qa_mv_filter_engine = MetaboIntAssessor(
data=mv_filter_data, pipeline_params=params)
qa_mv_filter_engine.execute_assessment(output_dir=qa_step2_dir)
2026-07-23 21:03:20.942 | INFO | __main__:<module>:1 - QA-Step 02: Quality Assessment of High-MV Filtered Data... 2026-07-23 21:03:20.945 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output\QA_02_MV_Filtered.
2026-07-23 21:03:30.810 | INFO | assessment:execute_assessment:542 - Assessor summary dashboard saved as: .\tutorial_output\QA_02_MV_Filtered\QA_Summary_Dashboard.svg 2026-07-23 21:03:30.810 | SUCCESS | assessment:execute_assessment:543 - Data quality assessment completed. 2026-07-23 21:03:30.811 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_assessment": 00:00:09.866. 2026-07-23 21:03:30.810 | SUCCESS | assessment:execute_assessment:543 - Data quality assessment completed. 2026-07-23 21:03:30.811 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_assessment": 00:00:09.866.
Step 03: Signal Drift and Batch Effect Mitigation¶
Standardizes signal intensities using two selectable strategies:
- Classical Multi-Stage: Fits intra-batch drift (e.g., QC-RLSC/SVR/RFSC) followed by inter-batch median alignment.
- Global Model Correction: evaluates SERRF, RUV-III, and WaveICA 2.0 as full-matrix correction strategies alongside the QC-anchored regression methods.
In [7]:
logger.info("Step 03: Signal Drift & Batch Effect Correction...")
step3_dir = os.path.join(OUTPUT_DIR, "03_Corrected_Data")
sc_engine = MetaboIntCorrector(data=mv_filter_data, pipeline_params=params)
corrected_stages = sc_engine.execute_signal_correction(output_dir=step3_dir)
final_corr_data = list(corrected_stages.values())[-1]
2026-07-23 21:03:30.838 | INFO | __main__:<module>:1 - Step 03: Signal Drift & Batch Effect Correction... 2026-07-23 21:03:30.841 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output\03_Corrected_Data. 2026-07-23 21:03:30.843 | INFO | correction:execute_signal_correction:1674 - AUTO mode enabled. Evaluating multiple methods. 2026-07-23 21:03:30.844 | INFO | correction:_evaluate_correction_candidates:1399 - --- Evaluating Method: SERRF --- 2026-07-23 21:03:30.846 | INFO | correction:_prepare_serrf_correlation_matrix:1341 - Calculating Spearman correlation on features... 2026-07-23 21:03:30.853 | INFO | correction:fit_transform:606 - Initializing High-Performance Hybrid SERRF Corrector...
SERRF: 100%|█████████████████████████████████████████████████████████████████████| 347/347 [Elapsed: 00:13 | ETA: 00:00]
2026-07-23 21:03:47.302 | INFO | correction:_evaluate_correction_candidates:1582 - SERRF Eval QC RSD: 10.00% 2026-07-23 21:03:47.303 | INFO | correction:_evaluate_correction_candidates:1399 - --- Evaluating Method: RUV-III --- 2026-07-23 21:03:47.313 | INFO | correction:_prepare_ruv_control_features:1378 - RUV-III Control Features: 20 total (5 predefined, 17 empirical). 2026-07-23 21:03:47.314 | INFO | correction:fit_transform:1046 - Executing RUV-III (k=5)... 2026-07-23 21:03:47.317 | WARNING | correction:fit_transform:1061 - NaNs detected. Applying median imputation... 2026-07-23 21:03:48.043 | INFO | correction:_evaluate_correction_candidates:1582 - RUV-III Eval QC RSD: 11.09% 2026-07-23 21:03:48.043 | INFO | correction:_evaluate_correction_candidates:1399 - --- Evaluating Method: WaveICA 2.0 --- 2026-07-23 21:03:48.044 | INFO | correction:fit_transform:993 - Executing WaveICA 2.0 correction... 2026-07-23 21:03:48.640 | INFO | correction:_evaluate_correction_candidates:1582 - WaveICA 2.0 Eval QC RSD: 17.57% 2026-07-23 21:03:48.641 | INFO | correction:_evaluate_correction_candidates:1399 - --- Evaluating Method: QC-RLSC --- 2026-07-23 21:03:48.641 | INFO | correction:fit_transform:320 - Phase 1: Executing Intra-batch drift correction with QC-RLSC... 2026-07-23 21:03:53.378 | INFO | correction:fit_transform:437 - Phase 2: Executing Inter-batch median alignment... 2026-07-23 21:03:54.350 | INFO | correction:_evaluate_correction_candidates:1582 - QC-RLSC Eval QC RSD: 14.42% 2026-07-23 21:03:54.351 | INFO | correction:_evaluate_correction_candidates:1399 - --- Evaluating Method: QC-RFSC --- 2026-07-23 21:03:54.351 | INFO | correction:fit_transform:320 - Phase 1: Executing Intra-batch drift correction with QC-RFSC...
SC [B1]: 100%|███████████████████████████████████████████████████████████████████| 347/347 [Elapsed: 00:24 | ETA: 00:00] SC [B2]: 100%|███████████████████████████████████████████████████████████████████| 347/347 [Elapsed: 00:17 | ETA: 00:00] SC [B3]: 100%|███████████████████████████████████████████████████████████████████| 347/347 [Elapsed: 00:17 | ETA: 00:00]
2026-07-23 21:04:54.478 | INFO | correction:fit_transform:437 - Phase 2: Executing Inter-batch median alignment...
2026-07-23 21:04:55.083 | INFO | correction:_evaluate_correction_candidates:1582 - QC-RFSC Eval QC RSD: 13.20% 2026-07-23 21:04:55.084 | INFO | correction:_evaluate_correction_candidates:1399 - --- Evaluating Method: QC-SVR --- 2026-07-23 21:04:55.085 | INFO | correction:fit_transform:320 - Phase 1: Executing Intra-batch drift correction with QC-SVR...
SC [B1]: 100%|███████████████████████████████████████████████████████████████████| 347/347 [Elapsed: 00:00 | ETA: 00:00] SC [B2]: 100%|███████████████████████████████████████████████████████████████████| 347/347 [Elapsed: 00:00 | ETA: 00:00] SC [B3]: 100%|███████████████████████████████████████████████████████████████████| 347/347 [Elapsed: 00:00 | ETA: 00:00]
2026-07-23 21:04:56.016 | INFO | correction:fit_transform:437 - Phase 2: Executing Inter-batch median alignment... 2026-07-23 21:04:56.639 | INFO | correction:_evaluate_correction_candidates:1582 - QC-SVR Eval QC RSD: 14.32% 2026-07-23 21:04:56.640 | SUCCESS | correction:execute_signal_correction:1702 - Auto selection: SERRF is optimal (score = 0.663, Eval QC RSD = 10.00%). 2026-07-23 21:04:56.641 | INFO | correction:execute_signal_correction:1720 - Assembling correction diagnostic dashboard...
2026-07-23 21:05:02.535 | INFO | correction:execute_signal_correction:1751 - Correction candidate dashboard saved as: .\tutorial_output\03_Corrected_Data\Correction_Candidate_Dashboard_SERRF.svg 2026-07-23 21:05:02.632 | INFO | correction:execute_signal_correction:1809 - Generating IS plots for SERRF... 2026-07-23 21:05:02.633 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output\03_Corrected_Data\Internal_Standard_Scatters. 2026-07-23 21:05:06.260 | INFO | correction:execute_signal_correction:1857 - Bypassing IS baseline prediction for SERRF. 2026-07-23 21:05:06.261 | SUCCESS | correction:execute_signal_correction:1861 - Signal drift correction (SERRF) completed. 2026-07-23 21:05:06.274 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_signal_correction": 00:01:35.433.
QA-Step 03: Quality Assessment of Corrected Data¶
Evaluates correction efficacy across both multi-stage and unified strategies. Tracks multi-batch alignment, intra-batch smoothing, and feature-wise RSD reduction to verify technical error suppression. Ensures biological group clustering is driven by intrinsic traits.
In [8]:
qa_step3_dir = os.path.join(OUTPUT_DIR, "QA_03_Corrected_Data")
qa_engines_dict = {}
for stage_name, stage_data in corrected_stages.items():
logger.info(f"Executing Quality Assessment (QA) for: {stage_name}")
qa_corr_engine = MetaboIntAssessor(data=stage_data, pipeline_params=params)
qa_corr_engine.execute_assessment(
output_dir=os.path.join(qa_step3_dir, stage_name))
qa_engines_dict[stage_name] = qa_corr_engine
2026-07-23 21:05:06.303 | INFO | __main__:<module>:5 - Executing Quality Assessment (QA) for: SERRF 2026-07-23 21:05:06.305 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output\QA_03_Corrected_Data\SERRF.
2026-07-23 21:05:16.670 | INFO | assessment:execute_assessment:542 - Assessor summary dashboard saved as: .\tutorial_output\QA_03_Corrected_Data\SERRF\QA_Summary_Dashboard.svg 2026-07-23 21:05:16.671 | SUCCESS | assessment:execute_assessment:543 - Data quality assessment completed. 2026-07-23 21:05:16.671 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_assessment": 00:00:10.366. 2026-07-23 21:05:16.671 | SUCCESS | assessment:execute_assessment:543 - Data quality assessment completed. 2026-07-23 21:05:16.671 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_assessment": 00:00:10.366.
Step 04: Low-Quality Feature Filtering¶
Performs a rigorous reproducibility check using the analytical variance (RSD) of Pooled QC samples and Blank/QC ratio. Features exhibiting unacceptable technical variance post-correction are permanently eliminated from the quantitative matrix.
In [9]:
logger.info("Step 04: Low-Quality Feature Filtering...")
step4_dir = os.path.join(OUTPUT_DIR, "04_Quality_Filtered")
fltr_low_quality_engine = MetaboIntFilter(
data=final_corr_data, pipeline_params=params)
low_quality_filter_data = fltr_low_quality_engine.execute_quality_filtering(
output_dir=step4_dir
)
2026-07-23 21:05:16.697 | INFO | __main__:<module>:1 - Step 04: Low-Quality Feature Filtering... 2026-07-23 21:05:16.701 | INFO | filtering:execute_quality_filtering:670 - Features before filtering: 347 2026-07-23 21:05:16.708 | INFO | filtering:execute_quality_filtering:688 - Features after Blank/QC check: 261 2026-07-23 21:05:16.716 | INFO | filtering:execute_quality_filtering:707 - Features after QC RSD check: 248 2026-07-23 21:05:16.727 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output\04_Quality_Filtered. 2026-07-23 21:05:16.790 | INFO | filtering:execute_quality_filtering:744 - Data after low-quality features filtering saved as: .\tutorial_output\04_Quality_Filtered\Filtered_Data_Low-quality_Features.csv
2026-07-23 21:05:17.903 | INFO | filtering:_execute_s2_visualization:842 - Low-quality Filter summary dashboard saved as: .\tutorial_output\04_Quality_Filtered\Low-quality_Filtering_Dashboard.svg 2026-07-23 21:05:17.904 | SUCCESS | filtering:execute_quality_filtering:756 - Low-quality features filtering completed. 2026-07-23 21:05:17.905 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_quality_filtering": 00:00:01.203. 2026-07-23 21:05:17.904 | SUCCESS | filtering:execute_quality_filtering:756 - Low-quality features filtering completed. 2026-07-23 21:05:17.905 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_quality_filtering": 00:00:01.203.
QA-Step 04: Quality Assessment on Low-Quality Feature Filtered Data¶
A pre-imputation health check is performed on the refined dataset. This evaluation confirms that the surviving features represent high-fidelity biological signals, ensuring the matrix is optimally prepared for missing value reconstruction
In [10]:
logger.info("QA-Step 04: Quality Assessment on Low-Quality Feature Filtered Data...")
qa_step4_dir = os.path.join(OUTPUT_DIR, "QA_04_Quality_Filtered")
qa_low_quality_filter_engine = MetaboIntAssessor(
data=low_quality_filter_data, pipeline_params=params
)
qa_low_quality_filter_engine.execute_assessment(output_dir=qa_step4_dir)
2026-07-23 21:05:17.933 | INFO | __main__:<module>:1 - QA-Step 04: Quality Assessment on Low-Quality Feature Filtered Data... 2026-07-23 21:05:17.935 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output\QA_04_Quality_Filtered.
2026-07-23 21:05:28.729 | INFO | assessment:execute_assessment:542 - Assessor summary dashboard saved as: .\tutorial_output\QA_04_Quality_Filtered\QA_Summary_Dashboard.svg 2026-07-23 21:05:28.729 | SUCCESS | assessment:execute_assessment:543 - Data quality assessment completed. 2026-07-23 21:05:28.730 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_assessment": 00:00:10.794. 2026-07-23 21:05:28.729 | SUCCESS | assessment:execute_assessment:543 - Data quality assessment completed. 2026-07-23 21:05:28.730 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_assessment": 00:00:10.794.
Step 05: Missing Value Imputation¶
An autonomous multi-algorithm benchmarking simulation is executed for MAR. The optimal algorithm is programmatically selected based on its ability to reconstruct established distributions while minimizing bias in the original variance structure.
Execute min-value imputation for MNAR.
In [11]:
logger.info("Step 05: Missing Value Imputation...")
step5_dir = os.path.join(OUTPUT_DIR, "05_Imputation")
imp_engine = MetaboIntImputer(
data=low_quality_filter_data, pipeline_params=params)
imputed_data = imp_engine.execute_imputation(output_dir=step5_dir)
2026-07-23 21:05:28.760 | INFO | __main__:<module>:1 - Step 05: Missing Value Imputation... 2026-07-23 21:05:28.769 | INFO | imputation:execute_imputation:1326 - Hybrid Imputation Engine Initialized. MAR: Auto (Evaluating KNN=5, LLS (K=15), BPCA (PCs=2), MinProb, Median) | MNAR: QRILC | Sim_Mask: 0.05 2026-07-23 21:05:28.778 | INFO | imputation:execute_imputation:1339 - Applying QRILC to 3 MNAR features. 2026-07-23 21:05:29.802 | INFO | imputation:_select_best_imputation_method:1102 - Simulating "KNN" on MAR subset... 2026-07-23 21:05:30.326 | INFO | imputation:_select_best_imputation_method:1102 - Simulating "MinProb" on MAR subset... 2026-07-23 21:05:31.242 | INFO | imputation:_select_best_imputation_method:1102 - Simulating "QRILC" on MAR subset... 2026-07-23 21:05:31.879 | INFO | imputation:_select_best_imputation_method:1102 - Simulating "Median" on MAR subset... 2026-07-23 21:05:32.571 | INFO | imputation:_select_best_imputation_method:1102 - Simulating "LLS" on MAR subset... 2026-07-23 21:05:33.020 | INFO | imputation:_select_best_imputation_method:1102 - Simulating "BPCA" on MAR subset... 2026-07-23 21:05:33.781 | INFO | imputation:_select_best_imputation_method:1154 - Optimal MAR algorithm selected: KNN (score=0.888) 2026-07-23 21:05:33.782 | INFO | imputation:execute_imputation:1374 - Executing isolated 'KNN' on MAR features. 2026-07-23 21:05:33.896 | INFO | imputation:execute_imputation:1470 - Calculating imputation-related metrics... 2026-07-23 21:05:34.023 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output\05_Imputation. 2026-07-23 21:05:34.129 | INFO | imputation:execute_imputation:1480 - Generating diagnostic plots for imputation...
2026-07-23 21:05:37.122 | INFO | imputation:execute_imputation:1507 - Imputation dashboard saved as: .\tutorial_output\05_Imputation\Imputation_Dashboard_KNN.svg
2026-07-23 21:05:39.514 | INFO | imputation:execute_imputation:1524 - Imputer candidate NRMSE grid saved as: .\tutorial_output\05_Imputation\Imputation_Candidate_Dashboard_KNN.svg 2026-07-23 21:05:39.515 | SUCCESS | imputation:execute_imputation:1581 - Missing value imputation completed successfully. 2026-07-23 21:05:39.518 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_imputation": 00:00:10.755. 2026-07-23 21:05:39.515 | SUCCESS | imputation:execute_imputation:1581 - Missing value imputation completed successfully. 2026-07-23 21:05:39.518 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_imputation": 00:00:10.755.
QA-Step 05: Quality Assessment on Imputed Data¶
Evaluates the extent to which synthetic data points might introduce artificial clustering or distort natural biological correlations, with a specific focus on the stability of low-abundance signals.
In [12]:
logger.info("QA-Step 05: Quality Assessment of Imputated Data...")
qa_step5_dir = os.path.join(OUTPUT_DIR, "QA_05_Imputed_Data")
qa_imp_engine = MetaboIntAssessor(data=imputed_data, pipeline_params=params)
qa_imp_engine.execute_assessment(output_dir=qa_step5_dir)
2026-07-23 21:05:39.549 | INFO | __main__:<module>:1 - QA-Step 05: Quality Assessment of Imputated Data... 2026-07-23 21:05:39.551 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output\QA_05_Imputed_Data.
2026-07-23 21:05:50.277 | INFO | assessment:execute_assessment:542 - Assessor summary dashboard saved as: .\tutorial_output\QA_05_Imputed_Data\QA_Summary_Dashboard.svg 2026-07-23 21:05:50.277 | SUCCESS | assessment:execute_assessment:543 - Data quality assessment completed. 2026-07-23 21:05:50.278 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_assessment": 00:00:10.727. 2026-07-23 21:05:50.277 | SUCCESS | assessment:execute_assessment:543 - Data quality assessment completed. 2026-07-23 21:05:50.278 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_assessment": 00:00:10.727.
Step 06: Data Normalization¶
Applies advanced normalization techniques (e.g., PQN, VSN, Quantile) to stabilize heteroscedastic variance and harmonize global intensity scales across all biological samples.
In [13]:
logger.info("Step 06: Data Normalization...")
step6_dir = os.path.join(OUTPUT_DIR, "06_Normalized_Data")
norm_engine = MetaboIntNormalizer(imputed_data, pipeline_params=params)
normalized_data = norm_engine.execute_normalization(output_dir=step6_dir)
2026-07-23 21:05:50.309 | INFO | __main__:<module>:1 - Step 06: Data Normalization... 2026-07-23 21:05:50.312 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output\06_Normalized_Data. 2026-07-23 21:05:50.315 | INFO | normalization:execute_normalization:1488 - Permanently dropping 24 Blank samples. 2026-07-23 21:05:50.315 | INFO | normalization:execute_normalization:1490 - Applying Normalization | Method: Auto 2026-07-23 21:05:50.325 | INFO | normalization:_select_auto_normalization:1317 - Auto normalization evaluates fixed strategies on a common log-like view using QC RLE, QC variance stabilization, QC structure, and sample structure criteria. 2026-07-23 21:05:50.325 | INFO | normalization:_select_auto_normalization:1328 - Evaluating Auto normalization candidate: ROBUST_LOG_ONLY (log_transform=True). 2026-07-23 21:05:50.839 | INFO | normalization:_select_auto_normalization:1328 - Evaluating Auto normalization candidate: TIC (log_transform=True). 2026-07-23 21:05:51.343 | INFO | normalization:_select_auto_normalization:1328 - Evaluating Auto normalization candidate: MEDIAN (log_transform=True). 2026-07-23 21:05:51.834 | INFO | normalization:_select_auto_normalization:1328 - Evaluating Auto normalization candidate: PQN (log_transform=True). 2026-07-23 21:05:52.332 | INFO | normalization:_select_auto_normalization:1328 - Evaluating Auto normalization candidate: MDFC (log_transform=True). 2026-07-23 21:05:52.338 | INFO | normalization:calc_mdfc_normalization:1201 - Executing chunked parallel MDFC (backend='loky', cores=8, chunks=32)... 2026-07-23 21:05:54.930 | INFO | normalization:_select_auto_normalization:1328 - Evaluating Auto normalization candidate: QUANTILE (log_transform=True). 2026-07-23 21:05:55.443 | INFO | normalization:_select_auto_normalization:1328 - Evaluating Auto normalization candidate: VSN (log_transform=False). 2026-07-23 21:06:17.788 | INFO | normalization:_select_auto_normalization:1385 - Auto normalization candidate scores: ROBUST_LOG_ONLY=0.500, TIC=0.616, MEDIAN=0.521, PQN=0.714, MDFC=0.664, QUANTILE=0.399, VSN=0.514 2026-07-23 21:06:17.789 | INFO | normalization:_select_auto_normalization:1393 - Auto normalization selected PQN (score=0.714, margin=0.050). 2026-07-23 21:06:17.854 | INFO | normalization:execute_normalization:1518 - Auto normalization summary saved as: .\tutorial_output\06_Normalized_Data\Normalization_Auto_Summary.csv 2026-07-23 21:06:17.855 | INFO | normalization:execute_normalization:1521 - Generating diagnostic plots for normalization...
2026-07-23 21:06:21.314 | INFO | normalization:execute_normalization:1549 - Normalization summary dashboard saved as: .\tutorial_output\06_Normalized_Data\Normalization_Dashboard_PQN_Log2.svg 2026-07-23 21:06:21.315 | SUCCESS | normalization:execute_normalization:1550 - Data normalization completed successfully. 2026-07-23 21:06:21.315 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_normalization": 00:00:31.003. 2026-07-23 21:06:21.315 | SUCCESS | normalization:execute_normalization:1550 - Data normalization completed successfully. 2026-07-23 21:06:21.315 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_normalization": 00:00:31.003.
QA-Step 06: Quality Assessment on Normalized Data¶
Monitors spatial distribution fidelity using Jensen-Shannon Divergence (JSD) and Wasserstein metrics, evaluating whether the normalization successfully mitigated systematic biases without obliterating genuine biological differences.
In [14]:
logger.info("QA-Step 06: Quality Assessment of Normalized Data...")
qa_step6_dir = os.path.join(OUTPUT_DIR, "QA_06_Norm_Data")
qa_norm_engine = MetaboIntAssessor(data=normalized_data, pipeline_params=params)
qa_norm_engine.execute_assessment(output_dir=qa_step6_dir)
2026-07-23 21:06:21.342 | INFO | __main__:<module>:1 - QA-Step 06: Quality Assessment of Normalized Data... 2026-07-23 21:06:21.344 | WARNING | io_utils:_check_dir_exists:450 - No such directory, creating a new directory: .\tutorial_output\QA_06_Norm_Data.
2026-07-23 21:06:32.216 | INFO | assessment:execute_assessment:542 - Assessor summary dashboard saved as: .\tutorial_output\QA_06_Norm_Data\QA_Summary_Dashboard.svg 2026-07-23 21:06:32.217 | SUCCESS | assessment:execute_assessment:543 - Data quality assessment completed. 2026-07-23 21:06:32.217 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_assessment": 00:00:10.873. 2026-07-23 21:06:32.217 | SUCCESS | assessment:execute_assessment:543 - Data quality assessment completed. 2026-07-23 21:06:32.217 | SUCCESS | io_utils:time_wrap:479 - Execution time of "execute_assessment": 00:00:10.873.
Step 07: Sequential Audit Report Compilation¶
Assembles all intermediate vector graphics, mathematical metrics, and tracking logs to autonomously generate a comprehensive, human-readable HTML/PDF and Markdown audit report.
In [15]:
logger.info("Step 07: Sequential Audit Report Compilation...")
REPORT_DIR = "07_Report_Summary"
# 1. Process Visual Assets
visual_rep = ru.VisualAssetReporter(base_dir=OUTPUT_DIR)
visual_rep.compile_assessor_report(
report_folder=REPORT_DIR, is_multi_batch=is_multi_batch_flag
)
# 2. Dynamic QA Mapping
qa_corr_metrics = {}
stage_key_map = {
"Intra-batch corrected": "intra_batch_correction",
"Inter-batch corrected": "inter_batch_correction",
"SERRF": "global_correction",
"RUV-III": "global_correction",
"WaveICA 2.0": "global_correction",
}
for stage_name, engine in qa_engines_dict.items():
safe_key = stage_key_map.get(stage_name, "unknown_correction")
qa_corr_metrics[safe_key] = engine.assessment_metrics
# 3. Assemble Final Metrics
pipeline_metrics_objs = {
"raw_dataset": raw_data.dataset_metrics,
"high_mv_feature_filtering": mv_filter_data.mv_filtering_metrics,
"signal_correction": final_corr_data.correction_metrics,
"low_quality_feature_filtering":
low_quality_filter_data.quality_filtering_metrics,
"missing_value_imputation": imputed_data.imputation_metrics,
"normalization": normalized_data.normalization_metrics,
}
qa_metrics_objs = {
"raw_dataset": qa_raw_engine.assessment_metrics,
"high_mv_feature_filtering": qa_mv_filter_engine.assessment_metrics,
**qa_corr_metrics,
"low_quality_feature_filtering":
qa_low_quality_filter_engine.assessment_metrics,
"missing_value_imputation": qa_imp_engine.assessment_metrics,
"normalization": qa_norm_engine.assessment_metrics,
}
# 4. Initialize reporter and generate ONE markdown file
print(f"Initializing narrative reporter at workspace: {OUTPUT_DIR}")
md_reporter = ru.NarrativeStatsReporter(base_dir=OUTPUT_DIR)
md_reporter.generate_markdown(
pipeline_metrics=pipeline_metrics_objs,
qa_metrics=qa_metrics_objs,
report_folder=REPORT_DIR,
)
success = md_reporter.export_report(pdf_engine="weasyprint")
if success:
logger.success("PI-METABOQC PIPELINE COMPLETED SUCCESSFULLY.")
2026-07-23 21:06:32.248 | INFO | __main__:<module>:1 - Step 07: Sequential Audit Report Compilation... 2026-07-23 21:06:32.250 | INFO | report_utils:compile_assessor_report:324 - Multi-batch design detected. Assembling Batch Grid.
2026-07-23 21:06:34.252 | SUCCESS | report_utils:compile_assessor_report:369 - Report SVG assets compiled at: tutorial_output\07_Report_Summary\assets Initializing narrative reporter at workspace: .\tutorial_output 2026-07-23 21:06:34.363 | INFO | report_utils:generate_markdown:1265 - Generating COMPREHENSIVE narrative report... 2026-07-23 21:06:34.403 | SUCCESS | report_utils:generate_markdown:1275 - Comprehensive report generated: tutorial_output\07_Report_Summary\Report_Comprehensive.md 2026-07-23 21:06:34.404 | INFO | report_utils:generate_markdown:1265 - Generating BRIEF narrative report... 2026-07-23 21:06:34.417 | SUCCESS | report_utils:generate_markdown:1275 - Brief report generated: tutorial_output\07_Report_Summary\Report_Brief.md 2026-07-23 21:06:34.611 | INFO | report_utils:export_report:1426 - --- Exporting PDF for: Report_Comprehensive.md --- 2026-07-23 21:06:34.612 | INFO | report_utils:_render_weasyprint:1347 - Attempting PDF export via WeasyPrint... 2026-07-23 21:06:42.418 | SUCCESS | report_utils:_render_weasyprint:1370 - PDF generated: tutorial_output\07_Report_Summary\Report_Comprehensive.pdf 2026-07-23 21:06:42.419 | SUCCESS | report_utils:export_report:1454 - [Report_Comprehensive.md] completed using WeasyPrint. 2026-07-23 21:06:42.419 | INFO | report_utils:export_report:1426 - --- Exporting PDF for: Report_Brief.md --- 2026-07-23 21:06:42.420 | INFO | report_utils:_render_weasyprint:1347 - Attempting PDF export via WeasyPrint... 2026-07-23 21:06:44.701 | SUCCESS | report_utils:_render_weasyprint:1370 - PDF generated: tutorial_output\07_Report_Summary\Report_Brief.pdf 2026-07-23 21:06:44.702 | SUCCESS | report_utils:export_report:1454 - [Report_Brief.md] completed using WeasyPrint. 2026-07-23 21:06:44.704 | SUCCESS | __main__:<module>:58 - PI-METABOQC PIPELINE COMPLETED SUCCESSFULLY.
In [ ]: