Furen Xiao
33bc3c603f
Refactor the data loading and preprocessing pipeline to handle edge cases in medical imaging data, including NaN/Inf values, shape mismatches, and numerical instability during training. - Update `01_build_ntuh_manifest.py` with improved regex for T1c detection, spine exclusion, and deduplication logic based on acquisition timestamps. - Enhance `PatchDataset` in `src/dataset.py` to handle NaN/Inf values, clip intensity ranges, and ensure label/image shape alignment via padding/trimming. - Add a zero-gradient fallback in `src/training.py` to prevent DDP synchronization failures when encountering NaN/Inf losses. - Add `scripts/scan_procs.py` for process monitoring. - Increase DataLoader timeout to prevent hangs during heavy I/O.
40 lines
No EOL
1.4 KiB
Python
40 lines
No EOL
1.4 KiB
Python
"""Scan processed volumes for non-finite/corrupt data. Writes data/bad_procs.json."""
|
|
import os
|
|
import sys
|
|
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
|
import glob
|
|
import json
|
|
import numpy as np
|
|
import SimpleITK as sitk
|
|
from concurrent.futures import ProcessPoolExecutor, as_completed
|
|
|
|
|
|
def check(f):
|
|
try:
|
|
x = sitk.GetArrayFromImage(sitk.ReadImage(f))
|
|
return os.path.basename(f), bool(np.isfinite(x).all()), float(x.max()) if x.size else -1.0
|
|
except Exception as e: # noqa
|
|
return os.path.basename(f), False, repr(e)
|
|
|
|
|
|
def main():
|
|
files = sorted(glob.glob(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data", "proc", "*.nii.gz")))
|
|
print("checking", len(files), flush=True)
|
|
bad = []
|
|
with ProcessPoolExecutor(max_workers=48) as ex:
|
|
futs = [ex.submit(check, f) for f in files]
|
|
for i, fu in enumerate(as_completed(futs), 1):
|
|
k, ok, mx = fu.result()
|
|
if not ok or not np.isfinite(mx):
|
|
bad.append(k)
|
|
if i % 1000 == 0:
|
|
print(i, "checked", len(bad), "bad", flush=True)
|
|
print("BAD volumes:", len(bad))
|
|
for b in sorted(bad):
|
|
print(" ", b)
|
|
with open(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "data", "bad_procs.json"), "w") as f:
|
|
json.dump(sorted(bad), f, indent=1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |