Commit 1f0519b5 authored by Mark Hymers's avatar Mark Hymers Committed by Joe Lyons
Browse files

Handle existing files properly


Signed-off-by: Mark Hymers's avatarMark Hymers <mark.hymers@hankel.co.uk>
parent cf17bed3
......@@ -5,9 +5,9 @@
from .destinations import *
from .dicom import *
from .errors import *
from .meghdf5 import *
from .nifti import *
from .physio import *
from .plugin_base import *
from .processor import *
from .utils import *
......@@ -271,6 +271,10 @@ class IncomingProcessor(object):
studydir = join(patientid,
study.StudyDate + '_' + studydesc)
# Needed below to make sure we clear out if needed
orig_raw_file = study.raw_file
orig_proc_file = study.proc_file
with TemporaryDirectory(dir=self.dest_tmpdir) as tmpdir:
filebname = '%s.meghdf5' % study.StudyDateTime
......@@ -290,9 +294,9 @@ class IncomingProcessor(object):
else:
logging.info("Raw %s copy going to %s" % (ftype, newfname))
makedirs_chmod(tgt_dir, mode=studydest.mode_raw_dir, exist_ok=True)
copy2(filename, newfname)
chmod(newfname, studydest.mode_raw_file)
makedirs_chmod(tgt_dir, mode=studydest.mode_raw_dir, exist_ok=True)
copy2(filename, newfname)
chmod(newfname, studydest.mode_raw_file)
# Now work out if we should anonymise
if self.force_anonymisation or studydest.anonymise(patientid, origstudydesc):
......@@ -300,77 +304,94 @@ class IncomingProcessor(object):
anon_dir = studydest.get_anon_directory(patientid, origstudydesc)
anondname = join(anon_dir, studydir)
# Make copy of both raw and processed files into temp dir if available
tmp_raw_path = None
tmp_raw_hdf5 = None
tmp_proc_path = None
tmp_proc_hdf5 = None
# Check whether our anonymous files exist
filebname = '%s.meghdf5' % study.StudyDateTime
if study.raw_file is not None:
tmp_raw_path = abspath(join(tmpdir, 'raw.hdf5'))
copy2(study.raw_file, tmp_raw_path)
tmp_raw_hdf5 = h5py.File(tmp_raw_path, 'a')
tgt_file = join(anondname, 'raw', filebname)
if exists(tgt_file):
# We already have the file - hack it so so we don't make it again
logging.info("Anonymised raw file %s already exists: skipping" % (tgt_file))
study.raw_file = None
if study.proc_file is not None:
tmp_proc_path = abspath(join(tmpdir, 'processed.hdf5'))
copy2(study.proc_file, tmp_proc_path)
tmp_proc_hdf5 = h5py.File(tmp_proc_path, 'a')
# Run MEG plugins
for plugin in MEGPlugins:
if plugin.should_run(tmp_raw_hdf5, tmp_proc_hdf5):
logging.info("Running MEG Plugin: %s" % (plugin.__class__.__name__))
try:
plugin.run(tmp_raw_hdf5, tmp_proc_hdf5, tmpdir)
except YIASSeriesPluginError as e: # pragma: nocover
logging.error("MEG Plugin Error: %s : %s" % (plugin.__class__.__name__, e))
else:
logging.info("Skipping MEG Plugin: %s" % (plugin.__class__.__name__))
# Close the HDF5 files
tmp_raw_hdf5.close()
tmp_proc_hdf5.close()
tgt_file = join(anondname, 'processed', filebname)
if exists(tgt_file):
# We already have the file - hack it so so we don't make it again
logging.info("Anonymised processed file %s already exists: skipping" % (tgt_file))
study.proc_file = None
# Only continue if we are missing at least one file
if study.raw_file or study.proc_file:
# Make copy of both raw and processed files into temp dir if available
tmp_raw_path = None
tmp_raw_hdf5 = None
tmp_proc_path = None
tmp_proc_hdf5 = None
# Run anonymisation on the file (do this last of all as it calls h5repack)
if study.raw_file is not None:
logging.info("Running anonymisation on raw file")
anonymise_hdf5(tmp_raw_path, patientid)
if study.proc_file is not None:
logging.info("Running anonymisation on processed file")
anonymise_hdf5(tmp_proc_path, patientid)
if study.raw_file is not None:
tmp_raw_path = abspath(join(tmpdir, 'raw.hdf5'))
copy2(study.raw_file, tmp_raw_path)
tmp_raw_hdf5 = h5py.File(tmp_raw_path, 'a')
# Copy final versions of the files into the right place
try:
logging.info("Anonymous copies going to %s" % anondname)
makedirs_chmod(anondname, mode=studydest.mode_anon_dir)
if study.proc_file is not None:
tmp_proc_path = abspath(join(tmpdir, 'processed.hdf5'))
copy2(study.proc_file, tmp_proc_path)
tmp_proc_hdf5 = h5py.File(tmp_proc_path, 'a')
# Run MEG plugins
for plugin in MEGPlugins:
if plugin.should_run(tmp_raw_hdf5, tmp_proc_hdf5):
logging.info("Running MEG Plugin: %s" % (plugin.__class__.__name__))
try:
plugin.run(tmp_raw_hdf5, tmp_proc_hdf5, tmpdir)
except YIASSeriesPluginError as e: # pragma: nocover
logging.error("MEG Plugin Error: %s : %s" % (plugin.__class__.__name__, e))
else:
logging.info("Skipping MEG Plugin: %s" % (plugin.__class__.__name__))
# Close the HDF5 files
tmp_raw_hdf5.close()
tmp_proc_hdf5.close()
# Run anonymisation on the file (do this last of all as it calls h5repack)
if study.raw_file is not None:
makedirs_chmod(join(anondname, 'raw'), mode=studydest.mode_anon_dir)
logging.info("Running anonymisation on raw file")
anonymise_hdf5(tmp_raw_path, patientid)
if study.proc_file is not None:
makedirs_chmod(join(anondname, 'processed'), mode=studydest.mode_anon_dir)
except OSError as e:
if e.errno == EEXIST:
logging.info("Anonymous copy already stored at %s - skipping" % anondname)
else: # pragma: nocover
raise(e)
logging.info("Running anonymisation on processed file")
anonymise_hdf5(tmp_proc_path, patientid)
# Copy final versions of the files into the right place
try:
logging.info("Anonymous copies going to %s" % anondname)
makedirs_chmod(anondname, mode=studydest.mode_anon_dir)
if study.raw_file is not None:
makedirs_chmod(join(anondname, 'raw'), mode=studydest.mode_anon_dir)
if study.proc_file is not None:
makedirs_chmod(join(anondname, 'processed'), mode=studydest.mode_anon_dir)
except OSError as e:
if e.errno == EEXIST:
logging.info("Could not make directories %s" % anondname)
else: # pragma: nocover
raise(e)
filebname = '%s.meghdf5' % study.StudyDateTime
if study.raw_file is not None:
tgt_dir = join(anondname, 'raw')
copy2(tmp_raw_path, join(tgt_dir, filebname))
if study.raw_file is not None:
tgt_dir = join(anondname, 'raw')
copy2(tmp_raw_path, join(tgt_dir, filebname))
if study.proc_file is not None:
tgt_dir = join(anondname, 'processed')
copy2(tmp_proc_path, join(tgt_dir, filebname))
if study.proc_file is not None:
tgt_dir = join(anondname, 'processed')
copy2(tmp_proc_path, join(tgt_dir, filebname))
if self.remove_incoming_after:
# Unlink the raw and processed original files
try:
if study.raw_file is not None:
unlink(study.raw_file)
if orig_raw_file is not None:
unlink(orig_raw_file)
if study.proc_file is not None:
unlink(study.proc_file)
if orig_proc_file is not None:
unlink(orig_proc_file)
except OSError as e:
logging.error("Error removing files from incoming ({})".format(e))
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment