Commit 93a2dcc8 authored by Joe Lyons's avatar Joe Lyons 🚴
Browse files

Merge branch 'mh1/meg' into 'master'

Add support for MEGHDF5 data

Closes #13

See merge request ynic-public/yias!7
parents bfb22d41 ea4e2874
Pipeline #1357 passed with stage
in 12 minutes and 35 seconds
default:
image: debian:buster
image: ynic/debian:yias
before_script:
- apt -qq update && apt -qq install -y python3-numpy python3-dicom python3-nibabel python3-pytest python3-pytest-cov dcm2niix git
- apt -qq update && apt -qq install -y hdf5-tools python3-dateutil python3-h5py python3-numpy python3-dicom python3-nibabel python3-pytest python3-pytest-cov dcm2niix git
stages:
- build_test
......
[flake8]
max-line-length=120
[Unit]
Description=YIAS (MEG Scanner)
After=networking.target
[Service]
Type=simple
User=megdata
Group=megdata
UMask=0022
ExecStart=/srv/data/yias/yiasMEGProcessDaemon --log-level DEBUG /srv/data/incoming /srv/megdata-upload/pool
[Install]
WantedBy=multi-user.target
......@@ -5,9 +5,9 @@
from .destinations import *
from .dicom import *
from .errors import *
from .meghdf5 import *
from .nifti import *
from .physio import *
from .plugin_base import *
from .processor import *
from .utils import *
......@@ -78,9 +78,9 @@ __all__.append('Destination')
class RawAndAnonDestination(Destination):
regex_patient = re.compile('^[EeRr]\d+$')
regex_study = re.compile('^[CcPp]\d+[Aa]?([-:]\w+)?$')
regex_anon = re.compile('^[CcPp]\d+[Aa]([-:]\w+)?$')
regex_patient = re.compile(r'^[EeRr]\d+$')
regex_study = re.compile(r'^[CcPp]\d+[Aa]?([-:]\w+)?$')
regex_anon = re.compile(r'^[CcPp]\d+[Aa]([-:]\w+)?$')
anon_extra_studies = ['STRUCTURAL', 'T1ONLY', 'T1LONG']
def __init__(self, basedir=None, anon_dir=None, raw_dir=None, **kwargs):
......
......@@ -47,6 +47,16 @@ class YIASPhysioConversionError(YIASError):
__all__.append('YIASPhysioConversionError')
class YIASAnonError(YIASError):
# Holds some error information
def __init__(self, err): # pragma: nocover
self.err = err
def __str__(self): # pragma: nocover
return 'YIASAnonError: (%s)' % (self.err)
__all__.append('YIASAnonError')
class YIASSeriesPluginError(YIASError):
# Holds some error information
def __init__(self, err): # pragma: nocover
......@@ -66,3 +76,32 @@ class YIASStudyPluginError(YIASError):
return 'YIASStudyPluginError: (%s)' % (self.err)
__all__.append('YIASStudyPluginError')
class YIASMissingFileError(YIASError):
def __init__(self, filename):
self.filename = filename
def __str__(self): # pragma: nocover
return 'YIASMissingFileError: (%s)' % (self.filename)
__all__.append('YIASMissingFileError')
class YIASNotMEGHDF5Error(YIASError):
def __init__(self, filename, details):
self.filename = filename
self.details = details
def __str__(self): # pragma: nocover
return 'YIASNotMEGHDF5Error: (%s, %s)' % (self.filename, self.details)
__all__.append('YIASNotMEGHDF5Error')
class YIASMEGHDF5MetaMissing(YIASError):
def __init__(self, filename, details):
self.filename = filename
self.details = details
def __str__(self): # pragma: nocover
return 'YIASMEGHDF5MetaMissing: (%s, %s)' % (self.filename, self.details)
__all__.append('YIASMEGHDF5MetaMissing')
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>EditDialog</class>
<widget class="QDialog" name="EditDialog">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>490</width>
<height>142</height>
</rect>
</property>
<property name="windowTitle">
<string>Edit Scan Details</string>
</property>
<layout class="QGridLayout" name="gridLayout">
<item row="0" column="0" colspan="3">
<widget class="QLabel" name="filenameLabel">
<property name="text">
<string>Filename</string>
</property>
</widget>
</item>
<item row="1" column="0">
<widget class="QLabel" name="partLabel">
<property name="text">
<string>Participant ID:</string>
</property>
</widget>
</item>
<item row="4" column="1" colspan="2">
<widget class="QDialogButtonBox" name="buttonBox">
<property name="orientation">
<enum>Qt::Horizontal</enum>
</property>
<property name="standardButtons">
<set>QDialogButtonBox::Cancel|QDialogButtonBox::Save</set>
</property>
</widget>
</item>
<item row="1" column="1">
<widget class="QLineEdit" name="partOrig">
<property name="enabled">
<bool>false</bool>
</property>
</widget>
</item>
<item row="2" column="0">
<widget class="QLabel" name="scanLabel">
<property name="text">
<string>Scan ID:</string>
</property>
</widget>
</item>
<item row="2" column="1">
<widget class="QLineEdit" name="scanOrig">
<property name="enabled">
<bool>false</bool>
</property>
</widget>
</item>
<item row="2" column="2">
<widget class="QLineEdit" name="scanEdit"/>
</item>
<item row="1" column="2">
<widget class="QLineEdit" name="partEdit"/>
</item>
<item row="3" column="0" colspan="3">
<widget class="QLabel" name="label">
<property name="text">
<string>Note that Edit does not not rename the files on disk, it just edits the metadata</string>
</property>
</widget>
</item>
</layout>
</widget>
<resources/>
<connections>
<connection>
<sender>buttonBox</sender>
<signal>accepted()</signal>
<receiver>EditDialog</receiver>
<slot>accept()</slot>
<hints>
<hint type="sourcelabel">
<x>248</x>
<y>254</y>
</hint>
<hint type="destinationlabel">
<x>157</x>
<y>274</y>
</hint>
</hints>
</connection>
<connection>
<sender>buttonBox</sender>
<signal>rejected()</signal>
<receiver>EditDialog</receiver>
<slot>reject()</slot>
<hints>
<hint type="sourcelabel">
<x>316</x>
<y>260</y>
</hint>
<hint type="destinationlabel">
<x>286</x>
<y>274</y>
</hint>
</hints>
</connection>
</connections>
</ui>
<?xml version="1.0" encoding="UTF-8"?>
<ui version="4.0">
<class>MainWindow</class>
<widget class="QMainWindow" name="MainWindow">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>1258</width>
<height>781</height>
</rect>
</property>
<property name="windowTitle">
<string>YIAS Transfer GUI</string>
</property>
<widget class="QWidget" name="centralwidget">
<layout class="QVBoxLayout" name="verticalLayout">
<item>
<widget class="QTableView" name="scanView">
<property name="selectionMode">
<enum>QAbstractItemView::SingleSelection</enum>
</property>
</widget>
</item>
<item>
<widget class="QWidget" name="widget" native="true">
<layout class="QHBoxLayout" name="horizontalLayout">
<item>
<widget class="QPushButton" name="deleteButton">
<property name="text">
<string>&amp;Delete</string>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="editButton">
<property name="text">
<string>&amp;Edit</string>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="uploadButton">
<property name="text">
<string>&amp;Upload</string>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="refreshButton">
<property name="text">
<string>&amp;Refresh</string>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="exitButton">
<property name="text">
<string>E&amp;xit</string>
</property>
</widget>
</item>
</layout>
</widget>
</item>
</layout>
</widget>
<widget class="QMenuBar" name="menubar">
<property name="geometry">
<rect>
<x>0</x>
<y>0</y>
<width>1258</width>
<height>20</height>
</rect>
</property>
</widget>
<widget class="QStatusBar" name="statusbar"/>
</widget>
<resources/>
<connections/>
</ui>
#!/usr/bin/python3
from .protocol_copy import ProtocolCopy
from .transform_copy import TransformCopy
from .channel_name_fix import ChannelNameFix
#!/usr/bin/python3
import re
import logging
from ..plugin_base import MEGStudyPlugin
__all__ = []
RE_IONAME = re.compile('^s[0-9]{2}[lr]bf[01]-[0-9]{2}$')
class ChannelNameFix(MEGStudyPlugin):
"""Fixup channel names (reverse io_name and chan_name) if chan_names are of
the form sXX[lb]f[01]-[0-9]{2} (i.e the io_names)"""
def should_run_individual_file(self, hdf5):
"""
Check whether all channels in a single file match the regex
"""
run = True
# Only do this if all channels match the regex
for channel in hdf5['config']['channels']:
if RE_IONAME.search(channel) is None:
run = False
return run
def should_run(self, raw_hdf5, proc_hdf5):
"""
Check whether this is necessary / possible
"""
raw_need = False
proc_need = False
if raw_hdf5 is not None:
raw_need = self.should_run_individual_file(raw_hdf5)
if proc_hdf5 is not None:
proc_need = self.should_run_individual_file(raw_hdf5)
return raw_need or proc_need
def fix_up_file(self, hdf5):
"""
Fix up a single file
"""
chan_conf = hdf5['config']['channels']
sled_names = {}
for name in chan_conf:
if RE_IONAME.search(name) is None:
logging.error("ChannelNameFix: {} does not match pattern: skipping".format(name))
return False
if name in sled_names:
logging.error("ChannelNameFix: {} duplicate name: skipping".format(name))
return False
io_name = chan_conf[name].attrs['io_name']
if io_name in sled_names.values():
logging.error("ChannelNameFix: Duplicate io_name {} for sled {}: skipping".format(io_name, name))
return False
sled_names[name] = io_name
# Now re-map the following:
io_names = list(sled_names.values())
# 1. Channel names/groups in /config
for chan_name, io_name in sled_names.items():
# Make the io_name now be the sled_name
chan_conf[chan_name].attrs['io_name'] = chan_name
# Move the group
chan_conf.move(chan_name, io_name)
# 2. Channel names in channel_list in all acquisitions
for acqkey in hdf5['acquisitions']:
try:
# This is a painful way of checking whether we are a link
hdf5['acquisitions'].id.get_linkval(acqkey.encode('utf-8'))
logging.debug("ChannelNameFix: Skipping acq {} as it is a symlink".format(acqkey))
fixit = False
# Don't do anything if we get here
except ValueError:
fixit = True
if fixit:
# We're not a link - continue
logging.debug("ChannelNameFix: Fixing acq {}".format(acqkey))
for idx, chan_name in enumerate(hdf5['acquisitions'][acqkey]['channel_list']):
# Swap the channel name out
hdf5['acquisitions'][acqkey]['channel_list'][idx] = sled_names[chan_name]
# 3. Channel names in weights lists
for weights in hdf5['config']['weights']:
logging.debug("ChannelNameFix: Updating weights table {}".format(weights))
wtable = hdf5['config']['weights'][weights]
for idx, refchan in enumerate(wtable['ref_chans']):
wtable['ref_chans'][idx] = sled_names[refchan]
for idx, tgtchan in enumerate(wtable['tgt_chans']):
wtable['tgt_chans'][idx] = sled_names[tgtchan]
return True
def run(self, raw_hdf5, proc_hdf5, tmpdir):
"""
Executes the plugin.
raw_hdf5: h5py.File object or None
proc_hdf5: h5py.File object or None
tmpdir: Temporary directory to use if necessary
Will raise a YIASSeriesPluginError on an exception
Otherwise will return True if completed and False if nothing
needed to be done for one reason or another.
"""
from ..errors import YIASNiftiConversionError
# Run for each file individually
raw_ret = True
if self.should_run_individual_file(raw_hdf5):
logging.info("ChannelNameFix: called for raw file")
raw_ret = self.fix_up_file(raw_hdf5)
else:
logging.info("ChannelNameFix: not called for raw file")
proc_ret = True
if self.should_run_individual_file(proc_hdf5):
logging.info("ChannelNameFix: called for processed file")
proc_ret = self.fix_up_file(proc_hdf5)
else:
logging.info("ChannelNameFix: not called for processed file")
return raw_ret and proc_ret
__all__.append('TransformCopy')
#!/usr/bin/python3
import logging
from ..plugin_base import MEGStudyPlugin
__all__ = []
class ProtocolCopy(MEGStudyPlugin):
"""Copy the protocol information from processed to raw file if
missing in raw file and available in processed"""
def should_run(self, raw_hdf5, proc_hdf5):
"""
Check whether this is necessary / possible
"""
# Need both files for this
if raw_hdf5 is None or proc_hdf5 is None:
return False
# Check whether protocol info is already in the raw HDF5 file
if 'protocol' in raw_hdf5.get('config', {}):
return False
# Is protocol info available in processed file
if 'protocol' not in proc_hdf5.get('config', {}):
return False
return True
def run(self, raw_hdf5, proc_hdf5, tmpdir):
"""
Executes the plugin.
raw_hdf5: h5py.File object or None
proc_hdf5: h5py.File object or None
tmpdir: Temporary directory to use if necessary
Will raise a YIASSeriesPluginError on an exception
Otherwise will return True if completed and False if nothing
needed to be done for one reason or another.
"""
from ..errors import YIASNiftiConversionError
if not self.should_run(raw_hdf5, proc_hdf5): # pragma: nocover
logging.info("ProtocolCopy: called for invalid series")
return False
if 'config' not in raw_hdf5:
raw_hdf5.create_group('config')
if 'protocol' not in raw_hdf5['config']:
raw_hdf5['config'].create_group('protocol')
for field in ['protocol_name', 'initiating_user', 'definition']:
if field in proc_hdf5['config']['protocol'].attrs:
value = proc_hdf5['config']['protocol'].attrs[field]
raw_hdf5['config']['protocol'].attrs[field] = value
return True
__all__.append('ProtocolCopy')
#!/usr/bin/python3
import logging
from ..plugin_base import MEGStudyPlugin
__all__ = []
class TransformCopy(MEGStudyPlugin):
"""Copy the ccs_to_scs and fitted_coils information from processed to raw
file if missing in raw file and available in processed"""
def should_run(self, raw_hdf5, proc_hdf5):
"""
Check whether this is necessary / possible
"""
# Need both files for this
if raw_hdf5 is None or proc_hdf5 is None:
return False
shouldrun = False
# Find a list of ccs_to_scs_transform objects in the processed file and raw file
proc_trans = [x for x in sorted(proc_hdf5['acquisitions'].keys()) \
if x != 'default' and 'ccs_to_scs_transform' in proc_hdf5['acquisitions'][x]]
raw_trans = [x for x in sorted(raw_hdf5['acquisitions'].keys()) \
if x != 'default' and 'ccs_to_scs_transform' in raw_hdf5['acquisitions'][x]]
# If these are not the same, we have something to copy
if proc_trans != raw_trans:
shouldrun = True
# Find a list of fitted_coils objects in the processed file and raw file
proc_coils = [x for x in sorted(proc_hdf5['acquisitions'].keys()) \
if x != 'default' and 'fitted_coils' in proc_hdf5['acquisitions'][x]]
raw_coils = [x for x in sorted(raw_hdf5['acquisitions'].keys()) \
if x != 'default' and 'fitted_coils' in raw_hdf5['acquisitions'][x]]
# If these are not the same, we have something to copy
if proc_coils != raw_coils:
shouldrun = True
return shouldrun
def run(self, raw_hdf5, proc_hdf5, tmpdir):
"""
Executes the plugin.
raw_hdf5: h5py.File object or None
proc_hdf5: h5py.File object or None
tmpdir: Temporary directory to use if necessary
Will raise a YIASSeriesPluginError on an exception
Otherwise will return True if completed and False if nothing
needed to be done for one reason or another.
"""
from ..errors import YIASNiftiConversionError
if not self.should_run(raw_hdf5, proc_hdf5): # pragma: nocover
logging.info("TransformCopy: called for invalid series")
return False
for acq in raw_hdf5['acquisitions']:
# Skip default
if acq == 'default':
continue
# Only do this if we don't already have a copy
if 'ccs_to_scs_transform' not in raw_hdf5['acquisitions'][acq]:
# Copy the transform if it exists in proc_hdf5
if 'ccs_to_scs_transform' in proc_hdf5['acquisitions'][acq]:
data = proc_hdf5['acquisitions'][acq]['ccs_to_scs_transform']
raw_hdf5['acquisitions'][acq].create_dataset('ccs_to_scs_transform', data=data)
# And the same for fitted_coils
if 'fitted_coils' not in raw_hdf5['acquisitions'][acq]:
# Copy the data if it exists in proc_hdf5
if 'fitted_coils' in proc_hdf5['acquisitions'][acq]: