Commit e5169e0e authored by Mark Hymers's avatar Mark Hymers

Initial code import from NAF

Signed-off-by: Mark Hymers's avatarMark Hymers <mark.hymers@ynic.york.ac.uk>
parents
This diff is collapsed.
# vim: set noexpandtab ts=4 sw=4:
# There must be a nicer way to do this
PYTHONARCH=$(shell utils/pythonarch)
BINS=$(shell python setup.py --list-scripts)
DEVELVERSION ?= 0.DEVEL.$(shell date +%Y%m%d%H%M)
RELEASEVERSION ?= $(DEVELVERSION)
all: $(PY4UICFILES)
python setup.py build
doc: doc-html
doc-html: all
python setup.py build_sphinx
clean:
python setup.py clean
# Stupid python setuptools don't even clean the build directory
rm -fr build
rm -fr doc/build
test: all
cd build && PYTHONPATH=lib.$(PYTHONARCH)/:$${PYTHONPATH} nosetests -c ../nose.cfg lib.$(PYTHONARCH)/anamnesis
testv: all
cd build && PYTHONPATH=lib.$(PYTHONARCH)/:$${PYTHONPATH} nosetests --with-coverage --cover-package=anamnesis -c ../nose.cfg -s -v lib.$(PYTHONARCH)/anamnesis
install:
python setup.py install --prefix=$(DESTDIR)/usr
mkdir -p $(DESTDIR)/usr/share/doc/anam
cp -a doc/build/html $(DESTDIR)/usr/share/doc/anam
.PHONY: clean doc doc-html all test testv pythonarch
anamnesis
=========
This repository contains the anamnesis python module. This is a python module
which enables easy serialisation of python objects to/from HDF5 files as well
as between machines using the Message Passing Interface (MPI) via mpi4py.
anamnesis was originally part of the Neuroimaging Analysis Framework (NAF)
which is available from https://vcs.ynic.york.ac.uk/naf. It was split out as a
separate module in order to allow for its use in other modules.
Authors
=======
Mark Hymers <mark.hymers@ynic.york.ac.uk>
License
=======
This project is currently licensed under the GNU General Public Licence 2.0
or higher. For alternative license arrangements, please contact the authors.
Dependencies
============
See the requirements.txt file.
Some of these aren't strict dependencies, but are instead what we develop
against (i.e. we don't guarantee not to use features which only exist from that
release onwards).
#!/usr/bin/python
# vim: set expandtab ts=4 sw=4:
from .abstract import * # noqa: F403
from .mpihandler import * # noqa: F403
from .options import AnamOptions # noqa: F401
from .register import find_class, register_class, ClassRegister # noqa: F401
This diff is collapsed.
#!/usr/bin/python
# vim: set expandtab ts=4 sw=4:
import sys
from numpy.random import randn
from .options import AnamOptions
__all__ = []
class MPIHandler(object):
__shared_state = {}
def __init__(self, *args, **kwargs):
# Quick way of implementing a singleton
self.__dict__ = self.__shared_state
if not getattr(self, 'initialised', False):
self.initialised = True
self._done = False
self.setup(*args, **kwargs)
# Register a cleanup routine
import atexit
atexit.register(self.atexit_handler)
def setup(self, use_mpi=False):
if use_mpi:
from .mpiimps.mpifourpy import MPI4PyImplementor
self.handler = MPI4PyImplementor()
else:
from .mpiimps.null import NullMPIImplementor
self.handler = NullMPIImplementor()
self.output_file = sys.stdout
def write(self, s):
"""
Write progress information to the appropriate output file
:param s: String to write
:type s: str
"""
if AnamOptions().verbose:
self.output_file.write(s)
self.output_file.flush()
def write_progress(self, s):
"""
Write progress information to the appropriate output file but only on
the root MPI node and only if the progress option is set
:param s: String to write
:type s: str
"""
if self.rank == 0 and AnamOptions().progress:
self.output_file.write(s)
self.output_file.flush()
def write_root(self, s):
"""
Write progress information to the appropriate output file but only on
the root MPI node
:param s: String to write
:type s: str
"""
if self.rank == 0:
self.write(s)
def get_rank(self):
return self.handler.rank
rank = property(get_rank)
def get_size(self):
return self.handler.size
size = property(get_size)
def get_master(self):
return self.handler.master
master = property(get_master)
def recv(self, obj=None, source=0, tag=0, status=None):
return self.handler.recv(obj, source, tag, status)
def send(self, obj=None, dest=0, tag=0):
return self.handler.send(obj, dest, tag)
def bcast(self, data_in=None, root=0):
return self.handler.bcast(data_in, root)
def get_scatter_indices(self, num_pts):
"""
Return a list of indices which would be used for scattering num_pts
across the nodes
"""
return self.handler.get_scatter_indices(num_pts)
def scatter_array(self, data_in=None, root=0):
return self.handler.scatter_array(data_in, root)
def scatter_list(self, data_in=None, root=0):
return self.handler.scatter_list(data_in, root)
def gather(self, data_in, root=0):
return self.handler.gather(data_in, root)
def allgather(self, data_in, root=0):
return self.handler.allgather(data_in, root)
def gather_list(self, data_in, total_trials, return_all=False):
return self.handler.gather_list(data_in, total_trials, return_all)
def done(self):
self._done = True
def atexit_handler(self):
if not self._done:
self.abort()
def abort(self):
self.handler.abort()
__all__.append('MPIHandler')
def mpi_print(txt):
print "NODE %s: %s" % (MPIHandler().rank, txt)
__all__.append('mpi_print')
# Some test code
if __name__ == '__main__':
import mpi4py
print mpi4py.__version__
m = MPIHandler(use_mpi=True)
if m.rank == 0:
data = randn(471, 100)
else:
data = None
data = m.scatter_array(data)
print "NODE %s, %s" % (m.rank, data.shape)
final = m.gather(data)
if final is not None:
print "GATHER NODE %s, %s" % (m.rank, final.shape)
else:
print "GATHER NODE %s NONE" % (m.rank)
finalall = m.allgather(data)
print "ALLGATHER NODE %s, %s" % (m.rank, finalall.shape)
m.done()
# vim: set expandtab ts=4 sw=4:
This diff is collapsed.
#!/usr/bin/python
# vim: set expandtab ts=4 sw=4:
__all__ = []
class NullMPIImplementor(object):
"""
Singleton fake MPI Class which doesn't even depend on the MPI module being
available
"""
__shared_state = {}
def __init__(self, *args, **kwargs):
# Quick way of implementing a singleton
self.__dict__ = self.__shared_state
if not getattr(self, 'initialised', False):
self.initialised = True
self.setup(*args, **kwargs)
def setup(self, use_mpi=False):
self.rank = 0
self.size = 1
self.master = True
def recv(self, obj=None, source=0, tag=0, status=None):
return obj
def send(self, obj=None, dest=0, tag=0):
return obj
def bcast(self, data_in=None, root=0):
return data_in
def get_scatter_indices(self, num_pts):
return [(0, num_pts,)]
def scatter_array(self, data_in=None, root=0):
return data_in
def scatter_list(self, data_in=None, root=0):
return data_in
def gather(self, data_in, root=0):
return data_in
def allgather(self, data_in, root=0):
return data_in
def gather_list(self, data_in, total_trials, return_all=False):
return data_in
def abort(self):
return
__all__.append('NullMPIImplementor')
#!/usr/bin/python
# vim: set expandtab ts=4 sw=4:
from sys import stdout
__all__ = []
class AnamOptions(object):
"""
Singleton class for holding system-wide options
"""
__shared_state = {}
def __init__(self, *args, **kwargs):
# Quick way of implementing a singleton
self.__dict__ = self.__shared_state
if not getattr(self, 'initialised', False):
self.initialised = True
self.setup(*args, **kwargs)
def setup(self):
self.verbose = 0
self.progress = 0
def write(self, s):
if self.verbose > 0:
stdout.write(s)
def write_progress(self, s):
if self.progress > 0:
stdout.write(s)
__all__.append('AnamOptions')
# vim: set expandtab ts=4 sw=4:
# A set of old hints which are used when initialising ClassRegister
_OLD_HINTS = {
'naf.meg.abstracttransforms.FourByFourTransform':
'naf.meg.transforms.FourByFourTransform',
'naf.meg.abstracttransforms.AsciiFourByFourTransform':
'naf.meg.transforms.AsciiFourByFourTransform',
'naf.meg.abstracttransforms.NiftiTransform':
'naf.meg.transforms.NiftiTransform',
'naf.meg.abstracttransforms.AffineCoordSet':
'naf.meg.transforms.AffineCoordSet',
'naf.meg.beamformers.BeamformerMetric':
'naf.meg.beamformeranalyses.BeamformerMetric',
'naf.meg.beamformers.NAI':
'naf.meg.beamformeranalyses.NAI',
'naf.meg.beamformers.BFWeights':
'naf.meg.beamformeranalyses.BFWeights',
'naf.meg.beamformers.BFTimeSeries':
'naf.meg.beamformeranalyses.BFTimeSeries',
'naf.meg.beamformers.BFTimeSeriesMetric':
'naf.meg.beamformeranalyses.BFTimeSeriesMetric',
'naf.meg.beamformers.BFTimeSeriesMeanVar':
'naf.meg.beamformeranalyses.BFTimeSeriesMeanVar',
'naf.meg.beamformers.BFTimeSeriesT2':
'naf.meg.beamformeranalyses.BFTimeSeriesT2',
'naf.meg.beamformers.NonRadOrientBase':
'naf.meg.orientselectors.NonRadOrientBase',
'naf.meg.beamformers.NonRadOrientSekihara':
'naf.meg.orientselectors.NonRadOrientSekihara',
'naf.meg.beamformers.NonRadOrientVanVeen':
'naf.meg.orientselectors.NonRadOrientVanVeen',
'naf.meg.beamformers.NonRadOrientPower':
'naf.meg.orientselectors.NonRadOrientPower',
'naf.meg.beamformers.NonRadOrientSekiBrute':
'naf.meg.orientselectors.NonRadOrientSekiBrute',
'naf.meg.beamformers.NonRadOrientVanVeenBrute2D':
'naf.meg.orientselectors.NonRadOrientVanVeenBrute2D',
'naf.meg.beamformers.NonRadOrientVanVeenBrute1D':
'naf.meg.orientselectors.NonRadOrientVanVeenBrute1D',
'naf.meg.beamformers.NonRadOrientPowerBrute':
'naf.meg.orientselectors.NonRadOrientPowerBrute',
'naf.meg.beamformers.FreeOrientBase':
'naf.meg.orientselectors.FreeOrientBase',
'naf.meg.beamformers.FreeOrientSekihara':
'naf.meg.orientselectors.FreeOrientSekihara',
'naf.meg.beamformers.FreeOrientVanVeen':
'naf.meg.orientselectors.FreeOrientVanVeen',
'naf.meg.beamformers.FreeOrientPower':
'naf.meg.orientselectors.FreeOrientPower',
'naf.meg.coils.CoilGroup':
'naf.meg.channels.ChannelGroup',
'naf.meg.coils.CoilSet':
'naf.meg.channels.ChannelSet',
'naf.meg.coils.CoilType':
'naf.meg.channels.ChannelType',
'naf.meg.coils.CoilName':
'naf.meg.channels.ChannelName',
'naf.meg.coils.DictCoilDefiner':
'naf.meg.channels.DictChannelDefiner',
'naf.meg.btireaders.BTICoilName':
'naf.meg.btireaders.BTIChannelName'}
class ClassRegister(object):
__shared_state = {}
def __init__(self, *args, **kwargs):
# Quick way of implementing a singleton
self.__dict__ = self.__shared_state
if not getattr(self, 'initialised', False):
self.initialised = True
self.class_register = {}
# For historical reasons, we allow naf and ourselves
self.permitted_prefixes = ['naf', 'anamnesis']
# For further historical reasons, we load some of the
# old NAF aliases by default to ease the transition to
# the broken out, general anamnesis library
self.hints = _OLD_HINTS.copy()
def add_permitted_prefix(self, prefix):
"""
Adds a permitted prefix to the automatic class finder
"""
self.permitted_prefixes.append(prefix)
def check_permitted_prefix(self, cname):
"""
Checks whether a class name starts with a permitted prefix
:param cname: class name to check
:returns: True or False
"""
for pp in self.permitted_prefixes:
if cname.startswith(pp):
return True
return False
def add_hint(self, oldname, newname):
"""
Adds a "hint" to the system.
This is useful if you have renamed a class and need to ensure
that old serialisation files will still load. The loading code
will automatically substitute the old name found in the file
with the new name given here
:param oldname: The old full name of the class as a string (e.g.
'naf.meg.abstracttransforms.FourByFourTransform'
:param newname: The new full name of the class as a string (e.g.
'naf.meg.transforms.FourByFourTransform'
"""
self.hints[oldname] = newname
def check_hint(self, cname):
"""
Looks up any hinted class name for the given input name.
:param cname: Class name to look for a hint for
:returns: None if not found or full class name of replacement class if
available
"""
return self.hints.get(cname, None)
def register_class(cls, name=None):
c = ClassRegister()
if name is None:
name = '.'.join([cls.__module__, cls.__name__])
if name in c.class_register:
raise StandardError('Name %s already registered' % name)
c.class_register[name] = cls
for aname in getattr(cls(), 'hdf5_aliases', []):
if aname in c.class_register:
raise StandardError('Alias name %s already registered' % aname)
c.class_register[aname] = cls
def find_class(name):
c = ClassRegister()
cls = c.class_register.get(name, None)
if cls is not None:
return cls
# Try our best to find and import it whilst not being entirely insecure
nc = name.split('.')
cnt = len(nc)
while cnt > 0:
try:
cname = '.'.join(nc[0:cnt])
# A vague attempt at preventing people abusing this to cause random
# modules to load.
if not c.check_permitted_prefix(cname):
return None
__import__(cname)
# Try to find it again
cls = c.class_register.get(name, None)
if cls is not None:
break
except ImportError:
# Walk backwards for Christmas
pass
cnt -= 1
if cls is not None:
return cls
# If we still haven't found it, look up our hints
hint = c.check_hint(name)
if hint is not None:
return find_class(hint)
return None
__all__ = ['register_class', 'find_class']
"""Basic data storage class"""
# vim: set expandtab ts=4 sw=4:
from .abstract import AbstractAnam
from .register import register_class
__all__ = []
class Store(AbstractAnam):
"""
A placeholder object which can have everything placed into extra_data
such that it can be serialised in and out simply.
Often used for storing things which don't need their own dedicated class.
"""
def __init__(self):
AbstractAnam.__init__(self)
def __eq__(self, other):
seenkeys = {}
for k in self.extra_data.keys():
seenkeys[k] = 1
if k in other.extra_data:
if self.extra_data[k] != other.extra_data[k]:
return False
else:
return False
for k in other.extra_data.keys():
if k not in seenkeys:
return False
return True
def __ne__(self, other):
return not self.__eq__(other)
__all__.append('Store')
register_class(Store)
# vim: set expandtab ts=4 sw=4:
from .support import anamtestpath, array_assert, AnamTestBase # noqa: F401
#!/usr/bin/python
# vim: set expandtab ts=4 sw=4:
import os
from os.path import split, join, isdir, isfile, abspath, dirname
import shutil
import tempfile
import unittest
__all__ = []
def find_path(varname, sentinal, subdir=None):
"""
Looks for a directory using the environment variable given and raises an
exception if it can't find the sentinal file.
If subdir is set, it adds a sub directory to the directory found in
the environment variable
"""
dir = os.environ.get(varname, None)
if not dir:
raise StandardError("%s is not set, cannot find test files" % varname)
if subdir is not None:
dir = join(dir, subdir)
if not isdir(dir):
raise StandardError("%s is not a directory" % varname)
# Our test file
if not isfile(join(dir, sentinal)):
raise StandardError("%s does not seem to contain sentinal "
"file %s" % (varname, sentinal))
return abspath(dir)
def anamtestpath():
"""Looks for the directory of anam test data relative to the
current file"""
d, f = split(__file__)
return join(abspath(d), 'data')