From b12f5141e639e8076d7be0257ee4a75f7f7e1997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Thu, 2 Jan 2014 16:34:14 +0100 Subject: [PATCH 01/20] started developmnet --- blockedarray/CMakeLists.txt | 2 +- blockedarray/__init__.py | 3 +- blockedarray/adaptors.py | 37 +++++++++++++++++++ blockedarray/blockwisecc_py.cxx | 59 +++++++++++++++++++++++++++++++ blockedarray/test_blockedarray.py | 9 +++++ 5 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 blockedarray/adaptors.py diff --git a/blockedarray/CMakeLists.txt b/blockedarray/CMakeLists.txt index 2f746c5..1a54713 100644 --- a/blockedarray/CMakeLists.txt +++ b/blockedarray/CMakeLists.txt @@ -34,9 +34,9 @@ else() endif() target_link_libraries(_blockedarray + ${Boost_PYTHON_LIBRARIES} snappy ${PYTHON_LIBRARY} - ${Boost_PYTHON_LIBRARIES} ${HDF5_LIBRARY} ${HDF5_HL_LIBRARY} ${VIGRA_IMPEX_LIBRARY} diff --git a/blockedarray/__init__.py b/blockedarray/__init__.py index a9c83d9..b2f00bb 100644 --- a/blockedarray/__init__.py +++ b/blockedarray/__init__.py @@ -1,3 +1,2 @@ from _blockedarray import * - - +import adaptors diff --git a/blockedarray/adaptors.py b/blockedarray/adaptors.py new file mode 100644 index 0000000..8859d67 --- /dev/null +++ b/blockedarray/adaptors.py @@ -0,0 +1,37 @@ + +import numpy +from _blockedarray import Source3U8 as Source + + +class SourceABC(Source): + def __init__(self): + super(SourceABC, self).__init__() + + ''' + * selects only the region of interest given from the + * underlying data source. When readBlock() is used, the coordinates + * are relative to roi.q + ''' + def setRoi(self, roi): + raise NotImplementedError + #pass + + def shape(self): + raise NotImplementedError + #return (10, 10, 10) + + def readBlock(self, roi, output): + raise NotImplementedError + #return True + + +class TestSource(SourceABC): + def setRoi(self, roi): + pass + + def shape(self): + return numpy.asarray((100, 100, 10), dtype=numpy.long) + + def readBlock(self, roi, output): + output[...] = 0 + return True diff --git a/blockedarray/blockwisecc_py.cxx b/blockedarray/blockwisecc_py.cxx index fbd7a98..4810a60 100644 --- a/blockedarray/blockwisecc_py.cxx +++ b/blockedarray/blockwisecc_py.cxx @@ -129,12 +129,71 @@ void blockwiseCC() { class_("ConnectedComponents", init*, typename BCC::V>()) + .def(init*>()) + .def(init& >()) .def("writeResult", &BCC::writeResult, (arg("hdf5file"), arg("hdf5group"), arg("compression")=1)) ; } + +/* expose Source / Sink to python with inheritance callable from C++ */ +template +struct SourceWrap : Source, boost::python::wrapper > +{ +public: + + SourceWrap() {}; + SourceWrap(SourceWrap *other) {/* TODO */}; + + typedef boost::python::override override; + void setRoi(Roi roi) + { + if (override foo = this->get_override("setRoi")) + foo(roi); + Source::setRoi(roi); + } + + void default_setRoi(Roi roi) {this->Source::setRoi(roi);} + + typename Source::V shape() const + { + if (override foo = this->get_override("shape")) + return foo(); + return Source::shape(); + }; + + typename Source::V default_shape() const {return this->Source::shape();} + + bool readBlock(Roi roi, vigra::MultiArrayView& block) const + { + if (override foo = this->get_override("readBlock")) + return foo(roi, block); + return Source::readBlock(roi, block); + }; + + bool default_readBlock(Roi roi, vigra::MultiArrayView& block) const + { + return this->Source::readBlock(roi, block); + } +}; + +template +void exposeSource(const char* exposedName) { + using namespace boost::python; + + //implicitly_convertible(); + + class_, boost::noncopyable>(exposedName) + .def("setRoi", &Source::setRoi, &SourceWrap::default_setRoi) + .def("shape", &Source::shape, &SourceWrap::default_shape) + .def("readBlock", &Source::readBlock, &SourceWrap::default_readBlock) + ; + +} + void export_blockwiseCC() { blockwiseCC<2, float>(); blockwiseCC<3, float>(); + exposeSource<3,vigra::UInt8>("Source3U8"); } diff --git a/blockedarray/test_blockedarray.py b/blockedarray/test_blockedarray.py index aaf4a8f..372fde2 100644 --- a/blockedarray/test_blockedarray.py +++ b/blockedarray/test_blockedarray.py @@ -2,6 +2,7 @@ import vigra import h5py +from adaptors import TestSource from _blockedarray import * def rw(ba): @@ -74,6 +75,14 @@ def test1(): ba.setCompressionEnabled(True) ba.setCompressionEnabled(False) + +def test2(): + CC = dim3.ConnectedComponents + s = TestSource() + v = numpy.asarray((10, 10, 10), dtype=numpy.long) + cc = CC(s, v) + #cc = CC(s) + if __name__ == "__main__": test1() print "success" From 3ead3ee78f3992461414688ba15aa5ab8be4ecb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Fri, 3 Jan 2014 19:03:27 +0100 Subject: [PATCH 02/20] try again, fail better --- blockedarray/CMakeLists.txt | 1 + blockedarray/__init__.py | 2 +- blockedarray/{adaptors.py => adapters.py} | 15 +- blockedarray/adapters_py.cxx | 164 ++++++++++++++++++++++ blockedarray/adapters_py.h | 6 + blockedarray/blockwisecc_py.cxx | 59 +------- blockedarray/module_py.cxx | 6 +- blockedarray/test_blockedarray.py | 2 +- 8 files changed, 189 insertions(+), 66 deletions(-) rename blockedarray/{adaptors.py => adapters.py} (72%) create mode 100644 blockedarray/adapters_py.cxx create mode 100644 blockedarray/adapters_py.h diff --git a/blockedarray/CMakeLists.txt b/blockedarray/CMakeLists.txt index 1a54713..4b9ecb5 100644 --- a/blockedarray/CMakeLists.txt +++ b/blockedarray/CMakeLists.txt @@ -24,6 +24,7 @@ add_library(_blockedarray SHARED module_py.cxx blockedarray_py.cxx blockwisecc_py.cxx + adapters_py.cxx ${EXTRA_SRCS} ) diff --git a/blockedarray/__init__.py b/blockedarray/__init__.py index b2f00bb..2b375a5 100644 --- a/blockedarray/__init__.py +++ b/blockedarray/__init__.py @@ -1,2 +1,2 @@ from _blockedarray import * -import adaptors +import adapters diff --git a/blockedarray/adaptors.py b/blockedarray/adapters.py similarity index 72% rename from blockedarray/adaptors.py rename to blockedarray/adapters.py index 8859d67..2b40548 100644 --- a/blockedarray/adaptors.py +++ b/blockedarray/adapters.py @@ -12,26 +12,29 @@ def __init__(self): * underlying data source. When readBlock() is used, the coordinates * are relative to roi.q ''' - def setRoi(self, roi): + def pySetRoi(self, roi): raise NotImplementedError #pass - def shape(self): + def pyShape(self): raise NotImplementedError #return (10, 10, 10) - def readBlock(self, roi, output): + def pyReadBlock(self, roi, output): raise NotImplementedError #return True class TestSource(SourceABC): - def setRoi(self, roi): + def __init__(self): + super(TestSource, self).__init__() + + def pySetRoi(self, roi): pass - def shape(self): + def pyShape(self): return numpy.asarray((100, 100, 10), dtype=numpy.long) - def readBlock(self, roi, output): + def pyReadBlock(self, roi, output): output[...] = 0 return True diff --git a/blockedarray/adapters_py.cxx b/blockedarray/adapters_py.cxx new file mode 100644 index 0000000..3e2344d --- /dev/null +++ b/blockedarray/adapters_py.cxx @@ -0,0 +1,164 @@ +#define PY_ARRAY_UNIQUE_SYMBOL blockedarray_PyArray_API +#define NO_IMPORT_ARRAY + +#include "Python.h" + +#include +#include + +#include +#include + +#include +#include +#include + +#include "adapters_py.h" + +#include + +using namespace BW; + +/* + +template +class PySourceABC : public Source +{ +public: + + typedef typename Source::V TinyVec; + + PySourceABC() {}; + virtual ~PySourceABC() {}; + + void setRoi(Roi roi) + { + this->pySetRoi(roi); + } + + TinyVec shape() const + { + return this->pyShape(); + } + + bool readBlock(Roi roi, vigra::MultiArrayView& block) const + { + bool ret; + + //temporary NumpyArray + vigra::NumpyArray tempArray(roi.q-roi.p); + + if (!this->pyReadBlock(roi, tempArray)) + return false; + + //TODO copy data to MAV + + return true; + } + + virtual void pySetRoi(Roi roi) = 0; + virtual typename Source::V pyShape() const = 0; + virtual bool pyReadBlock(Roi roi, vigra::NumpyArray& block) const = 0; +}; +*/ + +/* expose Source / Sink to python with inheritance callable from C++ */ + +/* +template +struct PySourceABCWrap : PySourceABC, + boost::python::wrapper >, + boost::python::wrapper > +{ +public: + + typedef typename Source::V TinyVec; + + PySourceABCWrap() {}; + virtual ~PySourceABCWrap() {}; + + void pySetRoi(Roi roi) + { + this->get_override("pySetRoi")(roi); + } + + TinyVec pyShape() const + { + return this->get_override("pyShape")(); + }; + + bool pyReadBlock(Roi roi, vigra::NumpyArray& block) const + { + return this->get_override("pyReadBlock"); + }; +}; +*/ + +template +struct PySourceABC : Source, boost::python::wrapper > +{ +public: + + typedef typename Source::V TinyVec; + + PySourceABC() {}; + virtual ~PySourceABC() {}; + + void setRoi(Roi roi) + { + this->pySetRoi(roi); + } + + TinyVec shape() const + { + return this->pyShape(); + } + + bool readBlock(Roi roi, vigra::MultiArrayView& block) const + { + bool ret; + + //temporary NumpyArray + vigra::NumpyArray tempArray(roi.q-roi.p); + + if (!this->pyReadBlock(roi, tempArray)) + return false; + + //TODO copy data to MAV + + return true; + } + + void pySetRoi(Roi roi) + { + this->get_override("pySetRoi")(roi); + } + + TinyVec pyShape() const + { + return this->get_override("pyShape")(); + }; + + bool pyReadBlock(Roi roi, vigra::NumpyArray& block) const + { + return this->get_override("pyReadBlock"); + }; +}; + + + +template +void exposeSource(const char* exposedName) { + using namespace boost::python; + + class_, boost::noncopyable>(exposedName) + .def("pySetRoi", pure_virtual(&PySourceABC::pySetRoi)) + .def("pyShape", pure_virtual(&PySourceABC::pyShape)) + .def("pyReadBlock", pure_virtual(&PySourceABC::pyReadBlock)) + ; + +} + +void export_adapters() { + exposeSource<3,vigra::UInt8>("Source3U8"); +} diff --git a/blockedarray/adapters_py.h b/blockedarray/adapters_py.h new file mode 100644 index 0000000..e733953 --- /dev/null +++ b/blockedarray/adapters_py.h @@ -0,0 +1,6 @@ +#ifndef ADAPTERS_PY_H +#define ADAPTERS_PY_H + +void export_adapters(); + +#endif /* ADAPTERS_PY_H */ \ No newline at end of file diff --git a/blockedarray/blockwisecc_py.cxx b/blockedarray/blockwisecc_py.cxx index 4810a60..fd74df7 100644 --- a/blockedarray/blockwisecc_py.cxx +++ b/blockedarray/blockwisecc_py.cxx @@ -107,6 +107,8 @@ void blockwiseCC() { class_ >("Source", no_init); class_ >("Sink", no_init); +// class_ >("Source"); +// class_ >("Sink"); class_, bases > >("SourceHDF5", init()) @@ -129,71 +131,14 @@ void blockwiseCC() { class_("ConnectedComponents", init*, typename BCC::V>()) - .def(init*>()) - .def(init& >()) .def("writeResult", &BCC::writeResult, (arg("hdf5file"), arg("hdf5group"), arg("compression")=1)) ; } -/* expose Source / Sink to python with inheritance callable from C++ */ -template -struct SourceWrap : Source, boost::python::wrapper > -{ -public: - - SourceWrap() {}; - SourceWrap(SourceWrap *other) {/* TODO */}; - - typedef boost::python::override override; - void setRoi(Roi roi) - { - if (override foo = this->get_override("setRoi")) - foo(roi); - Source::setRoi(roi); - } - - void default_setRoi(Roi roi) {this->Source::setRoi(roi);} - - typename Source::V shape() const - { - if (override foo = this->get_override("shape")) - return foo(); - return Source::shape(); - }; - - typename Source::V default_shape() const {return this->Source::shape();} - - bool readBlock(Roi roi, vigra::MultiArrayView& block) const - { - if (override foo = this->get_override("readBlock")) - return foo(roi, block); - return Source::readBlock(roi, block); - }; - - bool default_readBlock(Roi roi, vigra::MultiArrayView& block) const - { - return this->Source::readBlock(roi, block); - } -}; - -template -void exposeSource(const char* exposedName) { - using namespace boost::python; - - //implicitly_convertible(); - - class_, boost::noncopyable>(exposedName) - .def("setRoi", &Source::setRoi, &SourceWrap::default_setRoi) - .def("shape", &Source::shape, &SourceWrap::default_shape) - .def("readBlock", &Source::readBlock, &SourceWrap::default_readBlock) - ; - -} void export_blockwiseCC() { blockwiseCC<2, float>(); blockwiseCC<3, float>(); - exposeSource<3,vigra::UInt8>("Source3U8"); } diff --git a/blockedarray/module_py.cxx b/blockedarray/module_py.cxx index 42ce85d..8bf933d 100644 --- a/blockedarray/module_py.cxx +++ b/blockedarray/module_py.cxx @@ -34,15 +34,19 @@ #include #include +#include #include "blockedarray_py.h" #include "blockwisecc_py.h" +#include "adapters_py.h" + BOOST_PYTHON_MODULE_INIT(_blockedarray) { - _import_array(); + //_import_array(); vigra::import_vigranumpy(); boost::python::numeric::array::set_module_and_type("numpy", "ndarray"); export_blockedArray(); export_blockwiseCC(); + export_adapters(); } \ No newline at end of file diff --git a/blockedarray/test_blockedarray.py b/blockedarray/test_blockedarray.py index 372fde2..f5fa594 100644 --- a/blockedarray/test_blockedarray.py +++ b/blockedarray/test_blockedarray.py @@ -2,7 +2,7 @@ import vigra import h5py -from adaptors import TestSource +from adapters import TestSource from _blockedarray import * def rw(ba): From ab3263bd2fb9c956cf99ec3156e694582f9e678a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Sat, 4 Jan 2014 18:48:04 +0100 Subject: [PATCH 03/20] getting there --- blockedarray/adapters.py | 6 +- blockedarray/adapters_py.cxx | 112 ++++++++++++------------------ blockedarray/blockwisecc_py.cxx | 2 + blockedarray/test_blockedarray.py | 6 +- include/bw/connectedcomponents.h | 5 ++ 5 files changed, 60 insertions(+), 71 deletions(-) diff --git a/blockedarray/adapters.py b/blockedarray/adapters.py index 2b40548..85cd680 100644 --- a/blockedarray/adapters.py +++ b/blockedarray/adapters.py @@ -25,9 +25,9 @@ def pyReadBlock(self, roi, output): #return True -class TestSource(SourceABC): +class ExampleSource(SourceABC): def __init__(self): - super(TestSource, self).__init__() + super(ExampleSource, self).__init__() def pySetRoi(self, roi): pass @@ -36,5 +36,7 @@ def pyShape(self): return numpy.asarray((100, 100, 10), dtype=numpy.long) def pyReadBlock(self, roi, output): + print(roi) + print(output) output[...] = 0 return True diff --git a/blockedarray/adapters_py.cxx b/blockedarray/adapters_py.cxx index 3e2344d..1654680 100644 --- a/blockedarray/adapters_py.cxx +++ b/blockedarray/adapters_py.cxx @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -19,10 +20,38 @@ using namespace BW; -/* + +/* ROI conversion */ +template +struct Roi_to_python_tuple +{ + typedef typename Roi::V TinyVec; + static PyObject* convert(const Roi &roi) + { + // p + boost::python::object iteratorP = boost::python::iterator()(roi.p); + boost::python::list p(iteratorP); + // q + boost::python::object iteratorQ = boost::python::iterator()(roi.q); + boost::python::list q(iteratorQ); + + boost::python::tuple t = boost::python::make_tuple(p, q); + return boost::python::incref(t.ptr()); + } +}; + +void registerConverters() +{ + boost::python::to_python_converter, Roi_to_python_tuple<1> >(); + boost::python::to_python_converter, Roi_to_python_tuple<2> >(); + boost::python::to_python_converter, Roi_to_python_tuple<3> >(); + boost::python::to_python_converter, Roi_to_python_tuple<4> >(); + boost::python::to_python_converter, Roi_to_python_tuple<5> >(); +} + template -class PySourceABC : public Source +struct PySourceABC : Source, boost::python::wrapper > { public: @@ -30,7 +59,7 @@ class PySourceABC : public Source PySourceABC() {}; virtual ~PySourceABC() {}; - + void setRoi(Roi roi) { this->pySetRoi(roi); @@ -45,38 +74,17 @@ class PySourceABC : public Source { bool ret; - //temporary NumpyArray + //temporary NumpyArray, because MultiArrayView is not convertible to python vigra::NumpyArray tempArray(roi.q-roi.p); if (!this->pyReadBlock(roi, tempArray)) return false; - //TODO copy data to MAV + block.copy(tempArray); return true; } - virtual void pySetRoi(Roi roi) = 0; - virtual typename Source::V pyShape() const = 0; - virtual bool pyReadBlock(Roi roi, vigra::NumpyArray& block) const = 0; -}; -*/ - -/* expose Source / Sink to python with inheritance callable from C++ */ - -/* -template -struct PySourceABCWrap : PySourceABC, - boost::python::wrapper >, - boost::python::wrapper > -{ -public: - - typedef typename Source::V TinyVec; - - PySourceABCWrap() {}; - virtual ~PySourceABCWrap() {}; - void pySetRoi(Roi roi) { this->get_override("pySetRoi")(roi); @@ -89,60 +97,31 @@ struct PySourceABCWrap : PySourceABC, bool pyReadBlock(Roi roi, vigra::NumpyArray& block) const { - return this->get_override("pyReadBlock"); + return this->get_override("pyReadBlock")(roi, block); }; }; -*/ template -struct PySourceABC : Source, boost::python::wrapper > -{ -public: - - typedef typename Source::V TinyVec; - - PySourceABC() {}; - virtual ~PySourceABC() {}; +class PySinkABC : Sink { + public: + typedef typename Roi::V V; - void setRoi(Roi roi) - { - this->pySetRoi(roi); - } - - TinyVec shape() const - { - return this->pyShape(); - } - - bool readBlock(Roi roi, vigra::MultiArrayView& block) const + PySinkABC() {}; + virtual ~PySinkABC() {}; + + bool writeBlock(Roi roi, const vigra::MultiArrayView& block) { bool ret; - //temporary NumpyArray - vigra::NumpyArray tempArray(roi.q-roi.p); + vigra::NumpyArray tempArray(block); - if (!this->pyReadBlock(roi, tempArray)) - return false; - - //TODO copy data to MAV - - return true; } - void pySetRoi(Roi roi) - { - this->get_override("pySetRoi")(roi); - } - - TinyVec pyShape() const + bool pyWriteBlock(Roi roi, const vigra::NumpyArray& block) { - return this->get_override("pyShape")(); + return this->get_override("pyReadBlock")(roi, block); }; - bool pyReadBlock(Roi roi, vigra::NumpyArray& block) const - { - return this->get_override("pyReadBlock"); - }; }; @@ -161,4 +140,5 @@ void exposeSource(const char* exposedName) { void export_adapters() { exposeSource<3,vigra::UInt8>("Source3U8"); + registerConverters(); } diff --git a/blockedarray/blockwisecc_py.cxx b/blockedarray/blockwisecc_py.cxx index fd74df7..8fbec4f 100644 --- a/blockedarray/blockwisecc_py.cxx +++ b/blockedarray/blockwisecc_py.cxx @@ -133,6 +133,8 @@ void blockwiseCC() { init*, typename BCC::V>()) .def("writeResult", &BCC::writeResult, (arg("hdf5file"), arg("hdf5group"), arg("compression")=1)) + .def("writeToSink", &BCC::writeToSink, + (arg("sink"), arg("blockShape"))) ; } diff --git a/blockedarray/test_blockedarray.py b/blockedarray/test_blockedarray.py index f5fa594..7d6c9ba 100644 --- a/blockedarray/test_blockedarray.py +++ b/blockedarray/test_blockedarray.py @@ -2,7 +2,7 @@ import vigra import h5py -from adapters import TestSource +from adapters import ExampleSource from _blockedarray import * def rw(ba): @@ -76,9 +76,9 @@ def test1(): ba.setCompressionEnabled(False) -def test2(): +def testConnectedComponents(): CC = dim3.ConnectedComponents - s = TestSource() + s = ExampleSource() v = numpy.asarray((10, 10, 10), dtype=numpy.long) cc = CC(s, v) #cc = CC(s) diff --git a/include/bw/connectedcomponents.h b/include/bw/connectedcomponents.h index bc8c704..6c1b5d4 100644 --- a/include/bw/connectedcomponents.h +++ b/include/bw/connectedcomponents.h @@ -271,6 +271,11 @@ class ConnectedComponents { std::cout << std::endl; out.close(); } + + void writeToSink(Sink* sink, V blockShape) //TODO blockShape needed? + { + return; + } private: Source* blockProvider_; From 51ed35628460a5dbeb4c2ce9c483c3268bbb370f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Sun, 5 Jan 2014 17:00:04 +0100 Subject: [PATCH 04/20] source and sink working in python --- blockedarray/adapters.py | 75 +++++++++++++++++++++--- blockedarray/adapters_py.cxx | 53 ++++++++++++++--- blockedarray/test_adapters.py | 52 ++++++++++++++++ blockedarray/test_blockedarray.py | 8 --- blockedarray/test_connectedcomponents.py | 38 ++++++++++++ include/bw/sink.h | 4 +- include/bw/source.h | 2 +- 7 files changed, 206 insertions(+), 26 deletions(-) create mode 100644 blockedarray/test_adapters.py create mode 100644 blockedarray/test_connectedcomponents.py diff --git a/blockedarray/adapters.py b/blockedarray/adapters.py index 85cd680..a9fa49a 100644 --- a/blockedarray/adapters.py +++ b/blockedarray/adapters.py @@ -1,6 +1,7 @@ -import numpy +import numpy as np from _blockedarray import Source3U8 as Source +from _blockedarray import Sink3U8 as Sink class SourceABC(Source): @@ -10,7 +11,7 @@ def __init__(self): ''' * selects only the region of interest given from the * underlying data source. When readBlock() is used, the coordinates - * are relative to roi.q + * are relative to roi[0] ''' def pySetRoi(self, roi): raise NotImplementedError @@ -25,18 +26,78 @@ def pyReadBlock(self, roi, output): #return True -class ExampleSource(SourceABC): +class SinkABC(Sink): def __init__(self): - super(ExampleSource, self).__init__() + super(SinkABC, self).__init__() + + def pyWriteBlock(self, roi, block): + raise NotImplementedError + #return True + + +class DummySource(SourceABC): + def __init__(self): + super(DummySource, self).__init__() def pySetRoi(self, roi): pass def pyShape(self): - return numpy.asarray((100, 100, 10), dtype=numpy.long) + return np.asarray((100, 100, 10), dtype=np.long) def pyReadBlock(self, roi, output): - print(roi) - print(output) output[...] = 0 return True + + +class ExampleSource(SourceABC): + def __init__(self, vol): + # the call to super() is super-important! + super(ExampleSource, self).__init__() + self._vol = vol + self._p = np.zeros((len(vol.shape),), dtype=np.long) + self._q = np.asarray(vol.shape, dtype=np.long) + + def pySetRoi(self, roi): + assert len(roi) == 2 + self._p = np.asarray(roi[0], dtype=np.long) + self._q = np.asarray(roi[1], dtype=np.long) + + def pyShape(self): + return self._vol.shape + + def pyReadBlock(self, roi, output): + assert len(roi) == 2 + roiP = np.asarray(roi[0]) + roiQ = np.asarray(roi[1]) + p = self._p + roiP + q = p + roiQ - roiP + if np.any(q > self._q): + raise IndexError("Requested roi is too large for selected " + "roi (previous call to setRoi)") + s = _roi2slice(p, q) + output[...] = self._vol[s] + return True + + +class ExampleSink(SinkABC): + def __init__(self): + super(ExampleSink, self).__init__() + self.vol = None + + def pyWriteBlock(self, roi, block): + assert len(roi) == 2 + if self.vol is None: + shape = _v2tup(self.shape) + self.vol = np.zeros(shape, dtype=np.uint8) + s = _roi2slice(roi[0], roi[1]) + self.vol[s] = block + + +def _v2tup(v, d=3): + return tuple([v[i] for i in range(d)]) + + +def _roi2slice(p, q): + s = [slice(p[i], q[i]) for i in range(len(p))] + return tuple(s) diff --git a/blockedarray/adapters_py.cxx b/blockedarray/adapters_py.cxx index 1654680..669eb24 100644 --- a/blockedarray/adapters_py.cxx +++ b/blockedarray/adapters_py.cxx @@ -22,6 +22,14 @@ using namespace BW; /* ROI conversion */ + +template +boost::python::list tinyVecToList(const typename Roi::V &vec) +{ + boost::python::object iterator = boost::python::iterator::V>()(vec); + return boost::python::list(iterator); +} + template struct Roi_to_python_tuple { @@ -29,11 +37,9 @@ struct Roi_to_python_tuple static PyObject* convert(const Roi &roi) { // p - boost::python::object iteratorP = boost::python::iterator()(roi.p); - boost::python::list p(iteratorP); + boost::python::list p = tinyVecToList(roi.p); // q - boost::python::object iteratorQ = boost::python::iterator()(roi.q); - boost::python::list q(iteratorQ); + boost::python::list q = tinyVecToList(roi.q); boost::python::tuple t = boost::python::make_tuple(p, q); return boost::python::incref(t.ptr()); @@ -102,7 +108,7 @@ struct PySourceABC : Source, boost::python::wrapper > }; template -class PySinkABC : Sink { +struct PySinkABC : Sink, boost::python::wrapper > { public: typedef typename Roi::V V; @@ -111,17 +117,35 @@ class PySinkABC : Sink { bool writeBlock(Roi roi, const vigra::MultiArrayView& block) { - bool ret; - vigra::NumpyArray tempArray(block); - + return this->pyWriteBlock(roi, tempArray); } bool pyWriteBlock(Roi roi, const vigra::NumpyArray& block) { - return this->get_override("pyReadBlock")(roi, block); + return this->get_override("pyWriteBlock")(roi, block); }; + + /* Accessor functions for the shapes */ + boost::python::list getShape() const + { + return tinyVecToList(this->shape_); + } + + void setShape(V shape) + { + this->shape_ = shape; + } + boost::python::list getBlockShape() const + { + return tinyVecToList(this->blockShape_); + } + + void setBlockShape(V shape) + { + this->blockShape_ = shape; + } }; @@ -135,10 +159,21 @@ void exposeSource(const char* exposedName) { .def("pyShape", pure_virtual(&PySourceABC::pyShape)) .def("pyReadBlock", pure_virtual(&PySourceABC::pyReadBlock)) ; +} + +template +void exposeSink(const char* exposedName) { + using namespace boost::python; + class_, boost::noncopyable>(exposedName) + .def("pyWriteBlock", pure_virtual(&PySinkABC::pyWriteBlock)) + .add_property("shape", &PySinkABC::getShape, &PySinkABC::setShape) + .add_property("blockShape", &PySinkABC::getBlockShape, &PySinkABC::setBlockShape) + ; } void export_adapters() { exposeSource<3,vigra::UInt8>("Source3U8"); + exposeSink<3,vigra::UInt8>("Sink3U8"); registerConverters(); } diff --git a/blockedarray/test_adapters.py b/blockedarray/test_adapters.py new file mode 100644 index 0000000..2bbfdaf --- /dev/null +++ b/blockedarray/test_adapters.py @@ -0,0 +1,52 @@ +import numpy as np +import unittest + +from adapters import ExampleSource, ExampleSink + + +class TestSource(unittest.TestCase): + + def setUp(self): + self.vol = np.random.randint(0, 2**32, (100, 100, 10)).astype(np.uint32) + + def testExampleSource(self): + vol = self.vol + s = ExampleSource(vol) + + roi = [np.zeros((len(vol.shape),)), vol.shape] + newVol = np.zeros(vol.shape, dtype=np.uint32) + assert s.pyReadBlock(roi, newVol) + assert np.all(vol == newVol) + + def testRoi(self): + vol = self.vol + s = ExampleSource(vol) + + reqRoi = [(50, 50, 2), (70, 70, 4)] + s.pySetRoi(reqRoi) + roi = [(0, 0, 0), (20, 20, 2)] + newVol = np.zeros((20, 20, 2), dtype=np.uint32) + assert s.pyReadBlock(roi, newVol) + assert np.all(vol[50:70, 50:70, 2:4] == newVol) + + def testRoi2(self): + vol = self.vol + s = ExampleSource(vol) + roi = [(0, 0, 2), (20, 20, 4)] + newVol = np.zeros((20, 20, 2), dtype=np.uint32) + assert s.pyReadBlock(roi, newVol) + + +class TestSink(unittest.TestCase): + + def setUp(self): + pass + + def testExampleSink(self): + s = ExampleSink() + s.shape = (100, 100, 10) + s.blockShape = (10, 10, 10) + s.pyWriteBlock([(15, 20, 2), (30, 30, 6)], + np.ones((15, 10, 4), dtype=np.uint8)) + + assert np.all(s.vol[15:30, 20:30, 2:6] == 1) diff --git a/blockedarray/test_blockedarray.py b/blockedarray/test_blockedarray.py index 7d6c9ba..42ee75e 100644 --- a/blockedarray/test_blockedarray.py +++ b/blockedarray/test_blockedarray.py @@ -2,7 +2,6 @@ import vigra import h5py -from adapters import ExampleSource from _blockedarray import * def rw(ba): @@ -76,13 +75,6 @@ def test1(): ba.setCompressionEnabled(False) -def testConnectedComponents(): - CC = dim3.ConnectedComponents - s = ExampleSource() - v = numpy.asarray((10, 10, 10), dtype=numpy.long) - cc = CC(s, v) - #cc = CC(s) - if __name__ == "__main__": test1() print "success" diff --git a/blockedarray/test_connectedcomponents.py b/blockedarray/test_connectedcomponents.py new file mode 100644 index 0000000..73ae126 --- /dev/null +++ b/blockedarray/test_connectedcomponents.py @@ -0,0 +1,38 @@ +import numpy as np +import unittest + +from adapters import DummySource, ExampleSource +from _blockedarray import dim3 + + +class TestConnectedComponents(unittest.TestCase): + + def setUp(self): + shape = (100, 100, 100) + self.blockShape = (10, 10, 10) + vol = np.zeros(shape, dtype=np.uint8) + vol[7:12, 7:12, 7:12] = 42 + vol[:, :, 72:75] = 111 + vol[:, 72:75, :] = 111 + self.vol = vol + + def testBlockShape(self): + CC = dim3.ConnectedComponents + s = DummySource() + + v = (10, 10, 10) + cc = CC(s, v) + + v = np.asarray((10, 10, 10)) + cc = CC(s, v) + + v = np.asarray((10, 10, 10), dtype=np.long) + cc = CC(s, v) + + v = np.asarray((10, 10, 10), dtype=np.int) + cc = CC(s, v) + + def testCC(self): + CC = dim3.ConnectedComponents + s = ExampleSource(self.vol) + assert CC(s, self.blockShape) diff --git a/include/bw/sink.h b/include/bw/sink.h index 1308aa6..85e7ff3 100644 --- a/include/bw/sink.h +++ b/include/bw/sink.h @@ -44,13 +44,15 @@ class Sink { public: typedef typename Roi::V V; - Sink() {} + Sink() {}; virtual ~Sink() {}; + /* has to be called before any calls to writeBlock */ void setShape(V shape) { shape_ = shape; } + /* has to be called before any calls to writeBlock */ void setBlockShape(V shape) { blockShape_ = shape; } diff --git a/include/bw/source.h b/include/bw/source.h index 0f8a92c..910d835 100644 --- a/include/bw/source.h +++ b/include/bw/source.h @@ -44,7 +44,7 @@ class Source { public: typedef typename Roi::V V; - Source() {} + Source() {}; virtual ~Source() {}; /** From f4ca707be73716efde9db472cf2e08176ae4a77c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Mon, 6 Jan 2014 13:51:37 +0100 Subject: [PATCH 05/20] python interface for cc complete --- blockedarray/adapters.py | 2 +- blockedarray/adapters_py.cxx | 5 +- blockedarray/blockwisecc_py.cxx | 2 +- blockedarray/test_connectedcomponents.py | 65 +++++++++++++++++++++--- include/bw/connectedcomponents.h | 19 ++++++- 5 files changed, 80 insertions(+), 13 deletions(-) diff --git a/blockedarray/adapters.py b/blockedarray/adapters.py index a9fa49a..0c7a095 100644 --- a/blockedarray/adapters.py +++ b/blockedarray/adapters.py @@ -1,7 +1,7 @@ import numpy as np from _blockedarray import Source3U8 as Source -from _blockedarray import Sink3U8 as Sink +from _blockedarray import Sink3 as Sink class SourceABC(Source): diff --git a/blockedarray/adapters_py.cxx b/blockedarray/adapters_py.cxx index 669eb24..356e7ce 100644 --- a/blockedarray/adapters_py.cxx +++ b/blockedarray/adapters_py.cxx @@ -53,6 +53,9 @@ void registerConverters() boost::python::to_python_converter, Roi_to_python_tuple<3> >(); boost::python::to_python_converter, Roi_to_python_tuple<4> >(); boost::python::to_python_converter, Roi_to_python_tuple<5> >(); + + // I really don't know why this is not called in vigra + vigra::NumpyArrayConverter::LabelType> >(); } @@ -174,6 +177,6 @@ void exposeSink(const char* exposedName) { void export_adapters() { exposeSource<3,vigra::UInt8>("Source3U8"); - exposeSink<3,vigra::UInt8>("Sink3U8"); + exposeSink<3,ConnectedComponents<3>::LabelType>("Sink3"); registerConverters(); } diff --git a/blockedarray/blockwisecc_py.cxx b/blockedarray/blockwisecc_py.cxx index 8fbec4f..5357fad 100644 --- a/blockedarray/blockwisecc_py.cxx +++ b/blockedarray/blockwisecc_py.cxx @@ -134,7 +134,7 @@ void blockwiseCC() { .def("writeResult", &BCC::writeResult, (arg("hdf5file"), arg("hdf5group"), arg("compression")=1)) .def("writeToSink", &BCC::writeToSink, - (arg("sink"), arg("blockShape"))) + (arg("sink"))) ; } diff --git a/blockedarray/test_connectedcomponents.py b/blockedarray/test_connectedcomponents.py index 73ae126..ba122c3 100644 --- a/blockedarray/test_connectedcomponents.py +++ b/blockedarray/test_connectedcomponents.py @@ -1,23 +1,29 @@ -import numpy as np import unittest +import tempfile +import os + +import numpy as np +from numpy.testing import assert_almost_equal +import vigra -from adapters import DummySource, ExampleSource +from adapters import DummySource, ExampleSource, ExampleSink from _blockedarray import dim3 +CC = dim3.ConnectedComponents + class TestConnectedComponents(unittest.TestCase): def setUp(self): - shape = (100, 100, 100) + shape = (100, 110, 120) self.blockShape = (10, 10, 10) vol = np.zeros(shape, dtype=np.uint8) - vol[7:12, 7:12, 7:12] = 42 + vol[7:12, 17:22, 27:32] = 42 vol[:, :, 72:75] = 111 vol[:, 72:75, :] = 111 self.vol = vol def testBlockShape(self): - CC = dim3.ConnectedComponents s = DummySource() v = (10, 10, 10) @@ -33,6 +39,49 @@ def testBlockShape(self): cc = CC(s, v) def testCC(self): - CC = dim3.ConnectedComponents - s = ExampleSource(self.vol) - assert CC(s, self.blockShape) + tmp = tempfile.NamedTemporaryFile(suffix='.h5', delete=False) + try: + h5f = (tmp.name, "/data") + s = ExampleSource(self.vol) + cc = CC(s, self.blockShape) + + cc.writeResult(h5f[0], h5f[1]) + labels = vigra.readHDF5(h5f[0], h5f[1]) + finally: + pass + os.remove(h5f[0]) + + # vigra and hdf5 have incompatible axis order, see + # http://ukoethe.github.io/vigra/doc/vigra/classvigra_1_1HDF5File.html#a638be2d51070009b02fed0f234f4b0bf + labels = np.transpose(labels, (2, 1, 0)) + + self._check(labels) + + def testCCsink(self): + source = ExampleSource(self.vol) + sink = ExampleSink() + cc = CC(source, self.blockShape) + cc.writeToSink(sink) + + self._check(sink.vol) + + def _check(self, labels): + comp1 = np.concatenate((labels[:, :, 72:75].flatten(), + labels[:, 72:75, :].flatten())) + comp2 = labels[7:12, 17:22, 27:32].flatten() + + # assert that each component has exactly one positive label + assert_almost_equal(comp1.var(), 0) + assert np.all(comp1 > 0) + assert_almost_equal(comp2.var(), 0) + assert np.all(comp2 > 0) + + # assert that each component is labeled with a unique label + assert comp1[0] != comp2[0] + + # assert that no other pixel is labeled + labels[:, :, 72:75] = 0 + labels[:, 72:75, :] = 0 + labels[7:12, 17:22, 27:32] = 0 + assert_almost_equal(labels.sum(), 0) + diff --git a/include/bw/connectedcomponents.h b/include/bw/connectedcomponents.h index 6c1b5d4..b853a84 100644 --- a/include/bw/connectedcomponents.h +++ b/include/bw/connectedcomponents.h @@ -255,6 +255,7 @@ class ConnectedComponents { vigra_precondition(compression >= 1 && compression <= 9, "compression must be >= 1 and <= 9"); std::cout << "* write " << hdf5file << "/" << hdf5group << std::endl; + HDF5File out(hdf5file, HDF5File::Open); out.createDataset(hdf5group, blockProvider_->shape(), 0, blockShape_, compression); for(size_t i=0; i* sink, V blockShape) //TODO blockShape needed? + void writeToSink(Sink* sink) { - return; + std::cout << "* writing to sink" << std::endl; + + sink->setShape(blockProvider_->shape()); + sink->setBlockShape(blockShape_); + + for(size_t i=0; i& roi = blockRois_[i].second; + + vigra::MultiArray cc(roi.q-roi.p); + ccBlocks_[i].readArray(cc); + + sink->writeBlock(roi, cc); + } + std::cout << std::endl; } private: From f808dc4fc2e6bef8ac4865baa30898d0ee39e829 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Mon, 20 Jan 2014 12:20:33 +0100 Subject: [PATCH 06/20] removed output to stdout --- include/bw/connectedcomponents.h | 36 ++++++++++++++++---------------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/include/bw/connectedcomponents.h b/include/bw/connectedcomponents.h index b853a84..916fc4b 100644 --- a/include/bw/connectedcomponents.h +++ b/include/bw/connectedcomponents.h @@ -139,13 +139,13 @@ class ConnectedComponents { vector offsets(blockRois_.size()+1); ccBlocks_.resize(blockRois_.size()); - std::cout << "* run connected components on each of " << blockRois_.size() << " blocks separately" << std::endl; +// std::cout << "* run connected components on each of " << blockRois_.size() << " blocks separately" << std::endl; size_t sizeBytes = 0; size_t sizeBytesUncompressed = 0; for(size_t i=0; i& block = blockRois_[i].second; //std::cout << " - CC on block " << block << std::endl; @@ -175,18 +175,18 @@ class ConnectedComponents { offsets[i+1] = offsets[i] + maxLabel + 1; } - std::cout << std::endl; - std::cout << " " << sizeBytes/(1024.0*1024.0) << " MB (vs. " << sizeBytesUncompressed/(1024.0*1024.0) << "MB uncompressed)" << std::endl; +// std::cout << std::endl; +// std::cout << " " << sizeBytes/(1024.0*1024.0) << " MB (vs. " << sizeBytesUncompressed/(1024.0*1024.0) << "MB uncompressed)" << std::endl; LabelType maxOffset = offsets[offsets.size()-1]; - std::cout << " initialize union find datastructure with maxOffset = " << maxOffset << std::endl; +// std::cout << " initialize union find datastructure with maxOffset = " << maxOffset << std::endl; UnionFindArray ufd(maxOffset); // // merge adjacent blocks // - std::cout << "* merge adjacent blocks" << std::endl; +// std::cout << "* merge adjacent blocks" << std::endl; //FIXME: use adjacency relation of blocks std::vector< std::pair > adjBlocks; @@ -202,7 +202,7 @@ class ConnectedComponents { } for(vector >::iterator it=adjBlocks.begin(); it!=adjBlocks.end(); ++it) { - std::cout << " pair " << std::distance(adjBlocks.begin(), it)+1 << "/" << adjBlocks.size() << " \r" << std::flush; +// std::cout << " pair " << std::distance(adjBlocks.begin(), it)+1 << "/" << adjBlocks.size() << " \r" << std::flush; size_t i = it->first; size_t j = it->second; @@ -220,15 +220,15 @@ class ConnectedComponents { block1, offsets[i], cc1, block2, offsets[j], cc2); } - std::cout << std::endl; +// std::cout << std::endl; LabelType maxLabel = ufd.makeContiguous(); - std::cout << "* relabel" << std::endl; +// std::cout << "* relabel" << std::endl; sizeBytes = 0; sizeBytesUncompressed = 0; for(size_t i=0; i& roi = blockRois_[i].second; //MultiArray cc = ccBlocks_[i]; @@ -244,8 +244,8 @@ class ConnectedComponents { sizeBytes += ccBlocks_[i].currentSizeBytes(); sizeBytesUncompressed += ccBlocks_[i].uncompressedSizeBytes(); } - std::cout << std::endl; - std::cout << " " << sizeBytes/(1024.0*1024.0) << " MB (vs. " << sizeBytesUncompressed/(1024.0*1024.0) << "MB uncompressed)" << std::endl; +// std::cout << std::endl; +// std::cout << " " << sizeBytes/(1024.0*1024.0) << " MB (vs. " << sizeBytesUncompressed/(1024.0*1024.0) << "MB uncompressed)" << std::endl; } @@ -254,12 +254,12 @@ class ConnectedComponents { vigra_precondition(compression >= 1 && compression <= 9, "compression must be >= 1 and <= 9"); - std::cout << "* write " << hdf5file << "/" << hdf5group << std::endl; +// std::cout << "* write " << hdf5file << "/" << hdf5group << std::endl; HDF5File out(hdf5file, HDF5File::Open); out.createDataset(hdf5group, blockProvider_->shape(), 0, blockShape_, compression); for(size_t i=0; i& roi = blockRois_[i].second; //cc = ccBlocks_[i]; @@ -269,19 +269,19 @@ class ConnectedComponents { out.writeBlock(hdf5group, roi.p, cc); } - std::cout << std::endl; +// std::cout << std::endl; out.close(); } void writeToSink(Sink* sink) { - std::cout << "* writing to sink" << std::endl; +// std::cout << "* writing to sink" << std::endl; sink->setShape(blockProvider_->shape()); sink->setBlockShape(blockShape_); for(size_t i=0; i& roi = blockRois_[i].second; vigra::MultiArray cc(roi.q-roi.p); @@ -289,7 +289,7 @@ class ConnectedComponents { sink->writeBlock(roi, cc); } - std::cout << std::endl; +// std::cout << std::endl; } private: From c1c1a24c2fb99e19992d0dc92a8d914f9213e231 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Mon, 20 Jan 2014 19:22:42 +0100 Subject: [PATCH 07/20] ubuntu does not know how to convert ndarray to tinyvector --- blockedarray/adapters.py | 2 +- blockedarray/adapters_py.cxx | 1 + blockedarray/test_adapters.py | 2 +- blockedarray/test_connectedcomponents.py | 2 ++ 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/blockedarray/adapters.py b/blockedarray/adapters.py index 0c7a095..0e73e7e 100644 --- a/blockedarray/adapters.py +++ b/blockedarray/adapters.py @@ -43,7 +43,7 @@ def pySetRoi(self, roi): pass def pyShape(self): - return np.asarray((100, 100, 10), dtype=np.long) + return (100, 100, 10) def pyReadBlock(self, roi, output): output[...] = 0 diff --git a/blockedarray/adapters_py.cxx b/blockedarray/adapters_py.cxx index 356e7ce..e3c448b 100644 --- a/blockedarray/adapters_py.cxx +++ b/blockedarray/adapters_py.cxx @@ -56,6 +56,7 @@ void registerConverters() // I really don't know why this is not called in vigra vigra::NumpyArrayConverter::LabelType> >(); + vigra::NumpyArrayConverter >(); } diff --git a/blockedarray/test_adapters.py b/blockedarray/test_adapters.py index 2bbfdaf..c633f97 100644 --- a/blockedarray/test_adapters.py +++ b/blockedarray/test_adapters.py @@ -7,7 +7,7 @@ class TestSource(unittest.TestCase): def setUp(self): - self.vol = np.random.randint(0, 2**32, (100, 100, 10)).astype(np.uint32) + self.vol = np.random.randint(0, 2**16-1, (100, 100, 10)).astype(np.uint32) def testExampleSource(self): vol = self.vol diff --git a/blockedarray/test_connectedcomponents.py b/blockedarray/test_connectedcomponents.py index ba122c3..9015dc3 100644 --- a/blockedarray/test_connectedcomponents.py +++ b/blockedarray/test_connectedcomponents.py @@ -47,6 +47,8 @@ def testCC(self): cc.writeResult(h5f[0], h5f[1]) labels = vigra.readHDF5(h5f[0], h5f[1]) + except Exception: + raise finally: pass os.remove(h5f[0]) From 51d7093594ad6ce833ab0e741aef7fb60663f1c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Wed, 22 Jan 2014 13:48:07 +0100 Subject: [PATCH 08/20] more doc on pyABC, tuple instead of ndarray --- blockedarray/adapters.py | 51 +++++++++++++++++++----- blockedarray/test_connectedcomponents.py | 3 ++ 2 files changed, 43 insertions(+), 11 deletions(-) diff --git a/blockedarray/adapters.py b/blockedarray/adapters.py index 0e73e7e..6c634d3 100644 --- a/blockedarray/adapters.py +++ b/blockedarray/adapters.py @@ -1,35 +1,64 @@ import numpy as np -from _blockedarray import Source3U8 as Source -from _blockedarray import Sink3 as Sink - - -class SourceABC(Source): +from _blockedarray import Source3U8 as _Source +from _blockedarray import Sink3 as _Sink + +## Source Interface +# +# This class provides the python interface to the C++ class `BW::Source`. If +# you inherit from this class, be sure to implement *all* abstract methods. +class SourceABC(_Source): + + ## Constructor + # + # Be sure to call super().__init__ if you override the constructor! def __init__(self): super(SourceABC, self).__init__() - ''' - * selects only the region of interest given from the - * underlying data source. When readBlock() is used, the coordinates - * are relative to roi[0] - ''' + ## Set a custom ROI + # selects only the region of interest given from the + # underlying data source. When readBlock() is used, the coordinates + # are relative to roi[0] + # + # @param roi a tuple containing 2 lists of length 3 (incl. start, excl. stop) def pySetRoi(self, roi): raise NotImplementedError #pass + ## Volume shape getter + # + # @return must be a tuple of (python) integers def pyShape(self): raise NotImplementedError #return (10, 10, 10) + ## Block getter + # + # read a block of data into a 3d numpy array + # + # @param roi a tuple containing 2 lists of length 3 (incl. start, excl. stop) + # @param output a 3d numpy.ndarray with shape roi[1]-roi[0] and dtype uint8 def pyReadBlock(self, roi, output): raise NotImplementedError #return True -class SinkABC(Sink): +## Sink Interface +# +# This class provides the python interface to the C++ class `BW::Sink`. If +# you inherit from this class, be sure to implement *all* abstract methods. +class SinkABC(_Sink): + + ## Constructor + # + # Be sure to call super().__init__ if you override the constructor! def __init__(self): super(SinkABC, self).__init__() + ## Write a block of data + # + # @param roi a tuple containing 2 lists of length 3 (incl. start, excl. stop) + # @param block a 3d numpy.ndarray with shape roi[1]-roi[0] and dtype int32 def pyWriteBlock(self, roi, block): raise NotImplementedError #return True diff --git a/blockedarray/test_connectedcomponents.py b/blockedarray/test_connectedcomponents.py index 9015dc3..62a6296 100644 --- a/blockedarray/test_connectedcomponents.py +++ b/blockedarray/test_connectedcomponents.py @@ -29,6 +29,8 @@ def testBlockShape(self): v = (10, 10, 10) cc = CC(s, v) + ''' + # these are not expected to work v = np.asarray((10, 10, 10)) cc = CC(s, v) @@ -37,6 +39,7 @@ def testBlockShape(self): v = np.asarray((10, 10, 10), dtype=np.int) cc = CC(s, v) + ''' def testCC(self): tmp = tempfile.NamedTemporaryFile(suffix='.h5', delete=False) From 89da0e38b9a53723dd407582e1a52a58ca29f92f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Thu, 20 Mar 2014 19:03:10 +0100 Subject: [PATCH 09/20] provide more types for standard source/sink --- blockedarray/blockwisecc_py.cxx | 95 +++++++++++++++++---------------- 1 file changed, 50 insertions(+), 45 deletions(-) diff --git a/blockedarray/blockwisecc_py.cxx b/blockedarray/blockwisecc_py.cxx index 5357fad..5da30a9 100644 --- a/blockedarray/blockwisecc_py.cxx +++ b/blockedarray/blockwisecc_py.cxx @@ -83,64 +83,69 @@ struct ExportV<4, V> { } }; -template -void blockwiseCC() { +template +void exportSpecificSource(std::string suffix) +{ + using namespace boost::python; - typedef ConnectedComponents BCC; - typedef Thresholding BWT; - typedef ChannelSelector BWCS; - typedef PyConnectedComponents PyBCC; - typedef SourceHDF5 HDF5BP_T; + const char *source = ("Source"+suffix).c_str(); + const char *sink = ("Sink"+suffix).c_str(); + class_ >(source, no_init); + class_ >(sink, no_init); +} - std::stringstream n; n << N; +template +void exportSourceForDim() +{ + exportSpecificSource("U8"); + //exportSpecificSource("U16"); + exportSpecificSource("U32"); + + exportSpecificSource("S8"); + //exportSpecificSource("S16"); + exportSpecificSource("S32"); + + exportSpecificSource("F"); + exportSpecificSource("D"); +} +template +void exportAllForDim() +{ + + using namespace boost::python; + + // set the correct module + std::stringstream n; n << N; + std::stringstream fullModname; fullModname << "_blockedarray.dim" << N; std::stringstream modName; modName << "dim" << N; - + //see: http://isolation-nation.blogspot.de/2008/09/packages-in-python-extension-modules.html object module(handle<>(borrowed(PyImport_AddModule(fullModname.str().c_str())))); scope().attr(modName.str().c_str()) = module; scope s = module; - - ExportV::export_(); - - class_ >("Source", no_init); - class_ >("Sink", no_init); -// class_ >("Source"); -// class_ >("Sink"); - - class_, bases > >("SourceHDF5", - init()) - ; - class_, bases > >("SinkHDF5", - init()) - ; - - class_("Thresholding", - init*, typename BWT::V>()) - .def("run", vigra::registerConverters(&BWT::run), - (arg("threshold"), arg("ifLower"), arg("ifHigher"), arg("sink"))) - ; - - class_("ChannelSelector", - init*, typename BWCS::V>()) - .def("run", vigra::registerConverters(&BWCS::run), - (arg("dimension"), arg("channel"), arg("sink"))) - ; - + + + // export source and sink + exportSourceForDim(); + + + typedef ConnectedComponents BCC; class_("ConnectedComponents", - init*, typename BCC::V>()) - .def("writeResult", &BCC::writeResult, - (arg("hdf5file"), arg("hdf5group"), arg("compression")=1)) - .def("writeToSink", &BCC::writeToSink, - (arg("sink"))) + init*, typename BCC::V>()) + .def("writeResult", &BCC::writeResult, + (arg("hdf5file"), arg("hdf5group"), arg("compression")=1)) + .def("writeToSink", &BCC::writeToSink, + (arg("sink"))) ; + } - -void export_blockwiseCC() { - blockwiseCC<2, float>(); - blockwiseCC<3, float>(); +void export_blockwiseCC() +{ + exportAllForDim<2>(); + exportAllForDim<3>(); } From cfc07813f994b946f426432eb2620908de07e797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Thu, 20 Mar 2014 20:45:40 +0100 Subject: [PATCH 10/20] removed unnecessary const attribute in sink --- blockedarray/adapters.py | 5 +++- blockedarray/adapters_py.cxx | 6 ++--- blockedarray/blockwisecc_py.cxx | 47 ++++++++------------------------- include/bw/sink.h | 4 +-- 4 files changed, 20 insertions(+), 42 deletions(-) diff --git a/blockedarray/adapters.py b/blockedarray/adapters.py index 6c634d3..08be5e4 100644 --- a/blockedarray/adapters.py +++ b/blockedarray/adapters.py @@ -1,4 +1,6 @@ +import vigra + import numpy as np from _blockedarray import Source3U8 as _Source from _blockedarray import Sink3 as _Sink @@ -117,7 +119,8 @@ def __init__(self): def pyWriteBlock(self, roi, block): assert len(roi) == 2 if self.vol is None: - shape = _v2tup(self.shape) + shape = self.shape + shape = _v2tup(shape) self.vol = np.zeros(shape, dtype=np.uint8) s = _roi2slice(roi[0], roi[1]) self.vol[s] = block diff --git a/blockedarray/adapters_py.cxx b/blockedarray/adapters_py.cxx index e3c448b..84f2d22 100644 --- a/blockedarray/adapters_py.cxx +++ b/blockedarray/adapters_py.cxx @@ -171,13 +171,13 @@ void exposeSink(const char* exposedName) { class_, boost::noncopyable>(exposedName) .def("pyWriteBlock", pure_virtual(&PySinkABC::pyWriteBlock)) - .add_property("shape", &PySinkABC::getShape, &PySinkABC::setShape) - .add_property("blockShape", &PySinkABC::getBlockShape, &PySinkABC::setBlockShape) + //.add_property("shape", &PySinkABC::getShape, &PySinkABC::setShape) + //.add_property("blockShape", &PySinkABC::getBlockShape, &PySinkABC::setBlockShape) ; } void export_adapters() { exposeSource<3,vigra::UInt8>("Source3U8"); exposeSink<3,ConnectedComponents<3>::LabelType>("Sink3"); - registerConverters(); +// registerConverters(); } diff --git a/blockedarray/blockwisecc_py.cxx b/blockedarray/blockwisecc_py.cxx index 5da30a9..859c6ca 100644 --- a/blockedarray/blockwisecc_py.cxx +++ b/blockedarray/blockwisecc_py.cxx @@ -48,42 +48,6 @@ using namespace BW; -template -struct PyConnectedComponents { - typedef ConnectedComponents BCC; -}; - -template -struct ExportV { - static void export_(); -}; - -template -struct ExportV<2, V> { - static void export_() { - using namespace boost::python; - class_("V", init()); - } -}; - - -template -struct ExportV<3, V> { - static void export_() { - using namespace boost::python; - class_("V", init()); - } -}; - -template -struct ExportV<4, V> { - static void export_() { - using namespace boost::python; - class_("V", init()); - } -}; - - template void exportSpecificSource(std::string suffix) { @@ -110,6 +74,7 @@ void exportSourceForDim() exportSpecificSource("D"); } + template void exportAllForDim() { @@ -132,6 +97,7 @@ void exportAllForDim() exportSourceForDim(); + // connected components class typedef ConnectedComponents BCC; class_("ConnectedComponents", init*, typename BCC::V>()) @@ -141,6 +107,15 @@ void exportAllForDim() (arg("sink"))) ; + + // ROI + typedef typename Roi::V V; + + class_< Roi >("Roi", init()) + .def_readonly("p", &Roi::p) + .def_readonly("q", &Roi::p) + ; + } diff --git a/include/bw/sink.h b/include/bw/sink.h index 85e7ff3..619a590 100644 --- a/include/bw/sink.h +++ b/include/bw/sink.h @@ -57,8 +57,8 @@ class Sink { blockShape_ = shape; } - V shape() const { return shape_; }; - V blockShape() const { return blockShape_; }; + V shape() { return shape_; }; + V blockShape() { return blockShape_; }; virtual bool writeBlock(Roi roi, const vigra::MultiArrayView& block) { return true; }; From fcbb833cd72471ed7ff65afcd2fe8f25a7689ea6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Fri, 21 Mar 2014 17:53:01 +0100 Subject: [PATCH 11/20] all adapters exported --- blockedarray/.gitignore | 1 + blockedarray/__init__.py | 1 - blockedarray/adapters.py | 6 ++- blockedarray/adapters_py.cxx | 84 +++++++++++--------------------- blockedarray/adapters_py.h | 3 +- blockedarray/blockwisecc_py.cxx | 53 +++++++++++++------- blockedarray/module_py.cxx | 2 - include/bw/connectedcomponents.h | 14 +++--- 8 files changed, 79 insertions(+), 85 deletions(-) diff --git a/blockedarray/.gitignore b/blockedarray/.gitignore index 76e8f19..4d83ce1 100644 --- a/blockedarray/.gitignore +++ b/blockedarray/.gitignore @@ -1,2 +1,3 @@ _blockedarray.so *.pyc +*.h5 diff --git a/blockedarray/__init__.py b/blockedarray/__init__.py index 2b375a5..6ad4ae4 100644 --- a/blockedarray/__init__.py +++ b/blockedarray/__init__.py @@ -1,2 +1 @@ from _blockedarray import * -import adapters diff --git a/blockedarray/adapters.py b/blockedarray/adapters.py index 08be5e4..b7cb9be 100644 --- a/blockedarray/adapters.py +++ b/blockedarray/adapters.py @@ -2,8 +2,10 @@ import vigra import numpy as np -from _blockedarray import Source3U8 as _Source -from _blockedarray import Sink3 as _Sink +from _blockedarray import * + +_Source = dim3.PySourceU8 +_Sink = dim3.PySinkU32 ## Source Interface # diff --git a/blockedarray/adapters_py.cxx b/blockedarray/adapters_py.cxx index 84f2d22..f7d711e 100644 --- a/blockedarray/adapters_py.cxx +++ b/blockedarray/adapters_py.cxx @@ -21,43 +21,6 @@ using namespace BW; -/* ROI conversion */ - -template -boost::python::list tinyVecToList(const typename Roi::V &vec) -{ - boost::python::object iterator = boost::python::iterator::V>()(vec); - return boost::python::list(iterator); -} - -template -struct Roi_to_python_tuple -{ - typedef typename Roi::V TinyVec; - static PyObject* convert(const Roi &roi) - { - // p - boost::python::list p = tinyVecToList(roi.p); - // q - boost::python::list q = tinyVecToList(roi.q); - - boost::python::tuple t = boost::python::make_tuple(p, q); - return boost::python::incref(t.ptr()); - } -}; - -void registerConverters() -{ - boost::python::to_python_converter, Roi_to_python_tuple<1> >(); - boost::python::to_python_converter, Roi_to_python_tuple<2> >(); - boost::python::to_python_converter, Roi_to_python_tuple<3> >(); - boost::python::to_python_converter, Roi_to_python_tuple<4> >(); - boost::python::to_python_converter, Roi_to_python_tuple<5> >(); - - // I really don't know why this is not called in vigra - vigra::NumpyArrayConverter::LabelType> >(); - vigra::NumpyArrayConverter >(); -} template @@ -130,10 +93,11 @@ struct PySinkABC : Sink, boost::python::wrapper > { return this->get_override("pyWriteBlock")(roi, block); }; - /* Accessor functions for the shapes */ - boost::python::list getShape() const + + /* UNNEEDED + V getShape() const { - return tinyVecToList(this->shape_); + return this->shape_; } void setShape(V shape) @@ -150,34 +114,44 @@ struct PySinkABC : Sink, boost::python::wrapper > { { this->blockShape_ = shape; } + */ }; template -void exposeSource(const char* exposedName) { +void exportSpecificSourceAdapter(std::string suffix) { using namespace boost::python; - class_, boost::noncopyable>(exposedName) + class_, boost::noncopyable>(("PySource" + suffix).c_str()) .def("pySetRoi", pure_virtual(&PySourceABC::pySetRoi)) .def("pyShape", pure_virtual(&PySourceABC::pyShape)) .def("pyReadBlock", pure_virtual(&PySourceABC::pyReadBlock)) - ; -} - -template -void exposeSink(const char* exposedName) { - using namespace boost::python; - - class_, boost::noncopyable>(exposedName) + ; + + class_, boost::noncopyable>(("PySink" + suffix).c_str()) .def("pyWriteBlock", pure_virtual(&PySinkABC::pyWriteBlock)) //.add_property("shape", &PySinkABC::getShape, &PySinkABC::setShape) //.add_property("blockShape", &PySinkABC::getBlockShape, &PySinkABC::setBlockShape) - ; + ; } -void export_adapters() { - exposeSource<3,vigra::UInt8>("Source3U8"); - exposeSink<3,ConnectedComponents<3>::LabelType>("Sink3"); -// registerConverters(); +template +void exportSourceAdaptersForDim() +{ + + exportSpecificSourceAdapter("U8"); + //exportSpecificSourceAdapter("U16"); + exportSpecificSourceAdapter("U32"); + + exportSpecificSourceAdapter("S8"); + //exportSpecificSourceAdapter("S16"); + exportSpecificSourceAdapter("S32"); + + exportSpecificSourceAdapter("F"); + exportSpecificSourceAdapter("D"); } + +template void exportSourceAdaptersForDim<2>(); +template void exportSourceAdaptersForDim<3>(); + diff --git a/blockedarray/adapters_py.h b/blockedarray/adapters_py.h index e733953..d44c237 100644 --- a/blockedarray/adapters_py.h +++ b/blockedarray/adapters_py.h @@ -1,6 +1,7 @@ #ifndef ADAPTERS_PY_H #define ADAPTERS_PY_H -void export_adapters(); +template +void exportSourceAdaptersForDim(); #endif /* ADAPTERS_PY_H */ \ No newline at end of file diff --git a/blockedarray/blockwisecc_py.cxx b/blockedarray/blockwisecc_py.cxx index 859c6ca..dd53d0b 100644 --- a/blockedarray/blockwisecc_py.cxx +++ b/blockedarray/blockwisecc_py.cxx @@ -43,11 +43,44 @@ #include #include "blockwisecc_py.h" +#include "adapters_py.h" #include using namespace BW; + +/* CC conversion */ +template +void exportCCForDim() +{ + using namespace boost::python; + using namespace BW; + typedef ConnectedComponents BCC; + class_("ConnectedComponents", + init*, typename BCC::V>()) + .def("writeResult", &BCC::writeResult, + (arg("hdf5file"), arg("hdf5group"), arg("compression")=1)) + .def("writeToSink", &BCC::writeToSink, + (arg("sink"))) + ; +} + +/* ROI conversion */ +template +void exportRoiForDim() +{ + using namespace boost::python; + using namespace BW; + typedef typename Roi::V V; + + class_< Roi >("Roi", init()) + .def_readonly("p", &Roi::p) + .def_readonly("q", &Roi::p) + ; +} + + template void exportSpecificSource(std::string suffix) { @@ -96,25 +129,11 @@ void exportAllForDim() // export source and sink exportSourceForDim(); + exportCCForDim(); - // connected components class - typedef ConnectedComponents BCC; - class_("ConnectedComponents", - init*, typename BCC::V>()) - .def("writeResult", &BCC::writeResult, - (arg("hdf5file"), arg("hdf5group"), arg("compression")=1)) - .def("writeToSink", &BCC::writeToSink, - (arg("sink"))) - ; - - - // ROI - typedef typename Roi::V V; + exportRoiForDim(); - class_< Roi >("Roi", init()) - .def_readonly("p", &Roi::p) - .def_readonly("q", &Roi::p) - ; + exportSourceAdaptersForDim(); } diff --git a/blockedarray/module_py.cxx b/blockedarray/module_py.cxx index 8bf933d..022570e 100644 --- a/blockedarray/module_py.cxx +++ b/blockedarray/module_py.cxx @@ -38,7 +38,6 @@ #include "blockedarray_py.h" #include "blockwisecc_py.h" -#include "adapters_py.h" BOOST_PYTHON_MODULE_INIT(_blockedarray) { @@ -48,5 +47,4 @@ BOOST_PYTHON_MODULE_INIT(_blockedarray) { export_blockedArray(); export_blockwiseCC(); - export_adapters(); } \ No newline at end of file diff --git a/include/bw/connectedcomponents.h b/include/bw/connectedcomponents.h index 916fc4b..81091e8 100644 --- a/include/bw/connectedcomponents.h +++ b/include/bw/connectedcomponents.h @@ -51,12 +51,14 @@ #include namespace BW { + +typedef unsigned int LabelType; using vigra::detail::UnionFindArray; template void blockMerge( - UnionFindArray& ufd, + UnionFindArray& ufd, Roi roi1, T offset1, const vigra::MultiArrayView& block1, @@ -76,8 +78,8 @@ void blockMerge( typename ArrayView::iterator it1 = ov1.begin(); typename ArrayView::iterator it2 = ov2.begin(); for(; it1 != ov1.end(); ++it1, ++it2) { - int x = *it1+offset1; - int y = *it2+offset2; + LabelType x = *it1+offset1; + LabelType y = *it2+offset2; //if(*it1 == 0) ufd.makeUnion(0,y); //if(*it2 == 0) ufd.makeUnion(x,0); ufd.makeUnion(x, y); @@ -110,16 +112,14 @@ struct ConnectedComponentsComputer<3,T,LabelType> { /** * Compute connected components block-wise (less limited to RAM) */ -template +template class ConnectedComponents { public: typedef typename Roi::V V; - typedef int LabelType; - typedef CompressedArray Compressed; - ConnectedComponents(Source* blockProvider, V blockShape) + ConnectedComponents(Source* blockProvider, V blockShape) : blockProvider_(blockProvider) , blockShape_(blockShape) { From 903995e098d8f00c5532907ae22ccffffd48bba0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Sat, 22 Mar 2014 15:21:13 +0100 Subject: [PATCH 12/20] wrote labeling operator --- blockedarray/__init__.py | 11 +++ blockedarray/blockwisecc_py.cxx | 18 +++- blockedarray/opBlockedConnectedComponents.py | 91 +++++++++++++++++++ .../testOpBlockedConnectedComponents.py | 60 ++++++++++++ include/bw/connectedcomponents.h | 6 +- include/bw/roi.h | 10 ++ 6 files changed, 190 insertions(+), 6 deletions(-) create mode 100644 blockedarray/opBlockedConnectedComponents.py create mode 100644 blockedarray/testOpBlockedConnectedComponents.py diff --git a/blockedarray/__init__.py b/blockedarray/__init__.py index 6ad4ae4..1c59d2f 100644 --- a/blockedarray/__init__.py +++ b/blockedarray/__init__.py @@ -1 +1,12 @@ from _blockedarray import * + +from opBlockedConnectedComponents import OpBlockedConnectedComponents + +''' +try: + from opBlockedConnectedComponents import OpBlockedConnectedComponents +except ImportError as err: + import warnings + warnings.warn("Could not import CC operator: {}".format(str(err))) +''' + diff --git a/blockedarray/blockwisecc_py.cxx b/blockedarray/blockwisecc_py.cxx index dd53d0b..525dfd0 100644 --- a/blockedarray/blockwisecc_py.cxx +++ b/blockedarray/blockwisecc_py.cxx @@ -57,13 +57,23 @@ void exportCCForDim() using namespace boost::python; using namespace BW; typedef ConnectedComponents BCC; - class_("ConnectedComponents", + + class_ >("ConnectedComponents", init*, typename BCC::V>()) .def("writeResult", &BCC::writeResult, (arg("hdf5file"), arg("hdf5group"), arg("compression")=1)) .def("writeToSink", &BCC::writeToSink, (arg("sink"))) ; + /* + class_ >("ConnectedComponents", + init*, typename BCC::V>()) + .def("writeResult", &BCC::writeResult, + (arg("hdf5file"), arg("hdf5group"), arg("compression")=1)) + .def("writeToSink", &BCC::writeToSink, + (arg("sink"))) + ; + */ } /* ROI conversion */ @@ -75,8 +85,10 @@ void exportRoiForDim() typedef typename Roi::V V; class_< Roi >("Roi", init()) - .def_readonly("p", &Roi::p) - .def_readonly("q", &Roi::p) + .def_readwrite("p", &Roi::p) + .def_readwrite("q", &Roi::p) + .def("getP", &Roi::getP) + .def("getQ", &Roi::getQ) ; } diff --git a/blockedarray/opBlockedConnectedComponents.py b/blockedarray/opBlockedConnectedComponents.py new file mode 100644 index 0000000..122fb6b --- /dev/null +++ b/blockedarray/opBlockedConnectedComponents.py @@ -0,0 +1,91 @@ + +import numpy as np +import vigra + +from lazyflow.operator import Operator, InputSlot, OutputSlot +from lazyflow.operators import OpCompressedCache +from lazyflow.operators.opLabelVolume import OpNonLazyCC +from lazyflow.rtype import SubRegion + +from _blockedarray import dim2, dim3 + + +class OpBlockedConnectedComponents(OpNonLazyCC): + name = "OpBlockedConnectedComponents" + supportedDtypes = [np.uint8, np.uint32] + + def __init__(self, *args, **kwargs): + super(OpBlockedConnectedComponents, self).__init__(*args, **kwargs) + self._sourceMap3 = dict(zip(self.supportedDtypes, + [dim3.PySourceU8, dim3.PySourceU32])) + self._sinkMap3 = {np.uint32: dim3.PySinkU32} + + def _updateSlice(self, c, t, bg): + bg = self.Background[0, 0, 0, c, t].wait() + assert bg == 0, "Non-zero background not supported" + source = self._getSource(c, t) + sink = self._getSink(c, t) + #TODO enable 2d + blockShape = tuple([int(s) for s in self._cache.BlockShape.value[:3]]) + print(blockShape) + cc = dim3.ConnectedComponents(source, blockShape) + + cc.writeToSink(sink) + + ## factory for dynamic source creation + def _getSource(self, c, t): + SourceClass = self._sourceMap3[self.Input.meta.dtype] + shape = tuple([int(s) for s in self.Input.meta.shape[:3]]) + + class TempSource(SourceClass): + def __init__(self, *args, **kwargs): + self._slot = kwargs['slot'] + del kwargs['slot'] + super(TempSource, self).__init__(*args, **kwargs) + def pySetRoi(self, roi): + assert False, "Not supported" + def pyShape(self): + return shape + def pyReadBlock(self, roi, block): + start = roi.getP() + (c, t) + stop = roi.getQ() + (c+1, t+1) + subr = SubRegion(self._slot, start=start, stop=stop) + block[:] = self._slot.get(subr).wait()[..., 0, 0] + return True + + return TempSource(slot=self.Input) + + ## factory for dynamic sink creation + def _getSink(self, c, t): + SinkClass = self._sinkMap3[self.Output.meta.dtype] + + class TempSink(SinkClass): + def __init__(self, *args, **kwargs): + self._cache = kwargs['cache'] + del kwargs['cache'] + super(TempSink, self).__init__(*args, **kwargs) + def pyWriteBlock(self, roi, block): + block = vigra.taggedView(block, axistags='xyz') + block = block.withAxes(*'xyzct') + start = roi.getP() + (c, t) + stop = roi.getQ() + (c+1, t+1) + subr = SubRegion(self._cache.Input, start=start, stop=stop) + self._cache.setInSlot(self._cache.Input, (), subr, block) + return True + + return TempSink(cache=self._cache) + + +def printParents(obj, indent=0): + def printIndented(s): + print("{}{}".format(" "*indent, s)) + if type(obj) != type: + print("Class hierarchy for {} (is a {})".format(obj, type(obj))) + printParents(type(obj)) + else: + if indent > 15: + return + printIndented(obj) + for sub in obj.__bases__: + printParents(sub, indent=indent+1) + diff --git a/blockedarray/testOpBlockedConnectedComponents.py b/blockedarray/testOpBlockedConnectedComponents.py new file mode 100644 index 0000000..da46999 --- /dev/null +++ b/blockedarray/testOpBlockedConnectedComponents.py @@ -0,0 +1,60 @@ +import numpy as np +import vigra + +import unittest + +from lazyflow.graph import Graph + +from numpy.testing import assert_array_equal, assert_array_almost_equal + +#import blockedarray.OpBlockedConnectedComponents +from blockedarray import OpBlockedConnectedComponents + + + +class TestOpBlockedConnectedComponents(unittest.TestCase): + + def setUp(self): + bg = np.zeros((1,1,1,1,1), dtype=np.uint8) + bg = vigra.taggedView(bg, axistags='xyzct') + self.bg = bg + + def testSimpleUsage(self): + vol = np.random.randint(255, size=(1000, 100, 10)) + vol = vol.astype(np.uint8) + vol = vigra.taggedView(vol, axistags='xyz') + vol = vol.withAxes(*'xyzct') + + op = OpBlockedConnectedComponents(graph=Graph()) + op.Input.setValue(vol) + op.Background.setValue(self.bg) + + out = op.Output[...].wait() + + assert_array_equal(vol.shape, out.shape) + + def testCorrectLabeling(self): + vol = np.zeros((1000, 100, 10)) + vol = vol.astype(np.uint8) + vol = vigra.taggedView(vol, axistags='xyz') + + vol[20:40, 10:30, 2:4] = 1 + vol = vol.withAxes(*'xyzct') + + op = OpBlockedConnectedComponents(graph=Graph()) + op.Input.setValue(vol) + op.Background.setValue(self.bg) + + out = op.Output[...].wait() + tags = op.Output.meta.getTaggedShape() + out = vigra.taggedView(out, axistags=op.Output.meta.axistags) + + assert_array_equal(vol, out) + + +class TestSimpleThings(unittest.TestCase): + def testRoi(self): + from blockedarray import dim3 + roi = dim3.Roi((0,0,0), (2,3,4)) + p = roi.getP() + diff --git a/include/bw/connectedcomponents.h b/include/bw/connectedcomponents.h index 81091e8..53129a8 100644 --- a/include/bw/connectedcomponents.h +++ b/include/bw/connectedcomponents.h @@ -153,12 +153,12 @@ class ConnectedComponents { // vigra::srcMultiArrayRange( labels_.subarray(block.p, block.q) ), // vigra::destMultiArray( ccBlocks[i] ), vigra::NeighborCode3DSix(), 0); - MultiArray inBlock(block.q-block.p); + MultiArray inBlock(block.q-block.p); blockProvider_->readBlock(block, inBlock); MultiArray cc(block.q-block.p); - LabelType maxLabel = ConnectedComponentsComputer::compute(inBlock, cc); + LabelType maxLabel = ConnectedComponentsComputer::compute(inBlock, cc); //LabelType maxLabel = vigra::labelImageWithBackground( // inBlock, // cc, @@ -293,7 +293,7 @@ class ConnectedComponents { } private: - Source* blockProvider_; + Source* blockProvider_; V blockShape_; std::vector< std::pair > > blockRois_; diff --git a/include/bw/roi.h b/include/bw/roi.h index 7eb2dca..7f7344a 100644 --- a/include/bw/roi.h +++ b/include/bw/roi.h @@ -182,6 +182,16 @@ class Roi { } return ret; } + + V getP() + { + return p; + } + + V getQ() + { + return q; + } V p; V q; From 83207e554f1aba3fdca051a579088b498fa9e302 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Sat, 22 Mar 2014 15:51:51 +0100 Subject: [PATCH 13/20] uint32 support --- blockedarray/blockwisecc_py.cxx | 28 +++++++++---------- blockedarray/opBlockedConnectedComponents.py | 7 +++-- .../testOpBlockedConnectedComponents.py | 18 ++++++++++++ 3 files changed, 36 insertions(+), 17 deletions(-) diff --git a/blockedarray/blockwisecc_py.cxx b/blockedarray/blockwisecc_py.cxx index 525dfd0..862861d 100644 --- a/blockedarray/blockwisecc_py.cxx +++ b/blockedarray/blockwisecc_py.cxx @@ -50,30 +50,28 @@ using namespace BW; -/* CC conversion */ -template -void exportCCForDim() +template +void exportSpecificCC(std::string suffix) { using namespace boost::python; using namespace BW; - typedef ConnectedComponents BCC; + typedef ConnectedComponents BCC; - class_ >("ConnectedComponents", - init*, typename BCC::V>()) + class_(("ConnectedComponents"+suffix).c_str(), + init*, typename BCC::V>()) .def("writeResult", &BCC::writeResult, (arg("hdf5file"), arg("hdf5group"), arg("compression")=1)) .def("writeToSink", &BCC::writeToSink, (arg("sink"))) ; - /* - class_ >("ConnectedComponents", - init*, typename BCC::V>()) - .def("writeResult", &BCC::writeResult, - (arg("hdf5file"), arg("hdf5group"), arg("compression")=1)) - .def("writeToSink", &BCC::writeToSink, - (arg("sink"))) - ; - */ +} + +/* CC conversion */ +template +void exportCCForDim() +{ + exportSpecificCC("U8"); + exportSpecificCC("U32"); } /* ROI conversion */ diff --git a/blockedarray/opBlockedConnectedComponents.py b/blockedarray/opBlockedConnectedComponents.py index 122fb6b..99a2851 100644 --- a/blockedarray/opBlockedConnectedComponents.py +++ b/blockedarray/opBlockedConnectedComponents.py @@ -19,6 +19,9 @@ def __init__(self, *args, **kwargs): self._sourceMap3 = dict(zip(self.supportedDtypes, [dim3.PySourceU8, dim3.PySourceU32])) self._sinkMap3 = {np.uint32: dim3.PySinkU32} + self._ccMap3 = dict(zip(self.supportedDtypes, + [dim3.ConnectedComponentsU8, + dim3.ConnectedComponentsU32])) def _updateSlice(self, c, t, bg): bg = self.Background[0, 0, 0, c, t].wait() @@ -27,8 +30,8 @@ def _updateSlice(self, c, t, bg): sink = self._getSink(c, t) #TODO enable 2d blockShape = tuple([int(s) for s in self._cache.BlockShape.value[:3]]) - print(blockShape) - cc = dim3.ConnectedComponents(source, blockShape) + CC = self._ccMap3[self.Input.meta.dtype] + cc = CC(source, blockShape) cc.writeToSink(sink) diff --git a/blockedarray/testOpBlockedConnectedComponents.py b/blockedarray/testOpBlockedConnectedComponents.py index da46999..9d3994f 100644 --- a/blockedarray/testOpBlockedConnectedComponents.py +++ b/blockedarray/testOpBlockedConnectedComponents.py @@ -51,6 +51,24 @@ def testCorrectLabeling(self): assert_array_equal(vol, out) + def testCorrectLabelingInt(self): + vol = np.zeros((1000, 100, 10)) + vol = vol.astype(np.uint32) + vol = vigra.taggedView(vol, axistags='xyz') + + vol[20:40, 10:30, 2:4] = 1 + vol = vol.withAxes(*'xyzct') + + op = OpBlockedConnectedComponents(graph=Graph()) + op.Input.setValue(vol) + op.Background.setValue(self.bg) + + out = op.Output[...].wait() + tags = op.Output.meta.getTaggedShape() + out = vigra.taggedView(out, axistags=op.Output.meta.axistags) + + assert_array_equal(vol, out) + class TestSimpleThings(unittest.TestCase): def testRoi(self): From fcdf0010b1bc1313d2a9d4d3ce2f438775645cf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Tue, 25 Mar 2014 18:05:56 +0100 Subject: [PATCH 14/20] introduced ROI interface to adapters, better roi interface --- blockedarray/adapters.py | 27 +++++++++----- blockedarray/adapters_py.cxx | 10 +++--- blockedarray/blockwisecc_py.cxx | 10 +++--- blockedarray/opBlockedConnectedComponents.py | 35 ++++++++++--------- .../testOpBlockedConnectedComponents.py | 3 +- blockedarray/test_adapters.py | 11 ++++-- blockedarray/test_connectedcomponents.py | 2 +- include/bw/roi.h | 10 ++++++ 8 files changed, 70 insertions(+), 38 deletions(-) diff --git a/blockedarray/adapters.py b/blockedarray/adapters.py index b7cb9be..e97a0c7 100644 --- a/blockedarray/adapters.py +++ b/blockedarray/adapters.py @@ -4,9 +4,23 @@ import numpy as np from _blockedarray import * + +# This file contains examples for using the Source and Sink classes from +# blockedarray. These should be viewed as guidelines on how to use the exported +# interfaces +# dim[2|3].Source[U|S][8|16|32|64] +# dim[2|3].Sink[U|S][8|16|32|64] +# The ABC classes are for reference on what methods are exposed, the Example +# classes show how the base classes can be used. There is a complete workflow +# for blockwise connected components in the test file +# test_connectedcomponents.py + + +# examples deal with 8bit image input and 32bit image output _Source = dim3.PySourceU8 _Sink = dim3.PySinkU32 + ## Source Interface # # This class provides the python interface to the C++ class `BW::Source`. If @@ -92,17 +106,15 @@ def __init__(self, vol): self._q = np.asarray(vol.shape, dtype=np.long) def pySetRoi(self, roi): - assert len(roi) == 2 - self._p = np.asarray(roi[0], dtype=np.long) - self._q = np.asarray(roi[1], dtype=np.long) + self._p = np.asarray(roi.p, dtype=np.long) + self._q = np.asarray(roi.q, dtype=np.long) def pyShape(self): return self._vol.shape def pyReadBlock(self, roi, output): - assert len(roi) == 2 - roiP = np.asarray(roi[0]) - roiQ = np.asarray(roi[1]) + roiP = np.asarray(roi.p) + roiQ = np.asarray(roi.q) p = self._p + roiP q = p + roiQ - roiP if np.any(q > self._q): @@ -119,12 +131,11 @@ def __init__(self): self.vol = None def pyWriteBlock(self, roi, block): - assert len(roi) == 2 if self.vol is None: shape = self.shape shape = _v2tup(shape) self.vol = np.zeros(shape, dtype=np.uint8) - s = _roi2slice(roi[0], roi[1]) + s = _roi2slice(roi.p, roi.q) self.vol[s] = block diff --git a/blockedarray/adapters_py.cxx b/blockedarray/adapters_py.cxx index f7d711e..33548e6 100644 --- a/blockedarray/adapters_py.cxx +++ b/blockedarray/adapters_py.cxx @@ -94,7 +94,6 @@ struct PySinkABC : Sink, boost::python::wrapper > { }; - /* UNNEEDED V getShape() const { return this->shape_; @@ -105,16 +104,15 @@ struct PySinkABC : Sink, boost::python::wrapper > { this->shape_ = shape; } - boost::python::list getBlockShape() const + V getBlockShape() const { - return tinyVecToList(this->blockShape_); + return this->blockShape_; } void setBlockShape(V shape) { this->blockShape_ = shape; } - */ }; @@ -131,8 +129,8 @@ void exportSpecificSourceAdapter(std::string suffix) { class_, boost::noncopyable>(("PySink" + suffix).c_str()) .def("pyWriteBlock", pure_virtual(&PySinkABC::pyWriteBlock)) - //.add_property("shape", &PySinkABC::getShape, &PySinkABC::setShape) - //.add_property("blockShape", &PySinkABC::getBlockShape, &PySinkABC::setBlockShape) + .add_property("shape", &PySinkABC::getShape, &PySinkABC::setShape) + .add_property("blockShape", &PySinkABC::getBlockShape, &PySinkABC::setBlockShape) ; } diff --git a/blockedarray/blockwisecc_py.cxx b/blockedarray/blockwisecc_py.cxx index 862861d..1268a24 100644 --- a/blockedarray/blockwisecc_py.cxx +++ b/blockedarray/blockwisecc_py.cxx @@ -83,10 +83,12 @@ void exportRoiForDim() typedef typename Roi::V V; class_< Roi >("Roi", init()) - .def_readwrite("p", &Roi::p) - .def_readwrite("q", &Roi::p) - .def("getP", &Roi::getP) - .def("getQ", &Roi::getQ) + //.def_readwrite("p", &Roi::p) + //.def_readwrite("q", &Roi::p) + //.def("getP", &Roi::getP) + //.def("getQ", &Roi::getQ) + .add_property("p", &Roi::getP, &Roi::setP) + .add_property("q", &Roi::getQ, &Roi::setQ) ; } diff --git a/blockedarray/opBlockedConnectedComponents.py b/blockedarray/opBlockedConnectedComponents.py index 99a2851..d5fe97e 100644 --- a/blockedarray/opBlockedConnectedComponents.py +++ b/blockedarray/opBlockedConnectedComponents.py @@ -10,6 +10,10 @@ from _blockedarray import dim2, dim3 +## compute connected components blockwise +# +# Input must be 'xyzct' so this operator can be wrapped with +# lazyflow.OpLabelVolume class OpBlockedConnectedComponents(OpNonLazyCC): name = "OpBlockedConnectedComponents" supportedDtypes = [np.uint8, np.uint32] @@ -50,8 +54,8 @@ def pySetRoi(self, roi): def pyShape(self): return shape def pyReadBlock(self, roi, block): - start = roi.getP() + (c, t) - stop = roi.getQ() + (c+1, t+1) + start = roi.p + (c, t) + stop = roi.q + (c+1, t+1) subr = SubRegion(self._slot, start=start, stop=stop) block[:] = self._slot.get(subr).wait()[..., 0, 0] return True @@ -67,28 +71,27 @@ def __init__(self, *args, **kwargs): self._cache = kwargs['cache'] del kwargs['cache'] super(TempSink, self).__init__(*args, **kwargs) + self.shape = _v2tup(self._cache.Input.meta.shape) + self.blockShape = _v2tup(self._cache.BlockShape.value) def pyWriteBlock(self, roi, block): block = vigra.taggedView(block, axistags='xyz') block = block.withAxes(*'xyzct') - start = roi.getP() + (c, t) - stop = roi.getQ() + (c+1, t+1) + start = roi.p + (c, t) + stop = roi.q + (c+1, t+1) subr = SubRegion(self._cache.Input, start=start, stop=stop) self._cache.setInSlot(self._cache.Input, (), subr, block) return True return TempSink(cache=self._cache) + def setupOutputs(self): + super(OpBlockedConnectedComponents, self).setupOutputs() + assert len(self.Input.meta.shape) == 5, "Input must be 5d" + if self.Input.meta.axistags: + # if axistags are given, they must be xyzct + s = "".join(self.Input.meta.getAxisKeys()) + assert s == "xyzct", "Input must be in xyzct order, if any" -def printParents(obj, indent=0): - def printIndented(s): - print("{}{}".format(" "*indent, s)) - if type(obj) != type: - print("Class hierarchy for {} (is a {})".format(obj, type(obj))) - printParents(type(obj)) - else: - if indent > 15: - return - printIndented(obj) - for sub in obj.__bases__: - printParents(sub, indent=indent+1) +def _v2tup(v, d=3): + return tuple([int(v[i]) for i in range(d)]) diff --git a/blockedarray/testOpBlockedConnectedComponents.py b/blockedarray/testOpBlockedConnectedComponents.py index 9d3994f..a09f042 100644 --- a/blockedarray/testOpBlockedConnectedComponents.py +++ b/blockedarray/testOpBlockedConnectedComponents.py @@ -74,5 +74,6 @@ class TestSimpleThings(unittest.TestCase): def testRoi(self): from blockedarray import dim3 roi = dim3.Roi((0,0,0), (2,3,4)) - p = roi.getP() + p = roi.p + assert isinstance(p, tuple) diff --git a/blockedarray/test_adapters.py b/blockedarray/test_adapters.py index c633f97..cc494e2 100644 --- a/blockedarray/test_adapters.py +++ b/blockedarray/test_adapters.py @@ -2,6 +2,7 @@ import unittest from adapters import ExampleSource, ExampleSink +from _blockedarray import dim3 class TestSource(unittest.TestCase): @@ -13,7 +14,8 @@ def testExampleSource(self): vol = self.vol s = ExampleSource(vol) - roi = [np.zeros((len(vol.shape),)), vol.shape] + roi = dim3.Roi((0,)*len(vol.shape), + tuple(vol.shape)) newVol = np.zeros(vol.shape, dtype=np.uint32) assert s.pyReadBlock(roi, newVol) assert np.all(vol == newVol) @@ -23,8 +25,10 @@ def testRoi(self): s = ExampleSource(vol) reqRoi = [(50, 50, 2), (70, 70, 4)] + reqRoi = dim3.Roi(reqRoi[0], reqRoi[1]) s.pySetRoi(reqRoi) roi = [(0, 0, 0), (20, 20, 2)] + roi = dim3.Roi(roi[0], roi[1]) newVol = np.zeros((20, 20, 2), dtype=np.uint32) assert s.pyReadBlock(roi, newVol) assert np.all(vol[50:70, 50:70, 2:4] == newVol) @@ -33,6 +37,7 @@ def testRoi2(self): vol = self.vol s = ExampleSource(vol) roi = [(0, 0, 2), (20, 20, 4)] + roi = dim3.Roi(roi[0], roi[1]) newVol = np.zeros((20, 20, 2), dtype=np.uint32) assert s.pyReadBlock(roi, newVol) @@ -46,7 +51,9 @@ def testExampleSink(self): s = ExampleSink() s.shape = (100, 100, 10) s.blockShape = (10, 10, 10) - s.pyWriteBlock([(15, 20, 2), (30, 30, 6)], + roi = [(15, 20, 2), (30, 30, 6)] + roi = dim3.Roi(roi[0], roi[1]) + s.pyWriteBlock(roi, np.ones((15, 10, 4), dtype=np.uint8)) assert np.all(s.vol[15:30, 20:30, 2:6] == 1) diff --git a/blockedarray/test_connectedcomponents.py b/blockedarray/test_connectedcomponents.py index 62a6296..bc46da1 100644 --- a/blockedarray/test_connectedcomponents.py +++ b/blockedarray/test_connectedcomponents.py @@ -10,7 +10,7 @@ from _blockedarray import dim3 -CC = dim3.ConnectedComponents +CC = dim3.ConnectedComponentsU8 class TestConnectedComponents(unittest.TestCase): diff --git a/include/bw/roi.h b/include/bw/roi.h index 7f7344a..cd73fd9 100644 --- a/include/bw/roi.h +++ b/include/bw/roi.h @@ -192,6 +192,16 @@ class Roi { { return q; } + + void setP(V p) + { + this->p = p; + } + + void setQ(V q) + { + this->q = q; + } V p; V q; From 6fbcf3126ffd08b831a31608064f8e1fdfb8e06b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Tue, 25 Mar 2014 18:54:05 +0100 Subject: [PATCH 15/20] revert unnecessary file changes --- blockedarray/opBlockedConnectedComponents.py | 6 +++--- include/bw/sink.h | 2 +- include/bw/source.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/blockedarray/opBlockedConnectedComponents.py b/blockedarray/opBlockedConnectedComponents.py index d5fe97e..5f1469f 100644 --- a/blockedarray/opBlockedConnectedComponents.py +++ b/blockedarray/opBlockedConnectedComponents.py @@ -33,7 +33,7 @@ def _updateSlice(self, c, t, bg): source = self._getSource(c, t) sink = self._getSink(c, t) #TODO enable 2d - blockShape = tuple([int(s) for s in self._cache.BlockShape.value[:3]]) + blockShape = _v2tup(self._cache.BlockShape.value, dim=3) CC = self._ccMap3[self.Input.meta.dtype] cc = CC(source, blockShape) @@ -93,5 +93,5 @@ def setupOutputs(self): assert s == "xyzct", "Input must be in xyzct order, if any" -def _v2tup(v, d=3): - return tuple([int(v[i]) for i in range(d)]) +def _v2tup(v, dim=3): + return tuple([int(v[i]) for i in range(dim)]) diff --git a/include/bw/sink.h b/include/bw/sink.h index 619a590..71d529c 100644 --- a/include/bw/sink.h +++ b/include/bw/sink.h @@ -44,7 +44,7 @@ class Sink { public: typedef typename Roi::V V; - Sink() {}; + Sink() {} virtual ~Sink() {}; /* has to be called before any calls to writeBlock */ diff --git a/include/bw/source.h b/include/bw/source.h index 910d835..0f8a92c 100644 --- a/include/bw/source.h +++ b/include/bw/source.h @@ -44,7 +44,7 @@ class Source { public: typedef typename Roi::V V; - Source() {}; + Source() {} virtual ~Source() {}; /** From b57bee19db786e4569cee4f02b0654531c660fef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Tue, 25 Mar 2014 19:15:15 +0100 Subject: [PATCH 16/20] travis: update to newer h5py version --- .travis_scripts/requirements/development-stage2.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis_scripts/requirements/development-stage2.txt b/.travis_scripts/requirements/development-stage2.txt index 80035be..7f24205 100644 --- a/.travis_scripts/requirements/development-stage2.txt +++ b/.travis_scripts/requirements/development-stage2.txt @@ -1 +1 @@ -h5py==2.1.3 +h5py==2.2.1 From 039444693f23942833797478e427d29b86e2f828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Wed, 26 Mar 2014 12:53:06 +0100 Subject: [PATCH 17/20] adhere to coding style --- blockedarray/__init__.py | 9 ---- blockedarray/adapters_py.cxx | 45 +++++++------------ blockedarray/adapters_py.h | 2 +- blockedarray/blockwisecc_py.cxx | 21 +++------ .../testOpBlockedConnectedComponents.py | 12 +++++ include/bw/roi.h | 12 ++--- include/bw/sink.h | 8 ++-- include/bw/source.h | 8 ++-- 8 files changed, 47 insertions(+), 70 deletions(-) diff --git a/blockedarray/__init__.py b/blockedarray/__init__.py index 1c59d2f..a706a1c 100644 --- a/blockedarray/__init__.py +++ b/blockedarray/__init__.py @@ -1,12 +1,3 @@ from _blockedarray import * from opBlockedConnectedComponents import OpBlockedConnectedComponents - -''' -try: - from opBlockedConnectedComponents import OpBlockedConnectedComponents -except ImportError as err: - import warnings - warnings.warn("Could not import CC operator: {}".format(str(err))) -''' - diff --git a/blockedarray/adapters_py.cxx b/blockedarray/adapters_py.cxx index 33548e6..d2761f6 100644 --- a/blockedarray/adapters_py.cxx +++ b/blockedarray/adapters_py.cxx @@ -21,8 +21,6 @@ using namespace BW; - - template struct PySourceABC : Source, boost::python::wrapper > { @@ -30,21 +28,18 @@ struct PySourceABC : Source, boost::python::wrapper > typedef typename Source::V TinyVec; - PySourceABC() {}; - virtual ~PySourceABC() {}; + PySourceABC() {} + virtual ~PySourceABC() {} - void setRoi(Roi roi) - { + void setRoi(Roi roi) { this->pySetRoi(roi); } - TinyVec shape() const - { + TinyVec shape() const { return this->pyShape(); } - bool readBlock(Roi roi, vigra::MultiArrayView& block) const - { + bool readBlock(Roi roi, vigra::MultiArrayView& block) const { bool ret; //temporary NumpyArray, because MultiArrayView is not convertible to python @@ -58,18 +53,15 @@ struct PySourceABC : Source, boost::python::wrapper > return true; } - void pySetRoi(Roi roi) - { + void pySetRoi(Roi roi) { this->get_override("pySetRoi")(roi); } - TinyVec pyShape() const - { + TinyVec pyShape() const { return this->get_override("pyShape")(); }; - bool pyReadBlock(Roi roi, vigra::NumpyArray& block) const - { + bool pyReadBlock(Roi roi, vigra::NumpyArray& block) const { return this->get_override("pyReadBlock")(roi, block); }; }; @@ -82,35 +74,29 @@ struct PySinkABC : Sink, boost::python::wrapper > { PySinkABC() {}; virtual ~PySinkABC() {}; - bool writeBlock(Roi roi, const vigra::MultiArrayView& block) - { + bool writeBlock(Roi roi, const vigra::MultiArrayView& block) { vigra::NumpyArray tempArray(block); return this->pyWriteBlock(roi, tempArray); } - bool pyWriteBlock(Roi roi, const vigra::NumpyArray& block) - { + bool pyWriteBlock(Roi roi, const vigra::NumpyArray& block) { return this->get_override("pyWriteBlock")(roi, block); }; - V getShape() const - { + V getShape() const { return this->shape_; } - void setShape(V shape) - { + void setShape(V shape) { this->shape_ = shape; } - V getBlockShape() const - { + V getBlockShape() const { return this->blockShape_; } - void setBlockShape(V shape) - { + void setBlockShape(V shape) { this->blockShape_ = shape; } }; @@ -135,8 +121,7 @@ void exportSpecificSourceAdapter(std::string suffix) { } template -void exportSourceAdaptersForDim() -{ +void exportSourceAdaptersForDim() { exportSpecificSourceAdapter("U8"); //exportSpecificSourceAdapter("U16"); diff --git a/blockedarray/adapters_py.h b/blockedarray/adapters_py.h index d44c237..38c30b4 100644 --- a/blockedarray/adapters_py.h +++ b/blockedarray/adapters_py.h @@ -4,4 +4,4 @@ template void exportSourceAdaptersForDim(); -#endif /* ADAPTERS_PY_H */ \ No newline at end of file +#endif /* ADAPTERS_PY_H */ diff --git a/blockedarray/blockwisecc_py.cxx b/blockedarray/blockwisecc_py.cxx index 1268a24..8b9e518 100644 --- a/blockedarray/blockwisecc_py.cxx +++ b/blockedarray/blockwisecc_py.cxx @@ -51,8 +51,7 @@ using namespace BW; template -void exportSpecificCC(std::string suffix) -{ +void exportSpecificCC(std::string suffix) { using namespace boost::python; using namespace BW; typedef ConnectedComponents BCC; @@ -68,16 +67,14 @@ void exportSpecificCC(std::string suffix) /* CC conversion */ template -void exportCCForDim() -{ +void exportCCForDim() { exportSpecificCC("U8"); exportSpecificCC("U32"); } /* ROI conversion */ template -void exportRoiForDim() -{ +void exportRoiForDim() { using namespace boost::python; using namespace BW; typedef typename Roi::V V; @@ -94,8 +91,7 @@ void exportRoiForDim() template -void exportSpecificSource(std::string suffix) -{ +void exportSpecificSource(std::string suffix) { using namespace boost::python; const char *source = ("Source"+suffix).c_str(); @@ -105,8 +101,7 @@ void exportSpecificSource(std::string suffix) } template -void exportSourceForDim() -{ +void exportSourceForDim() { exportSpecificSource("U8"); //exportSpecificSource("U16"); exportSpecificSource("U32"); @@ -121,8 +116,7 @@ void exportSourceForDim() template -void exportAllForDim() -{ +void exportAllForDim() { using namespace boost::python; @@ -150,8 +144,7 @@ void exportAllForDim() } -void export_blockwiseCC() -{ +void export_blockwiseCC() { exportAllForDim<2>(); exportAllForDim<3>(); } diff --git a/blockedarray/testOpBlockedConnectedComponents.py b/blockedarray/testOpBlockedConnectedComponents.py index a09f042..fc9550f 100644 --- a/blockedarray/testOpBlockedConnectedComponents.py +++ b/blockedarray/testOpBlockedConnectedComponents.py @@ -76,4 +76,16 @@ def testRoi(self): roi = dim3.Roi((0,0,0), (2,3,4)) p = roi.p assert isinstance(p, tuple) + assert len(p) ==3 + q = roi.q + assert isinstance(q, tuple) + assert len(q) ==3 + + roi.p = q + roi.q = p + tempQ = roi.p + assert isinstance(tempQ, tuple) + assert tempQ == q + + diff --git a/include/bw/roi.h b/include/bw/roi.h index cd73fd9..3beaf8f 100644 --- a/include/bw/roi.h +++ b/include/bw/roi.h @@ -183,23 +183,19 @@ class Roi { return ret; } - V getP() - { + V getP() { return p; } - V getQ() - { + V getQ() { return q; } - void setP(V p) - { + void setP(V p) { this->p = p; } - void setQ(V q) - { + void setQ(V q) { this->q = q; } diff --git a/include/bw/sink.h b/include/bw/sink.h index 71d529c..d26bcd0 100644 --- a/include/bw/sink.h +++ b/include/bw/sink.h @@ -45,7 +45,7 @@ class Sink { typedef typename Roi::V V; Sink() {} - virtual ~Sink() {}; + virtual ~Sink() {} /* has to be called before any calls to writeBlock */ void setShape(V shape) { @@ -57,10 +57,10 @@ class Sink { blockShape_ = shape; } - V shape() { return shape_; }; - V blockShape() { return blockShape_; }; + V shape() { return shape_; } + V blockShape() { return blockShape_; } - virtual bool writeBlock(Roi roi, const vigra::MultiArrayView& block) { return true; }; + virtual bool writeBlock(Roi roi, const vigra::MultiArrayView& block) { return true; } protected: V shape_; diff --git a/include/bw/source.h b/include/bw/source.h index 0f8a92c..6e13837 100644 --- a/include/bw/source.h +++ b/include/bw/source.h @@ -45,17 +45,17 @@ class Source { typedef typename Roi::V V; Source() {} - virtual ~Source() {}; + virtual ~Source() {} /** * selects only the region of interest given from the * underlying data source. When readBlock() is used, the coordinates * are relative to roi.q */ - virtual void setRoi(Roi roi) {}; + virtual void setRoi(Roi roi) {} - virtual V shape() const { return V(); }; - virtual bool readBlock(Roi roi, vigra::MultiArrayView& block) const { return true; }; + virtual V shape() const { return V(); } + virtual bool readBlock(Roi roi, vigra::MultiArrayView& block) const { return true; } }; } /* namespace BW */ From 382bc2c68fd20b63cc2195e25d947ae424386f60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Wed, 26 Mar 2014 13:03:53 +0100 Subject: [PATCH 18/20] reintroduced const where applicable --- include/bw/roi.h | 4 ++-- include/bw/sink.h | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/include/bw/roi.h b/include/bw/roi.h index 3beaf8f..294076a 100644 --- a/include/bw/roi.h +++ b/include/bw/roi.h @@ -183,11 +183,11 @@ class Roi { return ret; } - V getP() { + V getP() const { return p; } - V getQ() { + V getQ() const { return q; } diff --git a/include/bw/sink.h b/include/bw/sink.h index d26bcd0..40c1e85 100644 --- a/include/bw/sink.h +++ b/include/bw/sink.h @@ -57,8 +57,8 @@ class Sink { blockShape_ = shape; } - V shape() { return shape_; } - V blockShape() { return blockShape_; } + V shape() const { return shape_; } + V blockShape() const { return blockShape_; } virtual bool writeBlock(Roi roi, const vigra::MultiArrayView& block) { return true; } From 66a161e4b22acd09d0f10e40646fd6e72015c22a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Wed, 26 Mar 2014 13:21:38 +0100 Subject: [PATCH 19/20] disabled old style lazyflow tests --- .../lazyflow_cpp/testOpArrayCacheCpp.py | 38 ++++--------------- 1 file changed, 8 insertions(+), 30 deletions(-) diff --git a/blockedarray/lazyflow_cpp/testOpArrayCacheCpp.py b/blockedarray/lazyflow_cpp/testOpArrayCacheCpp.py index 2ab1766..8bcce50 100644 --- a/blockedarray/lazyflow_cpp/testOpArrayCacheCpp.py +++ b/blockedarray/lazyflow_cpp/testOpArrayCacheCpp.py @@ -1,10 +1,13 @@ import threading +import unittest import numpy import vigra from lazyflow.graph import Graph from lazyflow.roi import sliceToRoi, roiToSlice from lazyflow.operators import OpArrayPiper -from lazyflow.operators.opArrayCacheCpp import OpArrayCacheCpp + +# does not exist anymore!! +#from lazyflow.operators.opArrayCacheCpp import OpArrayCacheCpp class KeyMaker(): def __getitem__(self, *args): @@ -26,7 +29,8 @@ def execute(self, slot, subindex, roi, result): super(OpArrayPiperWithAccessCount, self).execute(slot, subindex, roi, result) -class TestOpArrayCache(object): +@unittest.skip("OpArrayCacheCpp is outdated") +class TestOpArrayCache(unittest.TestCase): def setUp(self): print "RRRRRRRRRRRRRRRRRRRRRRRRRR" @@ -285,31 +289,5 @@ def handleDirty(slot, roi): sys.argv.append("--nologcapture") # Don't set the logging level to DEBUG. Leave it alone. ret = nose.run(defaultTest=__file__) - - - - - - - - - - - - - - - - - - - - - - - - - - - - if not ret: sys.exit(1) + if not ret: + sys.exit(1) From f13569068647a5322876b67e1de850546328b55e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Markus=20D=C3=B6ring?= Date: Wed, 26 Mar 2014 13:29:19 +0100 Subject: [PATCH 20/20] updated travis instructions to include python tests --- .travis.yml | 1 + .travis_scripts/requirements/development-stage2.txt | 1 + 2 files changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 9c00c87..7261231 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,3 +14,4 @@ before_script: - "export PYTHONPATH=$VIRTUAL_ENV/lib/python2.7/site-packages:$PYTHONPATH" script: - make test + - nosetests diff --git a/.travis_scripts/requirements/development-stage2.txt b/.travis_scripts/requirements/development-stage2.txt index 7f24205..ab103d7 100644 --- a/.travis_scripts/requirements/development-stage2.txt +++ b/.travis_scripts/requirements/development-stage2.txt @@ -1 +1,2 @@ h5py==2.2.1 +nose