From eaa0054e082de7e6fe6eab7d2435e5dfd1f8dc4f Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Fri, 16 Sep 2016 18:45:29 -0400
Subject: [PATCH 01/12] Add Dockerfile for development use

---
 DevDockerfile      | 6 ++++++
 docker-compose.yml | 4 +++-
 2 files changed, 9 insertions(+), 1 deletion(-)
 create mode 100644 DevDockerfile

diff --git a/DevDockerfile b/DevDockerfile
new file mode 100644
index 000000000..f52edbc93
--- /dev/null
+++ b/DevDockerfile
@@ -0,0 +1,6 @@
+FROM eywalker/jupyter
+
+MAINTAINER Edgar Y. Walker <edgar.walker@gmail.com>
+
+ADD . /src
+RUN pip install -e /src
diff --git a/docker-compose.yml b/docker-compose.yml
index 679a0023d..5714e34d9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -3,11 +3,13 @@ services:
   datajoint:
     build:
       context: .
-      dockerfile: JupyterDockerfile
+      dockerfile: DevDockerfile
     environment:
       - DJ_HOST=db
       - DJ_USER=root
       - DJ_PASS=simple
+    volumes:
+      - .:/src
     links:
       - db
     ports:

From 515b4a1eef3e3e4f3093cfb9ce2719682bf28741 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Fri, 16 Sep 2016 18:48:05 -0400
Subject: [PATCH 02/12] Add support for dict in BLOB

---
 datajoint/blob.py | 123 +++++++++++++++++++++++++++++++++++-----------
 1 file changed, 93 insertions(+), 30 deletions(-)

diff --git a/datajoint/blob.py b/datajoint/blob.py
index be930bba0..8f2eebfc0 100644
--- a/datajoint/blob.py
+++ b/datajoint/blob.py
@@ -3,7 +3,7 @@
 """
 
 import zlib
-from collections import OrderedDict
+from collections import OrderedDict, Mapping, Iterable
 import numpy as np
 from . import DataJointError
 
@@ -34,12 +34,18 @@
     b'ZL123\0': zlib.decompress
 }
 
+def forward_squeeze_shape(shape):
+    for p in range(len(shape)):
+        if shape[p] != 1:
+            break
+    return shape[p:]
 
 class BlobReader:
-    def __init__(self, blob, simplify=False):
+    def __init__(self, blob, simplify=False, as_dict=False):
         self._simplify = simplify
         self._blob = blob
         self._pos = 0
+        self._as_dict = as_dict
 
     @property
     def pos(self):
@@ -92,10 +98,10 @@ def read_array(self, advance=True, n_bytes=None):
         dtype = dtype_list[dtype_id]
         is_complex = self.read_value('uint32')
 
-        if dtype_id == 4: # if dealing with character array
+        if dtype_id == 4:  # if dealing with character array
             data = self.read_value(dtype, count=2 * n_elem)
-            data = data[::2].astype('<U1')
-            if n_dims == 2 and shape[0] == 1:
+            data = data[::2].astype('U1')
+            if self._simplify and n_dims == 2 and shape[0] == 1 or n_dims == 1:
                 compact = data.squeeze()
                 data = compact if compact.shape == () else np.array(''.join(data.squeeze()))
                 shape = (1,)
@@ -129,17 +135,22 @@ def read_structure(self, advance=True, n_bytes=None):
         dt = [(f, np.object) for f in field_names]
         raw_data = []
         for k in range(n_elem):
-            vals = []
+            values = []
             for i in range(n_field):
                 nb = int(self.read_value('uint64')) # dealing with a weird bug of numpy
-                vals.append(self.read_mym_data(n_bytes=nb))
-            raw_data.append(tuple(vals))
-        data = np.rec.array(raw_data, dtype=dt)
+                values.append(self.read_mym_data(n_bytes=nb))
+            raw_data.append(tuple(values))
         if n_bytes is not None:
             assert self.pos - start == n_bytes
         if not advance:
             self.pos = start
-        return self.simplify(data.reshape(shape, order='F'))
+
+        if self._as_dict and n_elem == 1:
+            data = dict(zip(field_names, values))
+            return data
+        else:
+            data = np.rec.array(raw_data, dtype=dt)
+            return self.simplify(data.reshape(shape, order='F'))
 
     def simplify(self, array):
         """
@@ -201,35 +212,87 @@ def __str__(self):
         return str(self._blob[self.pos:])
 
 
-def pack(obj):
-    """
-    Packs an object into a blob to be compatible with mym.mex
+def pack(obj, compress=True):
+    blob = b"mYm\0"
+    blob += pack_obj(obj)
 
-    :param obj: object to be packed
-    :type obj: numpy.ndarray
-    """
-    if not isinstance(obj, np.ndarray):
-        raise DataJointError("Only numpy arrays can be saved in blobs")
+    if compress:
+        compressed = b'ZL123\0' + np.uint64(len(blob)).tostring() + zlib.compress(blob)
+        if len(compressed) < len(blob):
+            blob = compressed
+
+    return blob
+
+
+def pack_obj(obj):
+    blob = b''
+    if isinstance(obj, np.ndarray):
+        blob += pack_array(obj)
+    elif isinstance(obj, Mapping):  # TODO: check if this is a good inheritance check for dict etc.
+        blob += pack_dict(obj)
+    elif isinstance(obj, str):
+        blob += pack_array(np.array(obj, dtype=np.dtype('c')))
+    elif isinstance(obj, Iterable):
+        blob += pack_array(np.array(obj))
+    else:
+        raise DataJointError("Packing object of type %s currently not supported!" % type(obj))
+
+    return blob
+
+
+def pack_array(array):
+    if not isinstance(array, np.ndarray):
+        raise ValueError("argument must be a numpy array!")
 
-    blob = b"mYm\0A"  # TODO: extend to process other data types besides arrays
-    blob += np.asarray((len(obj.shape),) + obj.shape, dtype=np.uint64).tostring()
+    blob = b"A"
+    blob += np.array((len(array.shape), ) + array.shape, dtype=np.uint64).tostring()
 
-    is_complex = np.iscomplexobj(obj)
+    is_complex = np.iscomplexobj(array)
     if is_complex:
-        obj, imaginary = np.real(obj), np.imag(obj)
+        array, imaginary = np.real(array), np.imag(array)
 
-    type_number = rev_class_id[obj.dtype]
-    assert dtype_list[type_number] == obj.dtype, 'ambiguous or unknown array type'
-    blob += np.asarray(type_number, dtype=np.uint32).tostring()
-    blob += np.int8(is_complex).tostring() + b'\0\0\0'
-    blob += obj.tostring(order='F')
+    type_number = rev_class_id[array.dtype]
+
+    if dtype_list[type_number] is None:
+        raise DataJointError("Type %s is ambiguous or unknown" % array.dtype)
+
+    blob += np.array(type_number, dtype=np.uint32).tostring()
+
+    blob += np.int32(is_complex).tostring()
+    if type_number == 4:  # if dealing with character array
+        blob += ('\x00'.join(array.tostring(order='F').decode()) + '\x00').encode()
+    else:
+        blob += array.tostring(order='F')
 
     if is_complex:
         blob += imaginary.tostring(order='F')
 
-    compressed = b'ZL123\0' + np.uint64(len(blob)).tostring() + zlib.compress(blob)
-    if len(compressed) < len(blob):
-        blob = compressed
+    return blob
+
+
+def pack_string(value):
+    return value.encode('ascii') + b'\0'
+
+
+def pack_dict(obj):
+    """
+    Write dictionary object as a singular structure array
+    :param obj: dictionary object to serialize. The fields must be simple scalar or an array.
+    """
+    obj = OrderedDict(obj)
+    blob = b'S'
+    blob += np.array((2, 1, 1), dtype=np.uint64).tostring()
+    blob += np.array(len(obj), dtype=np.uint32).tostring()
+
+    # write out field names
+    for k in obj:
+        blob += pack_string(k)
+
+    for k, v in obj.items():
+        blob_part = pack_obj(v)
+        blob += np.array(len(blob_part), dtype=np.uint64).tostring()
+        blob += blob_part
+
     return blob
 
 

From 2d5ab68add8485a9ee457e58b2013de03297f755 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Fri, 16 Sep 2016 18:53:16 -0400
Subject: [PATCH 03/12] Simplify character blob fetch

---
 datajoint/blob.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datajoint/blob.py b/datajoint/blob.py
index 8f2eebfc0..fbc575133 100644
--- a/datajoint/blob.py
+++ b/datajoint/blob.py
@@ -101,7 +101,7 @@ def read_array(self, advance=True, n_bytes=None):
         if dtype_id == 4:  # if dealing with character array
             data = self.read_value(dtype, count=2 * n_elem)
             data = data[::2].astype('U1')
-            if self._simplify and n_dims == 2 and shape[0] == 1 or n_dims == 1:
+            if n_dims == 2 and shape[0] == 1 or n_dims == 1:
                 compact = data.squeeze()
                 data = compact if compact.shape == () else np.array(''.join(data.squeeze()))
                 shape = (1,)

From 14ef74d0901f227f0ef23cdd6f21d93244000500 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Fri, 16 Sep 2016 19:05:58 -0400
Subject: [PATCH 04/12] Support packing int and float into blob

---
 datajoint/blob.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/datajoint/blob.py b/datajoint/blob.py
index fbc575133..8f572e636 100644
--- a/datajoint/blob.py
+++ b/datajoint/blob.py
@@ -234,6 +234,8 @@ def pack_obj(obj):
         blob += pack_array(np.array(obj, dtype=np.dtype('c')))
     elif isinstance(obj, Iterable):
         blob += pack_array(np.array(obj))
+    elif isinstance(obj, int) or isinstance(obj, float):
+        blob += pack_array(np.array(obj))
     else:
         raise DataJointError("Packing object of type %s currently not supported!" % type(obj))
 

From 7c4eeae7ea7ac996c56a25a512132fb5f41d130f Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Fri, 16 Sep 2016 19:06:26 -0400
Subject: [PATCH 05/12] Add simple test for packing and unpacking dict to/from
 blob

---
 tests/test_blob.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tests/test_blob.py b/tests/test_blob.py
index f2026e36f..b13d2a6e4 100644
--- a/tests/test_blob.py
+++ b/tests/test_blob.py
@@ -18,6 +18,9 @@ def test_pack():
     x = np.int16(np.random.randn(1, 2, 3))
     assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
 
+    x = {'name': 'Anonymous', 'age': 15}
+    assert(x == unpack(pack(x)), "Dict do not match!")
+
 
 def test_complex():
     z = np.random.randn(8, 10) + 1j*np.random.randn(8,10)

From 60b5e011bffb17cbb9f17e12ac65fae83da7e949 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Fri, 16 Sep 2016 19:38:20 -0400
Subject: [PATCH 06/12] Pass through unpacking option

---
 datajoint/blob.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/datajoint/blob.py b/datajoint/blob.py
index 8f572e636..cee19dbad 100644
--- a/datajoint/blob.py
+++ b/datajoint/blob.py
@@ -298,9 +298,9 @@ def pack_dict(obj):
     return blob
 
 
-def unpack(blob):
+def unpack(blob, **kwargs):
     if blob is None:
         return None
 
-    return BlobReader(blob).unpack()
+    return BlobReader(blob, **kwargs).unpack()
 

From 37eb37fbdfd08af49b9ac613c3152d4fd2e7a56d Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Fri, 16 Sep 2016 19:38:46 -0400
Subject: [PATCH 07/12] Use proper assert in tests

---
 tests/test_blob.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tests/test_blob.py b/tests/test_blob.py
index b13d2a6e4..989e1b442 100644
--- a/tests/test_blob.py
+++ b/tests/test_blob.py
@@ -1,8 +1,7 @@
-
-
 import numpy as np
 from datajoint.blob import pack, unpack
 from numpy.testing import assert_array_equal, raises
+from nose.tools import assert_false, assert_true
 
 
 def test_pack():
@@ -19,7 +18,7 @@ def test_pack():
     assert_array_equal(x, unpack(pack(x)), "Arrays do not match!")
 
     x = {'name': 'Anonymous', 'age': 15}
-    assert(x == unpack(pack(x)), "Dict do not match!")
+    assert_true(x == unpack(pack(x), as_dict=True), "Dict do not match!")
 
 
 def test_complex():

From f780c5d71d79d415c16ec658875c0b6688c1e551 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Fri, 16 Sep 2016 19:53:47 -0400
Subject: [PATCH 08/12] Add nose in DevDockerfile

---
 DevDockerfile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/DevDockerfile b/DevDockerfile
index f52edbc93..95f5cbd42 100644
--- a/DevDockerfile
+++ b/DevDockerfile
@@ -3,4 +3,5 @@ FROM eywalker/jupyter
 MAINTAINER Edgar Y. Walker <edgar.walker@gmail.com>
 
 ADD . /src
-RUN pip install -e /src
+RUN pip install -e /src &&\
+    pip install nose

From 351ee2b86be292db383ff06dc45c8663687a68e1 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Fri, 16 Sep 2016 20:24:21 -0400
Subject: [PATCH 09/12] Remove unnecessary utility funtion from blob

---
 datajoint/blob.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/datajoint/blob.py b/datajoint/blob.py
index cee19dbad..c5a23aeb9 100644
--- a/datajoint/blob.py
+++ b/datajoint/blob.py
@@ -34,11 +34,6 @@
     b'ZL123\0': zlib.decompress
 }
 
-def forward_squeeze_shape(shape):
-    for p in range(len(shape)):
-        if shape[p] != 1:
-            break
-    return shape[p:]
 
 class BlobReader:
     def __init__(self, blob, simplify=False, as_dict=False):

From d465d8e50d0ecead4b17a9e09e6c7c964ecb20d0 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Fri, 16 Sep 2016 20:39:42 -0400
Subject: [PATCH 10/12] Handle iterable insert into BLOB correctly

---
 datajoint/blob.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/datajoint/blob.py b/datajoint/blob.py
index c5a23aeb9..e9a454f6e 100644
--- a/datajoint/blob.py
+++ b/datajoint/blob.py
@@ -228,7 +228,7 @@ def pack_obj(obj):
     elif isinstance(obj, str):
         blob += pack_array(np.array(obj, dtype=np.dtype('c')))
     elif isinstance(obj, Iterable):
-        blob += pack_array(np.array(obj))
+        blob += pack_array(np.array(list(obj)))
     elif isinstance(obj, int) or isinstance(obj, float):
         blob += pack_array(np.array(obj))
     else:

From c2faca79469a4587cacfe5ae1e44b4fe60b8fc38 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Fri, 16 Sep 2016 20:48:52 -0400
Subject: [PATCH 11/12] Add tests for serializing lists and iterators into BLOB

---
 tests/test_blob.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tests/test_blob.py b/tests/test_blob.py
index 989e1b442..2b6e45212 100644
--- a/tests/test_blob.py
+++ b/tests/test_blob.py
@@ -1,7 +1,7 @@
 import numpy as np
 from datajoint.blob import pack, unpack
 from numpy.testing import assert_array_equal, raises
-from nose.tools import assert_false, assert_true
+from nose.tools import assert_equal, assert_true
 
 
 def test_pack():
@@ -20,6 +20,12 @@ def test_pack():
     x = {'name': 'Anonymous', 'age': 15}
     assert_true(x == unpack(pack(x), as_dict=True), "Dict do not match!")
 
+    x = [1, 2, 3, 4]
+    assert_array_equal(x, unpack(pack(x)), "List did not pack/unpack correctly")
+
+    x = [1, 2, 3, 4].__iter__()
+    assert_array_equal(x, unpack(pack(x)), "Iterator did not pack/unpack correctly")
+
 
 def test_complex():
     z = np.random.randn(8, 10) + 1j*np.random.randn(8,10)

From 478f76ab1c6f9de87fcbede4c0fca61fe185c644 Mon Sep 17 00:00:00 2001
From: "Edgar Y. Walker" <edgar.walker@gmail.com>
Date: Fri, 16 Sep 2016 20:53:19 -0400
Subject: [PATCH 12/12] Change testing of iterator serialization to BLOB

---
 tests/test_blob.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_blob.py b/tests/test_blob.py
index 2b6e45212..3cb35689a 100644
--- a/tests/test_blob.py
+++ b/tests/test_blob.py
@@ -23,8 +23,8 @@ def test_pack():
     x = [1, 2, 3, 4]
     assert_array_equal(x, unpack(pack(x)), "List did not pack/unpack correctly")
 
-    x = [1, 2, 3, 4].__iter__()
-    assert_array_equal(x, unpack(pack(x)), "Iterator did not pack/unpack correctly")
+    x = [1, 2, 3, 4]
+    assert_array_equal(x, unpack(pack(x.__iter__())), "Iterator did not pack/unpack correctly")
 
 
 def test_complex():