embray · Oct 22, 2015 · Sep 1, 2015 · Sep 11, 2015 · Sep 11, 2015 · Sep 11, 2015
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -866,6 +866,21 @@ Bug Fixes
     between values that are different types but have the same representation
     (ex: the string '0' versus the number 0). [#4122]
 
+  - Miscellaneous fixes for supporting Numpy 1.10. [#4228]
+
+  - Fixed an issue where writing a column of unicode strings to a FITS table
+    resulted in a quadrupling of size of the column (i.e. the format of the
+    FITS column was 4 characters for every one in the original strings).
+    [#4228]
+
+  - Added support for an obscure case (but nonetheless allowed by the FITS
+    standard) where a column has some TDIMn keyword, but a repeat count in
+    the TFORMn column greater than the number of elements implied by the
+    TDIMn.  For example TFORMn = 100I, but TDIMn = '(5,5)'.  In this case
+    the TDIMn implies 5x5 arrays in the column, but the TFORMn implies
+    a 100 element 1-D array in the column.  In this case the TDIM takes
+    precedence, and the remaining bytes in the column are ignored. [#4228]
+
 - ``astropy.io.misc``
 
 - ``astropy.io.registry``
@@ -899,7 +914,12 @@ Bug Fixes
 Other Changes and Additions
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-- Nothing changed yet.
+- All tests now pass against Numpy v1.10.x. This implies nominal support for
+  Numpy 1.10.x moving forward (but there may still be unknown issues). For
+  example, there is already a known performance issue with tables containing
+  large multi-dimensional columns--for example, tables that contain entire
+  images in one or more of their columns.  This is a known upstream issue in
+  Numpy. [#4259]
 
 
 1.0.5 (2015-10-05)

diff --git a/astropy/io/fits/_numpy_hacks.py b/astropy/io/fits/_numpy_hacks.py
@@ -0,0 +1,73 @@
+"""
+This module is for functions that do tricky things with Numpy arrays and dtypes
+that are not normally supported in Numpy (but can work in limited cases
+relevant to FITS) or that otherwise require workarounds.
+"""
+
+
+import numpy as np
+
+
+def realign_dtype(dtype, offsets):
+    """
+    Given a Numpy struct dtype object an a list of integer offsets, with one
+    offset per field in the dtype, returns a new dtype where each field has the
+    given offset.
+
+    All offsets must be non-negative integers, but otherwise have no
+    restrictions, and may overlap, per the usual rules for creating struct
+    dtypes.  The new dtype will have an itemsize equal to the offset of the
+    right-most field plus the width of that field.
+
+    One restriction of this function is that it must not be used with object
+    arrays--incorrect offsets may lead to invalid pointers in the arrays.
+    However, this function is really only meant for use by astropy.io.fits and
+    object arrays are not supported for FITS data anyhow.
+
+    This function is used primarily to get around a shortcoming in Numpy that
+    it is currently impossible to create dtypes with arbitrary offsets, *and*
+    that have zero-width fields.  Both of these features are needed for full
+    FITS support.  However, this will be fixed in a future version of Numpy at
+    which point use of this hack can be deprecated.  See
+    https://github.com/numpy/numpy/pull/6430
+    """
+
+    # Previously this was implemented in C, but then I realized that the C
+    # version is not needed--the workaround is to use dtype.__setstate__
+    # Note: There is a comment in the Numpy source code (see
+    # https://github.com/numpy/numpy/blob/v1.10.1/numpy/core/src/multiarray/descriptor.c#L2226)
+    # that this may be changed at some point.  But hopefully by then the fixes
+    # in #6430 will be implemented, making this hack unnecessary to begin with.
+
+    cls, args, state = dtype.__reduce__()
+
+    names, fields = state[3:5]
+    fields = fields.copy()
+
+    max_offset = 0
+    itemsize = state[5]  # Default to the original itemsize
+
+    if fields is None or len(offsets) != len(names):
+        raise ValueError(
+            "Dtype must be a structured dtype, and length of offsets list "
+            "must be the same as the number of fields.")
+
+    for name, offset in zip(names, offsets):
+        field = fields[name]
+        if offset == field[1]:
+            continue
+
+        fields[name] = (field[0], offset)
+
+        if offset > max_offset:
+            itemsize = offset + field[0].itemsize
+            max_offset = offset
+
+    new_typespec = '|V{0}'.format(itemsize)
+
+    new_state = state[:4] + (fields, itemsize) + state[6:]
+
+    new_dtype = cls(new_typespec, *args[1:])
+    new_dtype.__setstate__(new_state)
+
+    return new_dtype
diff --git a/astropy/io/fits/column.py b/astropy/io/fits/column.py
@@ -12,6 +12,7 @@
 import numpy as np
 from numpy import char as chararray
 
+from . import _numpy_hacks as nh
 from .card import Card, CARD_LENGTH
 from .util import (pairwise, _is_int, _convert_array, encode_ascii, cmp,
                    NotifierMixin)
@@ -944,6 +945,12 @@ def _convert_to_valid_data_type(self, array):
         else:
             format = self.format
             dims = self._dims
+
+            if dims:
+                shape = dims[:-1] if 'A' in format else dims
+                shape = (len(array),) + shape
+                array = array.reshape(shape)
+
             if 'P' in format or 'Q' in format:
                 return array
             elif 'A' in format:
@@ -1264,9 +1271,34 @@ def __getattr__(self, name):
 
     @lazyproperty
     def dtype(self):
-        dtypes = [f.dtype for idx, f in enumerate(self.formats)]
-        names = [n for idx, n in enumerate(self.names)]
-        return np.dtype(list(zip(names, dtypes)))
+        # Note: This previously returned a dtype that just used the raw field
+        # widths based on the format's repeat count, and did not incorporate
+        # field *shapes* as provided by TDIMn keywords.
+        # Now this incorporates TDIMn from the start, which makes *this* method
+        # a little more complicated, but simplifies code elsewhere (for example
+        # fields will have the correct shapes even in the raw recarray).
+        fields = []
+        offsets = [0]
+
+        for name, format_, dim in zip(self.names, self.formats, self._dims):
+            dt = format_.dtype
+
+            if len(offsets) < len(self.formats):
+                # Note: the size of the *original* format_ may be greater than
+                # one would expect from the number of elements determined by
+                # dim.  The FITS format allows this--the rest of the field is
+                # filled with undefined values.
+                offsets.append(offsets[-1] + dt.itemsize)
+
+            if dim:
+                if format_.format == 'A':
+                    dt = np.dtype((dt.char + str(dim[-1]), dim[:-1]))
+                else:
+                    dt = np.dtype((dt.base, dim))
+
+            fields.append((name, dt))
+
+        return nh.realign_dtype(np.dtype(fields), offsets)
 
     @lazyproperty
     def _arrays(self):
@@ -2003,7 +2035,14 @@ def _convert_record2fits(format):
 
     recformat, kind, dtype = _dtype_to_recformat(format)
     shape = dtype.shape
-    option = str(dtype.base.itemsize)
+    itemsize = dtype.base.itemsize
+    if dtype.char == 'U':
+        # Unicode dtype--itemsize is 4 times actual ASCII character length,
+        # which what matters for FITS column formats
+        # Use dtype.base--dtype may be a multi-dimensional dtype
+        itemsize = itemsize // 4
+
+    option = str(itemsize)
 
     ndims = len(shape)
     repeat = 1
@@ -2048,11 +2087,12 @@ def _dtype_to_recformat(dtype):
         dtype = np.dtype(dtype)
 
     kind = dtype.base.kind
-    itemsize = dtype.base.itemsize
-    recformat = kind + str(itemsize)
 
     if kind in ('U', 'S'):
         recformat = kind = 'a'
+    else:
+        itemsize = dtype.base.itemsize
+        recformat = kind + str(itemsize)
 
     return recformat, kind, dtype