import types
from abc import abstractmethod
from collections import OrderedDict
from copy import deepcopy
from uuid import uuid4
from warnings import warn

import h5py
import numpy as np
import pandas as pd

from .data_utils import DataIO, append_data, extend_data
from .utils import docval, get_docval, getargs, ExtenderMeta, get_data_shape, popargs, LabelledDict


def _set_exp(cls):
    """Set a class as being experimental"""
    cls._experimental = True


def _exp_warn_msg(cls):
    """Generate a warning message experimental features"""
    pfx = cls
    if isinstance(cls, type):
        pfx = cls.__name__
    msg = ('%s is experimental -- it may be removed in the future and '
           'is not guaranteed to maintain backward compatibility') % pfx
    return msg


class AbstractContainer(metaclass=ExtenderMeta):
    # The name of the class attribute that subclasses use to autogenerate properties
    # This parameterization is supplied in case users would like to configure
    # the class attribute name to something domain-specific

    _experimental = False

    _fieldsname = '__fields__'

    _data_type_attr = 'data_type'

    # Subclasses use this class attribute to add properties to autogenerate
    # Autogenerated properties will store values in self.__field_values
    __fields__ = tuple()

    # This field is automatically set by __gather_fields before initialization.
    # It holds all the values in __fields__ for this class and its parent classes.
    __fieldsconf = tuple()

    _pconf_allowed_keys = {'name', 'doc', 'settable'}

    # Override the _setter factor function, so directives that apply to
    # Container do not get used on Data
    @classmethod
    def _setter(cls, field):
        """
        Make a setter function for creating a :py:func:`property`
        """
        name = field['name']

        if not field.get('settable', True):
            return None

        def setter(self, val):
            if val is None:
                return
            if name in self.fields:
                msg = "can't set attribute '%s' -- already set" % name
                raise AttributeError(msg)
            self.fields[name] = val

        return setter

    @classmethod
    def _getter(cls, field):
        """
        Make a getter function for creating a :py:func:`property`
        """
        doc = field.get('doc')
        name = field['name']

        def getter(self):
            return self.fields.get(name)

        setattr(getter, '__doc__', doc)
        return getter

    @staticmethod
    def _check_field_spec(field):
        """
        A helper function for __gather_fields to make sure we are always working
        with a dict specification and that the specification contains the correct keys
        """
        tmp = field
        if isinstance(tmp, dict):
            if 'name' not in tmp:
                raise ValueError("must specify 'name' if using dict in __fields__")
        else:
            tmp = {'name': tmp}
        return tmp

    @classmethod
    def _check_field_spec_keys(cls, field_conf):
        for k in field_conf:
            if k not in cls._pconf_allowed_keys:
                msg = ("Unrecognized key '%s' in %s config '%s' on %s"
                       % (k, cls._fieldsname, field_conf['name'], cls.__name__))
                raise ValueError(msg)

    @classmethod
    def _get_fields(cls):
        return getattr(cls, cls._fieldsname)

    @classmethod
    def _set_fields(cls, value):
        return setattr(cls, cls._fieldsname, value)

    @classmethod
    def get_fields_conf(cls):
        return cls.__fieldsconf

    @ExtenderMeta.pre_init
    def __gather_fields(cls, name, bases, classdict):
        '''
        This classmethod will be called during class declaration in the metaclass to automatically
        create setters and getters for fields that need to be exported
        '''
        fields = cls._get_fields()
        if not isinstance(fields, tuple):
            msg = "'%s' must be of type tuple" % cls._fieldsname
            raise TypeError(msg)

        # check field specs and create map from field name to field conf dictionary
        fields_dict = OrderedDict()
        for f in fields:
            pconf = cls._check_field_spec(f)
            cls._check_field_spec_keys(pconf)
            fields_dict[pconf['name']] = pconf
        all_fields_conf = list(fields_dict.values())

        # check whether this class overrides __fields__
        if len(bases):
            # find highest base class that is an AbstractContainer (parent is higher than children)
            base_cls = None
            for base_cls in reversed(bases):
                if issubclass(base_cls, AbstractContainer):
                    break

            base_fields = base_cls._get_fields()  # tuple of field names from base class
            if base_fields is not fields:
                # check whether new fields spec already exists in base class
                fields_to_remove_from_base = list()
                for field_name in fields_dict:
                    if field_name in base_fields:
                        fields_to_remove_from_base.append(field_name)
                # prepend field specs from base class to fields list of this class
                # but only field specs that are not redefined in this class
                base_fields_conf = base_cls.get_fields_conf()  # tuple of fields configurations from base class
                base_fields_conf_to_add = list()
                for pconf in base_fields_conf:
                    if pconf['name'] not in fields_to_remove_from_base:
                        base_fields_conf_to_add.append(pconf)
                all_fields_conf[0:0] = base_fields_conf_to_add

        # create getter and setter if attribute does not already exist
        # if 'doc' not specified in __fields__, use doc from docval of __init__
        docs = {dv['name']: dv['doc'] for dv in get_docval(cls.__init__)}
        for field_conf in all_fields_conf:
            pname = field_conf['name']
            field_conf.setdefault('doc', docs.get(pname))
            if not hasattr(cls, pname):
                setattr(cls, pname, property(cls._getter(field_conf), cls._setter(field_conf)))

        cls._set_fields(tuple(field_conf['name'] for field_conf in all_fields_conf))
        cls.__fieldsconf = tuple(all_fields_conf)

    def __new__(cls, *args, **kwargs):
        """
        Static method of the object class called by Python to create the object first and then
         __init__() is called to initialize the object's attributes.

        NOTE: this method is called directly from ObjectMapper.__new_container__ during the process of
        constructing the object from builders that are read from a file.
        """
        inst = super().__new__(cls)
        if cls._experimental:
            warn(_exp_warn_msg(cls))
        inst.__container_source = kwargs.pop('container_source', None)
        inst.__parent = None
        inst.__children = list()
        inst.__modified = True
        inst.__object_id = kwargs.pop('object_id', str(uuid4()))
        # this variable is being passed in from ObjectMapper.__new_container__ and is
        # reset to False in that method after the object has been initialized by __init__
        inst._in_construct_mode = kwargs.pop('in_construct_mode', False)
        inst.parent = kwargs.pop('parent', None)
        return inst

    @docval({'name': 'name', 'type': str, 'doc': 'the name of this container'})
    def __init__(self, **kwargs):
        name = getargs('name', kwargs)
        if '/' in name:
            raise ValueError("name '" + name + "' cannot contain '/'")
        self.__name = name
        self.__field_values = dict()

    @property
    def name(self):
        '''
        The name of this Container
        '''
        return self.__name

    @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to search for', 'default': None})
    def get_ancestor(self, **kwargs):
        """
        Traverse parent hierarchy and return first instance of the specified data_type
        """
        data_type = getargs('data_type', kwargs)
        if data_type is None:
            return self.parent
        p = self.parent
        while p is not None:
            if getattr(p, p._data_type_attr) == data_type:
                return p
            p = p.parent
        return None

    @property
    def fields(self):
        return self.__field_values

    @property
    def object_id(self):
        if self.__object_id is None:
            self.__object_id = str(uuid4())
        return self.__object_id

    @docval({'name': 'recurse', 'type': bool,
             'doc': "whether or not to change the object ID of this container's children", 'default': True})
    def generate_new_id(self, **kwargs):
        """Changes the object ID of this Container and all of its children to a new UUID string."""
        recurse = getargs('recurse', kwargs)
        self.__object_id = str(uuid4())
        self.set_modified()
        if recurse:
            for c in self.children:
                c.generate_new_id(**kwargs)

    @property
    def modified(self):
        return self.__modified

    @docval({'name': 'modified', 'type': bool,
             'doc': 'whether or not this Container has been modified', 'default': True})
    def set_modified(self, **kwargs):
        modified = getargs('modified', kwargs)
        self.__modified = modified
        if modified and isinstance(self.parent, Container):
            self.parent.set_modified()

    @property
    def children(self):
        return tuple(self.__children)

    @docval({'name': 'child', 'type': 'Container',
             'doc': 'the child Container for this Container', 'default': None})
    def add_child(self, **kwargs):
        warn(DeprecationWarning('add_child is deprecated. Set the parent attribute instead.'))
        child = getargs('child', kwargs)
        if child is not None:
            # if child.parent is a Container, then the mismatch between child.parent and parent
            # is used to make a soft/external link from the parent to a child elsewhere
            # if child.parent is not a Container, it is either None or a Proxy and should be set to self
            if not isinstance(child.parent, AbstractContainer):
                # actually add the child to the parent in parent setter
                child.parent = self
        else:
            warn('Cannot add None as child to a container %s' % self.name)

    @classmethod
    def type_hierarchy(cls):
        return cls.__mro__

    @property
    def container_source(self):
        '''
        The source of this Container
        '''
        return self.__container_source

    @container_source.setter
    def container_source(self, source):
        if self.__container_source is not None:
            raise Exception('cannot reassign container_source')
        self.__container_source = source

    @property
    def parent(self):
        '''
        The parent Container of this Container
        '''
        # do it this way because __parent may not exist yet (not set in constructor)
        return getattr(self, '_AbstractContainer__parent', None)

    @parent.setter
    def parent(self, parent_container):
        if self.parent is parent_container:
            return

        if self.parent is not None:
            if isinstance(self.parent, AbstractContainer):
                raise ValueError(('Cannot reassign parent to Container: %s. '
                                  'Parent is already: %s.' % (repr(self), repr(self.parent))))
            else:
                if parent_container is None:
                    raise ValueError("Got None for parent of '%s' - cannot overwrite Proxy with NoneType" % repr(self))
                # NOTE this assumes isinstance(parent_container, Proxy) but we get a circular import
                # if we try to do that
                if self.parent.matches(parent_container):
                    self.__parent = parent_container
                    parent_container.__children.append(self)
                    parent_container.set_modified()
                else:
                    self.__parent.add_candidate(parent_container)
        else:
            self.__parent = parent_container
            if isinstance(parent_container, Container):
                parent_container.__children.append(self)
                parent_container.set_modified()

    def _remove_child(self, child):
        """Remove a child Container. Intended for use in subclasses that allow dynamic addition of child Containers."""
        if not isinstance(child, AbstractContainer):
            raise ValueError('Cannot remove non-AbstractContainer object from children.')
        if child not in self.children:
            raise ValueError("%s '%s' is not a child of %s '%s'." % (child.__class__.__name__, child.name,
                                                                     self.__class__.__name__, self.name))
        child.__parent = None
        self.__children.remove(child)
        child.set_modified()
        self.set_modified()

    def reset_parent(self):
        """Reset the parent of this Container to None and remove the Container from the children of its parent.

        Use with caution. This can result in orphaned containers and broken links.
        """
        if self.parent is None:
            return
        elif isinstance(self.parent, AbstractContainer):
            self.parent._remove_child(self)
        else:
            raise ValueError("Cannot reset parent when parent is not an AbstractContainer: %s" % repr(self.parent))


class Container(AbstractContainer):
    """A container that can contain other containers and has special functionality for printing."""

    _pconf_allowed_keys = {'name', 'child', 'required_name', 'doc', 'settable'}

    @classmethod
    def _setter(cls, field):
        """Returns a list of setter functions for the given field to be added to the class during class declaration."""
        super_setter = AbstractContainer._setter(field)
        ret = [super_setter]
        # create setter with check for required name
        # the AbstractContainer that is passed to the setter must have name = required_name
        if field.get('required_name', None) is not None:
            required_name = field['required_name']
            idx1 = len(ret) - 1

            def container_setter(self, val):
                if val is not None:
                    if not isinstance(val, AbstractContainer):
                        msg = ("Field '%s' on %s has a required name and must be a subclass of AbstractContainer."
                               % (field['name'], self.__class__.__name__))
                        raise ValueError(msg)
                    if val.name != required_name:
                        msg = ("Field '%s' on %s must be named '%s'."
                               % (field['name'], self.__class__.__name__, required_name))
                        raise ValueError(msg)
                ret[idx1](self, val)  # call the previous setter

            ret.append(container_setter)

        # create setter that accepts a value or tuple, list, or dict or values and sets the value's parent to self
        if field.get('child', False):
            idx2 = len(ret) - 1

            def container_setter(self, val):
                ret[idx2](self, val)  # call the previous setter
                if val is not None:
                    if isinstance(val, (tuple, list)):
                        pass
                    elif isinstance(val, dict):
                        val = val.values()
                    else:
                        val = [val]
                    for v in val:
                        if not isinstance(v.parent, Container):
                            v.parent = self
                        else:
                            # the ObjectMapper will create a link from self (parent) to v (child with existing parent)
                            # still need to mark self as modified
                            self.set_modified()

            ret.append(container_setter)
        return ret[-1]  # return the last setter (which should call the previous setters, if applicable)

    def __repr__(self):
        cls = self.__class__
        template = "%s %s.%s at 0x%d" % (self.name, cls.__module__, cls.__name__, id(self))
        if len(self.fields):
            template += "\nFields:\n"
        for k in sorted(self.fields):  # sorted to enable tests
            v = self.fields[k]
            # if isinstance(v, DataIO) or not hasattr(v, '__len__') or len(v) > 0:
            if hasattr(v, '__len__'):
                if isinstance(v, (np.ndarray, list, tuple)):
                    if len(v) > 0:
                        template += "  {}: {}\n".format(k, self.__smart_str(v, 1))
                elif v:
                    template += "  {}: {}\n".format(k, self.__smart_str(v, 1))
            else:
                template += "  {}: {}\n".format(k, v)
        return template

    @staticmethod
    def __smart_str(v, num_indent):
        """
        Print compact string representation of data.

        If v is a list, try to print it using numpy. This will condense the string
        representation of datasets with many elements. If that doesn't work, just print the list.

        If v is a dictionary, print the name and type of each element

        If v is a set, print it sorted

        If v is a neurodata_type, print the name of type

        Otherwise, use the built-in str()
        Parameters
        ----------
        v

        Returns
        -------
        str

        """

        if isinstance(v, list) or isinstance(v, tuple):
            if len(v) and isinstance(v[0], AbstractContainer):
                return Container.__smart_str_list(v, num_indent, '(')
            try:
                return str(np.asarray(v))
            except ValueError:
                return Container.__smart_str_list(v, num_indent, '(')
        elif isinstance(v, dict):
            return Container.__smart_str_dict(v, num_indent)
        elif isinstance(v, set):
            return Container.__smart_str_list(sorted(list(v)), num_indent, '{')
        elif isinstance(v, AbstractContainer):
            return "{} {}".format(getattr(v, 'name'), type(v))
        else:
            return str(v)

    @staticmethod
    def __smart_str_list(str_list, num_indent, left_br):
        if left_br == '(':
            right_br = ')'
        if left_br == '{':
            right_br = '}'
        if len(str_list) == 0:
            return left_br + ' ' + right_br
        indent = num_indent * 2 * ' '
        indent_in = (num_indent + 1) * 2 * ' '
        out = left_br
        for v in str_list[:-1]:
            out += '\n' + indent_in + Container.__smart_str(v, num_indent + 1) + ','
        if str_list:
            out += '\n' + indent_in + Container.__smart_str(str_list[-1], num_indent + 1)
        out += '\n' + indent + right_br
        return out

    @staticmethod
    def __smart_str_dict(d, num_indent):
        left_br = '{'
        right_br = '}'
        if len(d) == 0:
            return left_br + ' ' + right_br
        indent = num_indent * 2 * ' '
        indent_in = (num_indent + 1) * 2 * ' '
        out = left_br
        keys = sorted(list(d.keys()))
        for k in keys[:-1]:
            out += '\n' + indent_in + Container.__smart_str(k, num_indent + 1) + ' ' + str(type(d[k])) + ','
        if keys:
            out += '\n' + indent_in + Container.__smart_str(keys[-1], num_indent + 1) + ' ' + str(type(d[keys[-1]]))
        out += '\n' + indent + right_br
        return out


class Data(AbstractContainer):
    """
    A class for representing dataset containers
    """

    @docval({'name': 'name', 'type': str, 'doc': 'the name of this container'},
            {'name': 'data', 'type': ('scalar_data', 'array_data', 'data'), 'doc': 'the source of the data'})
    def __init__(self, **kwargs):
        data = popargs('data', kwargs)
        super().__init__(**kwargs)
        self.__data = data

    @property
    def data(self):
        return self.__data

    @property
    def shape(self):
        """
        Get the shape of the data represented by this container
        :return: Shape tuple
        :rtype: tuple of ints
        """
        return get_data_shape(self.__data)

    @docval({'name': 'dataio', 'type': DataIO, 'doc': 'the DataIO to apply to the data held by this Data'})
    def set_dataio(self, **kwargs):
        """
        Apply DataIO object to the data held by this Data object
        """
        dataio = getargs('dataio', kwargs)
        dataio.data = self.__data
        self.__data = dataio

    @docval({'name': 'func', 'type': types.FunctionType, 'doc': 'a function to transform *data*'})
    def transform(self, **kwargs):
        """
        Transform data from the current underlying state.

        This function can be used to permanently load data from disk, or convert to a different
        representation, such as a torch.Tensor
        """
        func = getargs('func', kwargs)
        self.__data = func(self.__data)
        return self

    def __bool__(self):
        if self.data is not None:
            if isinstance(self.data, (np.ndarray, tuple, list)):
                return len(self.data) != 0
            if self.data:
                return True
        return False

    def __len__(self):
        return len(self.__data)

    def __getitem__(self, args):
        return self.get(args)

    def get(self, args):
        if isinstance(self.data, (tuple, list)) and isinstance(args, (tuple, list, np.ndarray)):
            return [self.data[i] for i in args]
        if isinstance(self.data, h5py.Dataset) and isinstance(args, np.ndarray):
            # This is needed for h5py 2.9 compatability
            args = args.tolist()
        return self.data[args]

    def append(self, arg):
        self.__data = append_data(self.__data, arg)

    def extend(self, arg):
        """
        The extend_data method adds all the elements of the iterable arg to the
        end of the data of this Data container.

        :param arg: The iterable to add to the end of this VectorData
        """
        self.__data = extend_data(self.__data, arg)


class DataRegion(Data):

    @property
    @abstractmethod
    def data(self):
        '''
        The target data that this region applies to
        '''
        pass

    @property
    @abstractmethod
    def region(self):
        '''
        The region that indexes into data e.g. slice or list of indices
        '''
        pass


class MultiContainerInterface(Container):
    """Class that dynamically defines methods to support a Container holding multiple Containers of the same type.

    To use, extend this class and create a dictionary as a class attribute with any of the following keys:
    * 'attr' to name the attribute that stores the Container instances
    * 'type' to provide the Container object type (type or list/tuple of types, type can be a docval macro)
    * 'add' to name the method for adding Container instances
    * 'get' to name the method for getting Container instances
    * 'create' to name the method for creating Container instances (only if a single type is specified)

    If the attribute does not exist in the class, it will be generated. If it does exist, it should behave like a dict.

    The keys 'attr', 'type', and 'add' are required.
    """

    def __new__(cls, *args, **kwargs):
        if cls is MultiContainerInterface:
            raise TypeError("Can't instantiate class MultiContainerInterface.")
        if not hasattr(cls, '__clsconf__'):
            raise TypeError("MultiContainerInterface subclass %s is missing __clsconf__ attribute. Please check that "
                            "the class is properly defined." % cls.__name__)
        return super().__new__(cls, *args, **kwargs)

    @staticmethod
    def __add_article(noun):
        if isinstance(noun, tuple):
            noun = noun[0]
        if isinstance(noun, type):
            noun = noun.__name__
        if noun[0] in ('aeiouAEIOU'):
            return 'an %s' % noun
        return 'a %s' % noun

    @staticmethod
    def __join(argtype):
        """Return a grammatical string representation of a list or tuple of classes or text.

        Examples:
        cls.__join(Container) returns "Container"
        cls.__join((Container, )) returns "Container"
        cls.__join((Container, Data)) returns "Container or Data"
        cls.__join((Container, Data, Subcontainer)) returns "Container, Data, or Subcontainer"
        """

        def tostr(x):
            return x.__name__ if isinstance(x, type) else x

        if isinstance(argtype, (list, tuple)):
            args_str = [tostr(x) for x in argtype]
            if len(args_str) == 1:
                return args_str[0]
            if len(args_str) == 2:
                return " or ".join(tostr(x) for x in args_str)
            else:
                return ", ".join(tostr(x) for x in args_str[:-1]) + ', or ' + args_str[-1]
        else:
            return tostr(argtype)

    @classmethod
    def __make_get(cls, func_name, attr_name, container_type):
        doc = "Get %s from this %s" % (cls.__add_article(container_type), cls.__name__)

        @docval({'name': 'name', 'type': str, 'doc': 'the name of the %s' % cls.__join(container_type),
                 'default': None},
                rtype=container_type, returns='the %s with the given name' % cls.__join(container_type),
                func_name=func_name, doc=doc)
        def _func(self, **kwargs):
            name = getargs('name', kwargs)
            d = getattr(self, attr_name)
            ret = None
            if name is None:
                if len(d) > 1:
                    msg = ("More than one element in %s of %s '%s' -- must specify a name."
                           % (attr_name, cls.__name__, self.name))
                    raise ValueError(msg)
                elif len(d) == 0:
                    msg = "%s of %s '%s' is empty." % (attr_name, cls.__name__, self.name)
                    raise ValueError(msg)
                else:  # only one item in dict
                    for v in d.values():
                        ret = v
            else:
                ret = d.get(name)
                if ret is None:
                    msg = "'%s' not found in %s of %s '%s'." % (name, attr_name, cls.__name__, self.name)
                    raise KeyError(msg)
            return ret

        return _func

    @classmethod
    def __make_getitem(cls, attr_name, container_type):
        doc = "Get %s from this %s" % (cls.__add_article(container_type), cls.__name__)

        @docval({'name': 'name', 'type': str, 'doc': 'the name of the %s' % cls.__join(container_type),
                 'default': None},
                rtype=container_type, returns='the %s with the given name' % cls.__join(container_type),
                func_name='__getitem__', doc=doc)
        def _func(self, **kwargs):
            # NOTE this is the same code as the getter but with different error messages
            name = getargs('name', kwargs)
            d = getattr(self, attr_name)
            ret = None
            if name is None:
                if len(d) > 1:
                    msg = ("More than one %s in %s '%s' -- must specify a name."
                           % (cls.__join(container_type), cls.__name__, self.name))
                    raise ValueError(msg)
                elif len(d) == 0:
                    msg = "%s '%s' is empty." % (cls.__name__, self.name)
                    raise ValueError(msg)
                else:  # only one item in dict
                    for v in d.values():
                        ret = v
            else:
                ret = d.get(name)
                if ret is None:
                    msg = "'%s' not found in %s '%s'." % (name, cls.__name__, self.name)
                    raise KeyError(msg)
            return ret

        return _func

    @classmethod
    def __make_add(cls, func_name, attr_name, container_type):
        doc = "Add one or multiple %s objects to this %s" % (cls.__join(container_type), cls.__name__)

        @docval({'name': attr_name, 'type': (list, tuple, dict, container_type),
                 'doc': 'one or multiple %s objects to add to this %s' % (cls.__join(container_type),  cls.__name__)},
                func_name=func_name, doc=doc)
        def _func(self, **kwargs):
            container = getargs(attr_name, kwargs)
            if isinstance(container, container_type):
                containers = [container]
            elif isinstance(container, dict):
                containers = container.values()
            else:
                containers = container
            d = getattr(self, attr_name)
            for tmp in containers:
                if not isinstance(tmp.parent, Container):
                    tmp.parent = self
                else:
                    # the ObjectMapper will create a link from self (parent) to tmp (child with existing parent)
                    # still need to mark self as modified
                    self.set_modified()
                if tmp.name in d:
                    msg = "'%s' already exists in %s '%s'" % (tmp.name, cls.__name__, self.name)
                    raise ValueError(msg)
                d[tmp.name] = tmp
            return container

        return _func

    @classmethod
    def __make_create(cls, func_name, add_name, container_type):
        doc = "Create %s object and add it to this %s" % (cls.__add_article(container_type), cls.__name__)

        @docval(*get_docval(container_type.__init__), func_name=func_name, doc=doc,
                returns="the %s object that was created" % cls.__join(container_type), rtype=container_type)
        def _func(self, **kwargs):
            ret = container_type(**kwargs)
            getattr(self, add_name)(ret)
            return ret

        return _func

    @classmethod
    def __make_constructor(cls, clsconf):
        args = list()
        for conf in clsconf:
            attr_name = conf['attr']
            container_type = conf['type']
            args.append({'name': attr_name, 'type': (list, tuple, dict, container_type),
                         'doc': '%s to store in this interface' % cls.__join(container_type), 'default': dict()})

        args.append({'name': 'name', 'type': str, 'doc': 'the name of this container', 'default': cls.__name__})

        @docval(*args, func_name='__init__')
        def _func(self, **kwargs):
            super().__init__(name=kwargs['name'])
            for conf in clsconf:
                attr_name = conf['attr']
                add_name = conf['add']
                container = popargs(attr_name, kwargs)
                add = getattr(self, add_name)
                add(container)

        return _func

    @classmethod
    def __make_getter(cls, attr):
        """Make a getter function for creating a :py:func:`property`"""

        def _func(self):
            # initialize the field to an empty labeled dict if it has not yet been
            # do this here to avoid creating default __init__ which may or may not be overridden in
            # custom classes and dynamically generated classes
            if attr not in self.fields:
                def _remove_child(child):
                    if child.parent is self:
                        self._remove_child(child)
                self.fields[attr] = LabelledDict(attr, remove_callable=_remove_child)

            return self.fields.get(attr)

        return _func

    @classmethod
    def __make_setter(cls, add_name):
        """Make a setter function for creating a :py:func:`property`"""

        @docval({'name': 'val', 'type': (list, tuple, dict), 'doc': 'the sub items to add', 'default': None})
        def _func(self, **kwargs):
            val = getargs('val', kwargs)
            if val is None:
                return
            getattr(self, add_name)(val)

        return _func

    @ExtenderMeta.pre_init
    def __build_class(cls, name, bases, classdict):
        """Verify __clsconf__ and create methods based on __clsconf__.
        This method is called prior to __new__ and __init__ during class declaration in the metaclass.
        """
        if not hasattr(cls, '__clsconf__'):
            return

        multi = False
        if isinstance(cls.__clsconf__, dict):
            clsconf = [cls.__clsconf__]
        elif isinstance(cls.__clsconf__, list):
            multi = True
            clsconf = cls.__clsconf__
        else:
            raise TypeError("'__clsconf__' for MultiContainerInterface subclass %s must be a dict or a list of "
                            "dicts." % cls.__name__)

        for conf_index, conf_dict in enumerate(clsconf):
            cls.__build_conf_methods(conf_dict, conf_index, multi)

        # make __getitem__ (square bracket access) only if one conf type is defined
        if len(clsconf) == 1:
            attr = clsconf[0].get('attr')
            container_type = clsconf[0].get('type')
            setattr(cls, '__getitem__', cls.__make_getitem(attr, container_type))

        # create the constructor, only if it has not been overridden
        # i.e. it is the same method as the parent class constructor
        if '__init__' not in cls.__dict__:
            setattr(cls, '__init__', cls.__make_constructor(clsconf))

    @classmethod
    def __build_conf_methods(cls, conf_dict, conf_index, multi):
        # get add method name
        add = conf_dict.get('add')
        if add is None:
            msg = "MultiContainerInterface subclass %s is missing 'add' key in __clsconf__" % cls.__name__
            if multi:
                msg += " at index %d" % conf_index
            raise ValueError(msg)

        # get container attribute name
        attr = conf_dict.get('attr')
        if attr is None:
            msg = "MultiContainerInterface subclass %s is missing 'attr' key in __clsconf__" % cls.__name__
            if multi:
                msg += " at index %d" % conf_index
            raise ValueError(msg)

        # get container type
        container_type = conf_dict.get('type')
        if container_type is None:
            msg = "MultiContainerInterface subclass %s is missing 'type' key in __clsconf__" % cls.__name__
            if multi:
                msg += " at index %d" % conf_index
            raise ValueError(msg)

        # create property with the name given in 'attr' only if the attribute is not already defined
        if not hasattr(cls, attr):
            getter = cls.__make_getter(attr)
            setter = cls.__make_setter(add)
            doc = "a dictionary containing the %s in this %s" % (cls.__join(container_type), cls.__name__)
            setattr(cls, attr, property(getter, setter, None, doc))

        # create the add method
        setattr(cls, add, cls.__make_add(add, attr, container_type))

        # create the create method, only if a single container type is specified
        create = conf_dict.get('create')
        if create is not None:
            if isinstance(container_type, type):
                setattr(cls, create, cls.__make_create(create, add, container_type))
            else:
                msg = ("Cannot specify 'create' key in __clsconf__ for MultiContainerInterface subclass %s "
                       "when 'type' key is not a single type") % cls.__name__
                if multi:
                    msg += " at index %d" % conf_index
                raise ValueError(msg)

        # create the get method
        get = conf_dict.get('get')
        if get is not None:
            setattr(cls, get, cls.__make_get(get, attr, container_type))


class Row(object, metaclass=ExtenderMeta):
    """
    A class for representing rows from a Table.

    The Table class can be indicated with the __table__. Doing so
    will set constructor arguments for the Row class and ensure that
    Row.idx is set appropriately when a Row is added to the Table. It will
    also add functionality to the Table class for getting Row objects.

    Note, the Row class is not needed for working with Table objects. This
    is merely convenience functionality for working with Tables.
    """

    __table__ = None

    @property
    def idx(self):
        """The index of this row in its respective Table"""
        return self.__idx

    @idx.setter
    def idx(self, val):
        if self.__idx is None:
            self.__idx = val
        else:
            raise ValueError("cannot reset the ID of a row object")

    @property
    def table(self):
        """The Table this Row comes from"""
        return self.__table

    @table.setter
    def table(self, val):
        if val is not None:
            self.__table = val
        if self.idx is None:
            self.idx = self.__table.add_row(**self.todict())

    @ExtenderMeta.pre_init
    def __build_row_class(cls, name, bases, classdict):
        table_cls = getattr(cls, '__table__', None)
        if table_cls is not None:
            columns = getattr(table_cls, '__columns__')
            if cls.__init__ == bases[-1].__init__:  # check if __init__ is overridden
                columns = deepcopy(columns)
                func_args = list()
                for col in columns:
                    func_args.append(col)
                func_args.append({'name': 'table', 'type': Table, 'default': None,
                                  'help': 'the table this row is from'})
                func_args.append({'name': 'idx', 'type': int, 'default': None,
                                  'help': 'the index for this row'})

                @docval(*func_args)
                def __init__(self, **kwargs):
                    super(cls, self).__init__()
                    table, idx = popargs('table', 'idx', kwargs)
                    self.__keys = list()
                    self.__idx = None
                    self.__table = None
                    for k, v in kwargs.items():
                        self.__keys.append(k)
                        setattr(self, k, v)
                    self.idx = idx
                    self.table = table

                setattr(cls, '__init__', __init__)

                def todict(self):
                    return {k: getattr(self, k) for k in self.__keys}

                setattr(cls, 'todict', todict)

            # set this so Table.row gets set when a Table is instantiated
            table_cls.__rowclass__ = cls
        else:
            if bases != (object,):
                raise ValueError('__table__ must be set if sub-classing Row')

    def __eq__(self, other):
        return self.idx == other.idx and self.table is other.table

    def __str__(self):
        return "Row(%i, %s) = %s" % (self.idx, self.table.name, str(self.todict()))


class RowGetter:
    """
    A simple class for providing __getitem__ functionality that returns
    Row objects to a Table.
    """

    def __init__(self, table):
        self.table = table
        self.cache = dict()

    def __getitem__(self, idx):
        ret = self.cache.get(idx)
        if ret is None:
            row = self.table[idx]
            ret = self.table.__rowclass__(*row, table=self.table, idx=idx)
            self.cache[idx] = ret
        return ret


class Table(Data):
    r'''
    Subclasses should specify the class attribute \_\_columns\_\_.

    This should be a list of dictionaries with the following keys:

    - ``name``            the column name
    - ``type``            the type of data in this column
    - ``doc``             a brief description of what gets stored in this column

    For reference, this list of dictionaries will be used with docval to autogenerate
    the ``add_row`` method for adding data to this table.

    If \_\_columns\_\_ is not specified, no custom ``add_row`` method will be added.

    The class attribute __defaultname__ can also be set to specify a default name
    for the table class. If \_\_defaultname\_\_ is not specified, then ``name`` will
    need to be specified when the class is instantiated.

    A Table class can be paired with a Row class for conveniently working with rows of
    a Table. This pairing must be indicated in the Row class implementation. See Row
    for more details.
    '''

    # This class attribute is used to indicate which Row class should be used when
    # adding RowGetter functionality to the Table.
    __rowclass__ = None

    @ExtenderMeta.pre_init
    def __build_table_class(cls, name, bases, classdict):
        if hasattr(cls, '__columns__'):
            columns = getattr(cls, '__columns__')

            idx = dict()
            for i, col in enumerate(columns):
                idx[col['name']] = i
            setattr(cls, '__colidx__', idx)

            if cls.__init__ == bases[-1].__init__:  # check if __init__ is overridden
                name = {'name': 'name', 'type': str, 'doc': 'the name of this table'}
                defname = getattr(cls, '__defaultname__', None)
                if defname is not None:
                    name['default'] = defname  # override the name with the default name if present

                @docval(name,
                        {'name': 'data', 'type': ('array_data', 'data'), 'doc': 'the data in this table',
                         'default': list()})
                def __init__(self, **kwargs):
                    name, data = getargs('name', 'data', kwargs)
                    colnames = [i['name'] for i in columns]
                    super(cls, self).__init__(colnames, name, data)

                setattr(cls, '__init__', __init__)

            if cls.add_row == bases[-1].add_row:  # check if add_row is overridden

                @docval(*columns)
                def add_row(self, **kwargs):
                    return super(cls, self).add_row(kwargs)

                setattr(cls, 'add_row', add_row)

    @docval({'name': 'columns', 'type': (list, tuple), 'doc': 'a list of the columns in this table'},
            {'name': 'name', 'type': str, 'doc': 'the name of this container'},
            {'name': 'data', 'type': ('array_data', 'data'), 'doc': 'the source of the data', 'default': list()})
    def __init__(self, **kwargs):
        self.__columns = tuple(popargs('columns', kwargs))
        self.__col_index = {name: idx for idx, name in enumerate(self.__columns)}
        if getattr(self, '__rowclass__') is not None:
            self.row = RowGetter(self)
        super().__init__(**kwargs)

    @property
    def columns(self):
        return self.__columns

    @docval({'name': 'values', 'type': dict, 'doc': 'the values for each column'})
    def add_row(self, **kwargs):
        values = getargs('values', kwargs)
        if not isinstance(self.data, list):
            msg = 'Cannot append row to %s' % type(self.data)
            raise ValueError(msg)
        ret = len(self.data)
        row = [values[col] for col in self.columns]
        row = [v.idx if isinstance(v, Row) else v for v in row]
        self.data.append(tuple(row))
        return ret

    def which(self, **kwargs):
        '''
        Query a table
        '''
        if len(kwargs) != 1:
            raise ValueError("only one column can be queried")
        colname, value = kwargs.popitem()
        idx = self.__colidx__.get(colname)
        if idx is None:
            msg = "no '%s' column in %s" % (colname, self.__class__.__name__)
            raise KeyError(msg)
        ret = list()
        for i in range(len(self.data)):
            row = self.data[i]
            row_val = row[idx]
            if row_val == value:
                ret.append(i)
        return ret

    def __len__(self):
        return len(self.data)

    def __getitem__(self, args):
        idx = args
        col = None
        if isinstance(args, tuple):
            idx = args[1]
            if isinstance(args[0], str):
                col = self.__col_index.get(args[0])
            elif isinstance(args[0], int):
                col = args[0]
            else:
                raise KeyError('first argument must be a column name or index')
            return self.data[idx][col]
        elif isinstance(args, str):
            col = self.__col_index.get(args)
            if col is None:
                raise KeyError(args)
            return [row[col] for row in self.data]
        else:
            return self.data[idx]

    def to_dataframe(self):
        '''Produce a pandas DataFrame containing this table's data.
        '''

        data = {colname: self[colname] for ii, colname in enumerate(self.columns)}
        return pd.DataFrame(data)

    @classmethod
    @docval(
        {'name': 'df', 'type': pd.DataFrame, 'doc': 'input data'},
        {'name': 'name', 'type': str, 'doc': 'the name of this container', 'default': None},
        {
            'name': 'extra_ok',
            'type': bool,
            'doc': 'accept (and ignore) unexpected columns on the input dataframe',
            'default': False
        },
    )
    def from_dataframe(cls, **kwargs):
        '''Construct an instance of Table (or a subclass) from a pandas DataFrame. The columns of the dataframe
        should match the columns defined on the Table subclass.
        '''

        df, name, extra_ok = getargs('df', 'name', 'extra_ok', kwargs)

        cls_cols = list([col['name'] for col in getattr(cls, '__columns__')])
        df_cols = list(df.columns)

        missing_columns = set(cls_cols) - set(df_cols)
        extra_columns = set(df_cols) - set(cls_cols)

        if extra_columns:
            raise ValueError(
                'unrecognized column(s) {} for table class {} (columns {})'.format(
                    extra_columns, cls.__name__, cls_cols
                )
            )

        use_index = False
        if len(missing_columns) == 1 and list(missing_columns)[0] == df.index.name:
            use_index = True

        elif missing_columns:
            raise ValueError(
                'missing column(s) {} for table class {} (columns {}, provided {})'.format(
                    missing_columns, cls.__name__, cls_cols, df_cols
                )
            )

        data = []
        for index, row in df.iterrows():
            if use_index:
                data.append([
                    row[colname] if colname != df.index.name else index
                    for colname in cls_cols
                ])
            else:
                data.append(tuple([row[colname] for colname in cls_cols]))

        if name is None:
            return cls(data=data)
        return cls(name=name, data=data)
