hdmf-dev__hdmf-1270

{
  "status": "ok",
  "stdout": "src/hdmf/utils.py:796:def get_data_shape(data, strict_no_data_load=False):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/utils.py",
  "content": "        it = (getattr(cls, n) for n in dir(cls))\n        it = (a for a in it if hasattr(a, cls.__postinit))\n        for func in it:\n            func(name, bases, classdict)\n\n\ndef get_data_shape(data, strict_no_data_load=False):\n    \"\"\"\n    Helper function used to determine the shape of the given array.\n\n    In order to determine the shape of nested tuples, lists, and sets, this function\n    recursively inspects elements along the dimensions, assuming that the data has a regular,\n    rectangular shape. In the case of out-of-core iterators, this means that the first item\n    along each dimension would potentially be loaded into memory. Set strict_no_data_load=True\n    to enforce that this does not happen, at the cost that we may not be able to determine\n    the shape of the array.\n\n    :param data: Array for which we should determine the shape. Can be any object that supports __len__ or .shape.\n    :type data: List, numpy.ndarray, DataChunkIterator\n    :param strict_no_data_load: If True and data is an out-of-core iterator, None may be returned. If False (default),\n                                the first element of data may be loaded into memory.\n    :return: Tuple of ints indicating the size of known dimensions. Dimensions for which the size is unknown\n             will be set to None.\n    \"\"\"\n\n    def __get_shape_helper(local_data):\n        shape = list()\n        if hasattr(local_data, '__len__'):\n            shape.append(len(local_data))\n            if len(local_data):\n                el = next(iter(local_data))\n                if not isinstance(el, (str, bytes)):\n                    shape.extend(__get_shape_helper(el))\n        return tuple(shape)\n\n    # NOTE: data.maxshape will fail on empty h5py.Dataset without shape or maxshape. this will be fixed in h5py 3.0\n    if hasattr(data, 'maxshape'):\n        return data.maxshape\n    if hasattr(data, 'shape') and data.shape is not None:\n        return data.shape\n    if isinstance(data, dict):",
  "start_line": 790,
  "end_line": 830,
  "total_lines": 1155,
  "is_truncated": false
}

{
  "status": "ok",
  "filepath": "src/hdmf/utils.py",
  "content": "    if isinstance(data, dict):\n        return None\n    if hasattr(data, '__len__') and not isinstance(data, (str, bytes)):\n        if not strict_no_data_load or isinstance(data, (list, tuple, set)):\n            return __get_shape_helper(data)\n    return None\n\n\ndef pystr(s):\n    \"\"\"\n    Convert a string of characters to Python str object\n    \"\"\"\n    if isinstance(s, bytes):\n        return s.decode('utf-8')\n    else:\n        return s\n\n\ndef to_uint_array(arr):\n    \"\"\"\n    Convert a numpy array or array-like object to a numpy array of unsigned integers with the same dtype itemsize.\n\n    For example, a list of int32 values is converted to a numpy array with dtype uint32.\n    :raises ValueError: if input array contains values that are not unsigned integers or non-negative integers.\n    \"\"\"\n    if not isinstance(arr, np.ndarray):\n        arr = np.array(arr)\n    if np.issubdtype(arr.dtype, np.unsignedinteger):\n        return arr\n    if np.issubdtype(arr.dtype, np.integer):\n        if (arr < 0).any():",
  "start_line": 830,
  "end_line": 860,
  "total_lines": 1155,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "src/hdmf/backends/hdf5/h5_utils.py:120:class DatasetOfReferences(H5Dataset, ReferenceResolver, metaclass=ABCMeta):\nsrc/hdmf/data_utils.py:479:class DataChunkIterator(AbstractDataChunkIterator):\nsrc/hdmf/data_utils.py:763:class DataChunk:\nsrc/hdmf/data_utils.py:1005:class DataIO:\nsrc/hdmf/build/builders.py:320:class DatasetBuilder(BaseBuilder):\nsrc/hdmf/common/io/resources.py:12:           The DatasetBuilder for name is associated with data_type Data and container class Data,\nsrc/hdmf/spec/spec.py:659:class DatasetSpec(BaseStorageSpec):\nsrc/hdmf/monitor.py:19:class DataChunkProcessor(AbstractDataChunkIterator, metaclass=ABCMeta):\nsrc/hdmf/validate/validator.py:418:class DatasetValidator(BaseStorageValidator):\nsrc/hdmf/container.py:892:class Data(AbstractContainer):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/container.py",
  "content": "\n\nclass Data(AbstractContainer):\n    \"\"\"\n    A class for representing dataset containers\n    \"\"\"\n    @docval({'name': 'name', 'type': str, 'doc': 'the name of this container'},\n            {'name': 'data', 'type': ('scalar_data', 'array_data', 'data'), 'doc': 'the source of the data'})\n    def __init__(self, **kwargs):\n        data = popargs('data', kwargs)\n        super().__init__(**kwargs)\n\n        self._validate_new_data(data)\n        self.__data = data\n\n    @property\n    def data(self):\n        return self.__data\n\n    @property\n    def shape(self):\n        \"\"\"\n        Get the shape of the data represented by this container\n        :return: Shape tuple\n        :rtype: tuple of ints\n        \"\"\"\n        return get_data_shape(self.__data)\n\n    def set_data_io(\n        self,\n        data_io_class: Type[DataIO],\n        data_io_kwargs: dict,\n        data_chunk_iterator_class: Optional[Type[AbstractDataChunkIterator]] = None,\n        data_chunk_iterator_kwargs: dict = None,\n    ) -> None:\n        \"\"\"\n        Apply DataIO object to the data held by this Data object.\n\n        Parameters\n        ----------\n        data_io_class: Type[DataIO]\n            The DataIO to apply to the data held by this Data.\n        data_io_kwargs: dict\n            The keyword arguments to pass to the DataIO.\n        data_chunk_iterator_class: Type[AbstractDataChunkIterator]\n            The DataChunkIterator to use for the DataIO. If None, no DataChunkIterator is used.\n        data_chunk_iterator_kwargs: dict\n            The keyword arguments to pass to the DataChunkIterator.\n\n        Notes\n        -----\n        If data_chunk_iterator_class is not None, the data is wrapped in the DataChunkIterator before being wrapped in\n        the DataIO. This allows for rewriting the backend configuration of hdf5 datasets.\n        \"\"\"\n        data_chunk_iterator_kwargs = data_chunk_iterator_kwargs or dict()\n        data = self.__data\n        if data_chunk_iterator_class is not None:\n            data = data_chunk_iterator_class(data=data, **data_chunk_iterator_kwargs)\n        self.__data = data_io_class(data=data, **data_io_kwargs)\n\n    @docval({'name': 'func', 'type': types.FunctionType, 'doc': 'a function to transform *data*'})",
  "start_line": 890,
  "end_line": 950,
  "total_lines": 1619,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "src/hdmf/container.py:51:class AbstractContainer(metaclass=ExtenderMeta):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/container.py",
  "content": "           'is not guaranteed to maintain backward compatibility') % pfx\n    return msg\n\n\nclass HERDManager:\n    \"\"\"\n    This class manages whether to set/attach an instance of HERD to the subclass.\n    \"\"\"\n\n    @docval({'name': 'herd', 'type': 'HERD',\n             'doc': 'The external resources to be used for the container.'},)\n    def link_resources(self, **kwargs):\n        \"\"\"\n        Method to attach an instance of HERD in order to auto-add terms/references to data.\n        \"\"\"\n        self._herd = kwargs['herd']\n\n    def get_linked_resources(self):\n        return self._herd if hasattr(self, \"_herd\") else None\n\n\nclass AbstractContainer(metaclass=ExtenderMeta):\n    # The name of the class attribute that subclasses use to autogenerate properties\n    # This parameterization is supplied in case users would like to configure\n    # the class attribute name to something domain-specific\n\n    _experimental = False\n\n    _fieldsname = '__fields__'\n\n    _data_type_attr = 'data_type'\n\n    # Subclasses use this class attribute to add properties to autogenerate\n    # Autogenerated properties will store values in self.__field_values\n    __fields__ = tuple()\n\n    # This field is automatically set by __gather_fields before initialization.\n    # It holds all the values in __fields__ for this class and its parent classes.\n    __fieldsconf = tuple()\n\n    _pconf_allowed_keys = {'name', 'doc', 'settable'}\n\n    # Override the _setter factor function, so directives that apply to\n    # Container do not get used on Data\n    @classmethod\n    def _setter(cls, field):\n        \"\"\"\n        Make a setter function for creating a :py:func:`property`\n        \"\"\"\n        name = field['name']\n        if not field.get('settable', True):\n            return None\n\n        def setter(self, val):\n            if val is None:\n                return\n            if name in self.fields:\n                msg = \"can't set attribute '%s' -- already set\" % name\n                raise AttributeError(msg)\n            self.fields[name] = self._field_config(arg_name=name,\n                                                   val=val,\n                                                   type_map=self._get_type_map())\n        return setter\n\n    def _get_type_map(self):\n        from hdmf.common import get_type_map # circular import\n        return get_type_map()\n\n    @property\n    def data_type(self):\n        \"\"\"",
  "start_line": 30,
  "end_line": 100,
  "total_lines": 1619,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "970:    def __len__(self):\n1535:    def __len__(self):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/container.py",
  "content": "        return self\n\n    def __bool__(self):\n        if self.data is not None:\n            if isinstance(self.data, (np.ndarray, tuple, list)):\n                return len(self.data) != 0\n            if self.data:\n                return True\n        return False\n\n    def __len__(self):\n        return len(self.__data)\n\n    def __getitem__(self, args):\n        return self.get(args)\n\n    def get(self, args):\n        if isinstance(self.data, (tuple, list)) and isinstance(args, (tuple, list, np.ndarray)):\n            return [self.data[i] for i in args]\n        if isinstance(self.data, h5py.Dataset) and isinstance(args, np.ndarray):\n            # This is needed for h5py 2.9 compatibility",
  "start_line": 960,
  "end_line": 980,
  "total_lines": 1619,
  "is_truncated": false
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "",
  "details": {
    "stdout": "",
    "stderr": "",
    "exit_code": 1
  }
}

{
  "status": "ok",
  "filepath": "src/hdmf/container.py",
  "content": "            # This is needed for h5py 2.9 compatibility\n            args = args.tolist()\n        return self.data[args]\n\n    def append(self, arg):\n        self._validate_new_data_element(arg)\n        self.__data = append_data(self.__data, arg)\n\n    def extend(self, arg):\n        \"\"\"\n        The extend_data method adds all the elements of the iterable arg to the\n        end of the data of this Data container.\n\n        :param arg: The iterable to add to the end of this VectorData\n        \"\"\"\n        self._validate_new_data(arg)\n        self.__data = extend_data(self.__data, arg)\n\n    def _validate_new_data(self, data):\n        \"\"\"Function to validate a new array that will be set or added to data. Raises an error if the data is invalid.\n",
  "start_line": 980,
  "end_line": 1000,
  "total_lines": 1619,
  "is_truncated": false
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "Traceback (most recent call last):\n  File \"<stdin>\", line 11, in <module>\nAssertionError: Expected (2,), got (2, 2)\n",
  "details": {
    "stdout": "Shape: (2, 2)\n",
    "stderr": "Traceback (most recent call last):\n  File \"<stdin>\", line 11, in <module>\nAssertionError: Expected (2,), got (2, 2)\n",
    "exit_code": 1
  }
}

{
  "status": "ok",
  "filepath": "src/hdmf/utils.py",
  "content": "import collections\nimport copy as _copy\nimport re\nimport types\nimport warnings\nfrom abc import ABCMeta\nfrom enum import Enum\n\nimport h5py\nimport numpy as np\n\n\n__macros = {\n    'array_data': [np.ndarray, list, tuple, h5py.Dataset],\n    'scalar_data': [str, int, float, bytes, bool],\n    'data': []\n}\n\ntry:\n    # optionally accept zarr.Array as array data to support conversion of data from Zarr to HDMF\n    import zarr\n    __macros['array_data'].append(zarr.Array)\nexcept ImportError:\n    pass\n\n\n# code to signify how to handle positional arguments in docval\nAllowPositional = Enum('AllowPositional', 'ALLOWED WARNING ERROR')\n\n__supported_bool_types = (bool, np.bool_)\n__supported_uint_types = (np.uint8, np.uint16, np.uint32, np.uint64)\n__supported_int_types = (int, np.int8, np.int16, np.int32, np.int64)\n__supported_float_types = [float, np.float16, np.float32, np.float64]\nif hasattr(np, \"float128\"):  # pragma: no cover\n    __supported_float_types.append(np.float128)\nif hasattr(np, \"longdouble\"):  # pragma: no cover\n    # on windows python<=3.5, h5py floats resolve float64s as either np.float64 or np.longdouble\n    # non-deterministically. a future version of h5py will fix this. see #112\n    __supported_float_types.append(np.longdouble)\n__supported_float_types = tuple(__supported_float_types)\n__allowed_enum_types = (__supported_bool_types + __supported_uint_types + __supported_int_types\n                        + __supported_float_types + (str,))\n\n\ndef docval_macro(macro):\n    \"\"\"Class decorator to add the class to a list of types associated with the key macro in the __macros dict\n    \"\"\"\n\n    def _dec(cls):\n        if macro not in __macros:",
  "start_line": 1,
  "end_line": 50,
  "total_lines": 1155,
  "is_truncated": false
}

{
  "status": "ok",
  "filepath": "src/hdmf/container.py",
  "content": "import types\nfrom collections import OrderedDict\nfrom copy import deepcopy\nfrom typing import Type, Optional\nfrom uuid import uuid4\nfrom warnings import warn\nimport os\n\nimport h5py\nimport numpy as np\nimport pandas as pd\n\nfrom .data_utils import DataIO, append_data, extend_data, AbstractDataChunkIterator\nfrom .utils import (docval, get_docval, getargs, ExtenderMeta, get_data_shape, popargs, LabelledDict,\n                    get_basic_array_info, generate_array_html_repr)\n\nfrom .term_set import TermSet, TermSetWrapper\n\ndef _set_exp(cls):\n    \"\"\"Set a class as being experimental\"\"\"\n    cls._experimental = True\n\n\ndef _exp_warn_msg(cls):\n    \"\"\"Generate a warning message experimental features\"\"\"\n    pfx = cls\n    if isinstance(cls, type):\n        pfx = cls.__name__\n    msg = ('%s is experimental -- it may be removed in the future and '\n           'is not guaranteed to maintain backward compatibility') % pfx\n    return msg\n\n\nclass HERDManager:\n    \"\"\"\n    This class manages whether to set/attach an instance of HERD to the subclass.\n    \"\"\"\n\n    @docval({'name': 'herd', 'type': 'HERD',\n             'doc': 'The external resources to be used for the container.'},)\n    def link_resources(self, **kwargs):\n        \"\"\"\n        Method to attach an instance of HERD in order to auto-add terms/references to data.\n        \"\"\"\n        self._herd = kwargs['herd']\n\n    def get_linked_resources(self):\n        return self._herd if hasattr(self, \"_herd\") else None\n\n",
  "start_line": 1,
  "end_line": 50,
  "total_lines": 1619,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "Shape: (2,)\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "src/hdmf/backends/hdf5/h5_utils.py:22:from ...utils import docval, getargs, popargs, get_docval, get_data_shape\nsrc/hdmf/backends/hdf5/h5_utils.py:615:            return get_data_shape(self.data)\nsrc/hdmf/backends/hdf5/h5tools.py:22:from ...utils import (docval, getargs, popargs, get_data_shape, get_docval, StrDataset,\nsrc/hdmf/backends/hdf5/h5tools.py:1393:            data_shape = get_data_shape(data)\nsrc/hdmf/utils.py:127:    valshape = get_data_shape(value)\nsrc/hdmf/utils.py:295:                    valshape = get_data_shape(argval)\nsrc/hdmf/utils.py:306:                        valshape = get_data_shape(argval)\nsrc/hdmf/utils.py:352:                valshape = get_data_shape(argval)\nsrc/hdmf/utils.py:363:                    valshape = get_data_shape(argval)\nsrc/hdmf/utils.py:796:def get_data_shape(data, strict_no_data_load=False):\nsrc/hdmf/data_utils.py:18:from .utils import docval, getargs, popargs, docval_macro, get_data_shape\nsrc/hdmf/data_utils.py:552:            # Don't just call get_data_shape for a generator as that would potentially trigger loading of all the data\nsrc/hdmf/data_utils.py:554:                self.__maxshape = get_data_shape(self.data, strict_no_data_load=True)\nsrc/hdmf/data_utils.py:563:            self.__first_chunk_shape = get_data_shape(self.__next_chunk.data)\nsrc/hdmf/data_utils.py:629:                piece_shape = list(get_data_shape(iter_pieces[0]))\nsrc/hdmf/data_utils.py:736:            data_shape = get_data_shape(self.__next_chunk.data)\nsrc/hdmf/data_utils.py:867:    response.shape1 = get_data_shape(data1)\nsrc/hdmf/data_utils.py:868:    response.shape2 = get_data_shape(data2)\nsrc/hdmf/build/objectmapper.py:24:from ..utils import docval, getargs, ExtenderMeta, get_docval, get_data_shape, StrDataset\nsrc/hdmf/build/objectmapper.py:841:        data_shape = get_data_shape(data)\nsrc/hdmf/common/sparse.py:11:from ..utils import docval, popargs, to_uint_array,  get_data_shape, AllowPositional\nsrc/hdmf/common/sparse.py:33:            temp_shape = get_data_shape(data)\nsrc/hdmf/validate/validator.py:16:from ..utils import docval, getargs, pystr, get_data_shape\nsrc/hdmf/validate/validator.py:378:            shape = get_data_shape(value)\nsrc/hdmf/validate/validator.py:450:            shape = get_data_shape(data)\nsrc/hdmf/container.py:14:from .utils import (docval, get_docval, getargs, ExtenderMeta, get_data_shape, popargs, LabelledDict,\nsrc/hdmf/container.py:916:        return get_data_shape(self.__data)\n",
  "stderr": "",
  "exit_code": 0
}