hdmf-dev__hdmf-1270

{
  "status": "ok",
  "stdout": "src/hdmf/utils.py:796:def get_data_shape(data, strict_no_data_load=False):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/utils.py",
  "content": "def get_data_shape(data, strict_no_data_load=False):\n    \"\"\"\n    Helper function used to determine the shape of the given array.\n\n    In order to determine the shape of nested tuples, lists, and sets, this function\n    recursively inspects elements along the dimensions, assuming that the data has a regular,\n    rectangular shape. In the case of out-of-core iterators, this means that the first item\n    along each dimension would potentially be loaded into memory. Set strict_no_data_load=True\n    to enforce that this does not happen, at the cost that we may not be able to determine\n    the shape of the array.\n\n    :param data: Array for which we should determine the shape. Can be any object that supports __len__ or .shape.\n    :type data: List, numpy.ndarray, DataChunkIterator\n    :param strict_no_data_load: If True and data is an out-of-core iterator, None may be returned. If False (default),\n                                the first element of data may be loaded into memory.\n    :return: Tuple of ints indicating the size of known dimensions. Dimensions for which the size is unknown\n             will be set to None.\n    \"\"\"\n\n    def __get_shape_helper(local_data):\n        shape = list()\n        if hasattr(local_data, '__len__'):\n            shape.append(len(local_data))\n            if len(local_data):\n                el = next(iter(local_data))\n                if not isinstance(el, (str, bytes)):\n                    shape.extend(__get_shape_helper(el))\n        return tuple(shape)\n\n    # NOTE: data.maxshape will fail on empty h5py.Dataset without shape or maxshape. this will be fixed in h5py 3.0\n    if hasattr(data, 'maxshape'):\n        return data.maxshape\n    if hasattr(data, 'shape') and data.shape is not None:\n        return data.shape\n    if isinstance(data, dict):\n        return None\n    if hasattr(data, '__len__') and not isinstance(data, (str, bytes)):\n        if not strict_no_data_load or isinstance(data, (list, tuple, set)):\n            return __get_shape_helper(data)\n    return None\n\n\ndef pystr(s):\n    \"\"\"\n    Convert a string of characters to Python str object\n    \"\"\"\n    if isinstance(s, bytes):\n        return s.decode('utf-8')\n    else:\n        return s\n\n\ndef to_uint_array(arr):\n    \"\"\"\n    Convert a numpy array or array-like object to a numpy array of unsigned integers with the same dtype itemsize.",
  "start_line": 796,
  "end_line": 850,
  "total_lines": 1155,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "src/hdmf/backends/hdf5/h5_utils.py:120:class DatasetOfReferences(H5Dataset, ReferenceResolver, metaclass=ABCMeta):\nsrc/hdmf/data_utils.py:479:class DataChunkIterator(AbstractDataChunkIterator):\nsrc/hdmf/data_utils.py:763:class DataChunk:\nsrc/hdmf/data_utils.py:1005:class DataIO:\nsrc/hdmf/build/builders.py:320:class DatasetBuilder(BaseBuilder):\nsrc/hdmf/common/io/resources.py:12:           The DatasetBuilder for name is associated with data_type Data and container class Data,\nsrc/hdmf/spec/spec.py:659:class DatasetSpec(BaseStorageSpec):\nsrc/hdmf/monitor.py:19:class DataChunkProcessor(AbstractDataChunkIterator, metaclass=ABCMeta):\nsrc/hdmf/validate/validator.py:418:class DatasetValidator(BaseStorageValidator):\nsrc/hdmf/container.py:892:class Data(AbstractContainer):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/container.py",
  "content": "class Data(AbstractContainer):\n    \"\"\"\n    A class for representing dataset containers\n    \"\"\"\n    @docval({'name': 'name', 'type': str, 'doc': 'the name of this container'},\n            {'name': 'data', 'type': ('scalar_data', 'array_data', 'data'), 'doc': 'the source of the data'})\n    def __init__(self, **kwargs):\n        data = popargs('data', kwargs)\n        super().__init__(**kwargs)\n\n        self._validate_new_data(data)\n        self.__data = data\n\n    @property\n    def data(self):\n        return self.__data\n\n    @property\n    def shape(self):\n        \"\"\"\n        Get the shape of the data represented by this container\n        :return: Shape tuple\n        :rtype: tuple of ints\n        \"\"\"\n        return get_data_shape(self.__data)\n\n    def set_data_io(\n        self,\n        data_io_class: Type[DataIO],\n        data_io_kwargs: dict,\n        data_chunk_iterator_class: Optional[Type[AbstractDataChunkIterator]] = None,\n        data_chunk_iterator_kwargs: dict = None,\n    ) -> None:\n        \"\"\"\n        Apply DataIO object to the data held by this Data object.\n\n        Parameters\n        ----------\n        data_io_class: Type[DataIO]\n            The DataIO to apply to the data held by this Data.\n        data_io_kwargs: dict\n            The keyword arguments to pass to the DataIO.\n        data_chunk_iterator_class: Type[AbstractDataChunkIterator]\n            The DataChunkIterator to use for the DataIO. If None, no DataChunkIterator is used.\n        data_chunk_iterator_kwargs: dict\n            The keyword arguments to pass to the DataChunkIterator.\n\n        Notes\n        -----\n        If data_chunk_iterator_class is not None, the data is wrapped in the DataChunkIterator before being wrapped in\n        the DataIO. This allows for rewriting the backend configuration of hdf5 datasets.\n        \"\"\"\n        data_chunk_iterator_kwargs = data_chunk_iterator_kwargs or dict()\n        data = self.__data\n        if data_chunk_iterator_class is not None:\n            data = data_chunk_iterator_class(data=data, **data_chunk_iterator_kwargs)\n        self.__data = data_io_class(data=data, **data_io_kwargs)\n\n    @docval({'name': 'func', 'type': types.FunctionType, 'doc': 'a function to transform *data*'})",
  "start_line": 892,
  "end_line": 950,
  "total_lines": 1619,
  "is_truncated": false
}

{
  "status": "ok",
  "filepath": "src/hdmf/container.py",
  "content": "import types\nfrom collections import OrderedDict\nfrom copy import deepcopy\nfrom typing import Type, Optional\nfrom uuid import uuid4\nfrom warnings import warn\nimport os\n\nimport h5py\nimport numpy as np\nimport pandas as pd\n\nfrom .data_utils import DataIO, append_data, extend_data, AbstractDataChunkIterator\nfrom .utils import (docval, get_docval, getargs, ExtenderMeta, get_data_shape, popargs, LabelledDict,\n                    get_basic_array_info, generate_array_html_repr)\n\nfrom .term_set import TermSet, TermSetWrapper\n\ndef _set_exp(cls):\n    \"\"\"Set a class as being experimental\"\"\"\n    cls._experimental = True\n\n\ndef _exp_warn_msg(cls):\n    \"\"\"Generate a warning message experimental features\"\"\"\n    pfx = cls\n    if isinstance(cls, type):\n        pfx = cls.__name__\n    msg = ('%s is experimental -- it may be removed in the future and '\n           'is not guaranteed to maintain backward compatibility') % pfx\n    return msg\n\n\nclass HERDManager:\n    \"\"\"\n    This class manages whether to set/attach an instance of HERD to the subclass.\n    \"\"\"\n\n    @docval({'name': 'herd', 'type': 'HERD',\n             'doc': 'The external resources to be used for the container.'},)\n    def link_resources(self, **kwargs):\n        \"\"\"\n        Method to attach an instance of HERD in order to auto-add terms/references to data.\n        \"\"\"\n        self._herd = kwargs['herd']\n\n    def get_linked_resources(self):\n        return self._herd if hasattr(self, \"_herd\") else None\n\n\nclass AbstractContainer(metaclass=ExtenderMeta):\n    # The name of the class attribute that subclasses use to autogenerate properties\n    # This parameterization is supplied in case users would like to configure\n    # the class attribute name to something domain-specific\n\n    _experimental = False\n\n    _fieldsname = '__fields__'\n\n    _data_type_attr = 'data_type'\n\n    # Subclasses use this class attribute to add properties to autogenerate\n    # Autogenerated properties will store values in self.__field_values\n    __fields__ = tuple()\n\n    # This field is automatically set by __gather_fields before initialization.\n    # It holds all the values in __fields__ for this class and its parent classes.\n    __fieldsconf = tuple()\n\n    _pconf_allowed_keys = {'name', 'doc', 'settable'}\n\n    # Override the _setter factor function, so directives that apply to\n    # Container do not get used on Data\n    @classmethod\n    def _setter(cls, field):\n        \"\"\"\n        Make a setter function for creating a :py:func:`property`\n        \"\"\"\n        name = field['name']\n        if not field.get('settable', True):\n            return None\n\n        def setter(self, val):\n            if val is None:\n                return\n            if name in self.fields:\n                msg = \"can't set attribute '%s' -- already set\" % name\n                raise AttributeError(msg)\n            self.fields[name] = self._field_config(arg_name=name,\n                                                   val=val,\n                                                   type_map=self._get_type_map())\n        return setter\n\n    def _get_type_map(self):\n        from hdmf.common import get_type_map # circular import\n        return get_type_map()\n\n    @property\n    def data_type(self):\n        \"\"\"\n        Return the spec data type associated with this container.\n        \"\"\"\n        return getattr(self, self._data_type_attr)\n\n    def _field_config(self, arg_name, val, type_map):\n        \"\"\"\n        This method will be called in the setter. The termset configuration will be used (if loaded)\n        to check for a defined TermSet associated with the field. If found, the value of the field\n        will be wrapped with a TermSetWrapper.\n\n        Even though the path field in the configurator can be a list of paths, the config\n        itself is only one file. When a user loads custom configs, the config is appended/modified.\n        The modifications are not written to file, avoiding permanent modifications.\n        \"\"\"\n        configurator = type_map.type_config\n\n        if len(configurator.path)>0:\n            # The type_map has a config always set; however, when toggled off, the config path is empty.\n            CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0]))\n            termset_config = configurator.config\n        else:\n            return val\n\n        # If the val has been manually wrapped then skip checking the config for the attr\n        if isinstance(val, TermSetWrapper):\n            msg = \"Field value already wrapped with TermSetWrapper.\"\n            warn(msg)\n            return val\n\n        # check to see that the namespace for the container is in the config\n        if self.namespace not in termset_config['namespaces']:\n            msg = \"%s not found within loaded configuration.\" % self.namespace\n            warn(msg)\n            return val\n        else:\n            # check to see that the container type is in the config under the namespace\n            config_namespace = termset_config['namespaces'][self.namespace]\n            data_type = self.data_type\n\n            if data_type not in config_namespace['data_types']:\n                msg = '%s not found within the configuration for %s' % (data_type, self.namespace)\n                warn(msg)\n                return val\n            else:\n                # Get the ObjectMapper\n                obj_mapper = type_map.get_map(self)\n\n                # Get the spec for the constructor arg\n                spec = obj_mapper.get_carg_spec(arg_name)\n                if spec is None:",
  "start_line": 1,
  "end_line": 150,
  "total_lines": 1619,
  "is_truncated": true
}

{
  "status": "ok",
  "stdout": "970:    def __len__(self):\n1535:    def __len__(self):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/container.py",
  "content": "        return self\n\n    def __bool__(self):\n        if self.data is not None:\n            if isinstance(self.data, (np.ndarray, tuple, list)):\n                return len(self.data) != 0\n            if self.data:\n                return True\n        return False\n\n    def __len__(self):\n        return len(self.__data)\n\n    def __getitem__(self, args):\n        return self.get(args)\n\n    def get(self, args):\n        if isinstance(self.data, (tuple, list)) and isinstance(args, (tuple, list, np.ndarray)):\n            return [self.data[i] for i in args]\n        if isinstance(self.data, h5py.Dataset) and isinstance(args, np.ndarray):\n            # This is needed for h5py 2.9 compatibility",
  "start_line": 960,
  "end_line": 980,
  "total_lines": 1619,
  "is_truncated": false
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "",
  "details": {
    "stdout": "",
    "stderr": "",
    "exit_code": 1
  }
}

{
  "status": "ok",
  "stdout": "Shape of list: (2, 2)\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/utils.py",
  "content": "import collections\nimport copy as _copy\nimport re\nimport types\nimport warnings\nfrom abc import ABCMeta\nfrom enum import Enum\n\nimport h5py\nimport numpy as np\n\n\n__macros = {\n    'array_data': [np.ndarray, list, tuple, h5py.Dataset],\n    'scalar_data': [str, int, float, bytes, bool],\n    'data': []\n}\n\ntry:\n    # optionally accept zarr.Array as array data to support conversion of data from Zarr to HDMF\n    import zarr\n    __macros['array_data'].append(zarr.Array)\nexcept ImportError:\n    pass\n\n\n# code to signify how to handle positional arguments in docval\nAllowPositional = Enum('AllowPositional', 'ALLOWED WARNING ERROR')\n\n__supported_bool_types = (bool, np.bool_)\n__supported_uint_types = (np.uint8, np.uint16, np.uint32, np.uint64)\n__supported_int_types = (int, np.int8, np.int16, np.int32, np.int64)\n__supported_float_types = [float, np.float16, np.float32, np.float64]\nif hasattr(np, \"float128\"):  # pragma: no cover\n    __supported_float_types.append(np.float128)\nif hasattr(np, \"longdouble\"):  # pragma: no cover\n    # on windows python<=3.5, h5py floats resolve float64s as either np.float64 or np.longdouble\n    # non-deterministically. a future version of h5py will fix this. see #112\n    __supported_float_types.append(np.longdouble)\n__supported_float_types = tuple(__supported_float_types)\n__allowed_enum_types = (__supported_bool_types + __supported_uint_types + __supported_int_types\n                        + __supported_float_types + (str,))\n\n\ndef docval_macro(macro):\n    \"\"\"Class decorator to add the class to a list of types associated with the key macro in the __macros dict\n    \"\"\"\n\n    def _dec(cls):\n        if macro not in __macros:\n            __macros[macro] = list()\n        __macros[macro].append(cls)\n        return cls\n\n    return _dec\n\n\ndef get_docval_macro(key=None):\n    \"\"\"\n    Return a deepcopy of the docval macros, i.e., strings that represent a customizable list of types for use in docval.\n\n    :param key: Name of the macro. If key=None, then a dictionary of all macros is returned. Otherwise, a tuple of\n                the types associated with the key is returned.\n    \"\"\"\n    if key is None:\n        return _copy.deepcopy(__macros)\n    else:\n        return tuple(__macros[key])\n\n\ndef check_type(value, argtype, allow_none=False):\n    \"\"\"Check a value against a type\n\n       The difference between this function and :py:func:`isinstance` is that\n       it allows specifying a type as a string. Furthermore, strings allow for specifying more general\n       types, such as a simple numeric type (i.e. ``argtype=\"num\"``).\n\n       Args:\n           value (Any): the value to check\n           argtype (type, str): the type to check for\n           allow_none (bool): whether or not to allow None as a valid value\n\n\n       Returns:\n           bool: True if value is a valid instance of argtype\n    \"\"\"\n    if value is None:\n        return allow_none\n    if isinstance(argtype, str):\n        if argtype in __macros:\n            return check_type(value, __macros[argtype], allow_none=allow_none)\n        elif argtype == 'uint':\n            return __is_uint(value)\n        elif argtype == 'int':\n            return __is_int(value)\n        elif argtype == 'float':\n            return __is_float(value)\n        elif argtype == 'bool':\n            return __is_bool(value)\n        cls_names = []",
  "start_line": 1,
  "end_line": 100,
  "total_lines": 1155,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "Has _fieldsname: True\nValue of _fieldsname: __fields__\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "src/hdmf/build/classgenerator.py:218:        :param classdict: The dict to update with __fields__ (or a different parent_cls._fieldsname).\nsrc/hdmf/build/classgenerator.py:240:        classdict.setdefault(parent_cls._fieldsname, list()).append(fields_conf)\nsrc/hdmf/build/classgenerator.py:283:        :param classdict: The class dictionary to convert with '__fields__' key (or a different bases[0]._fieldsname)\nsrc/hdmf/build/classgenerator.py:290:            fields = classdict.get(b._fieldsname)\nsrc/hdmf/build/classgenerator.py:292:                classdict[b._fieldsname] = tuple(fields)\nsrc/hdmf/container.py:58:    _fieldsname = '__fields__'\nsrc/hdmf/container.py:202:                       % (k, cls._fieldsname, field_conf['name'], cls.__name__))\nsrc/hdmf/container.py:207:        return getattr(cls, cls._fieldsname)\nsrc/hdmf/container.py:211:        return setattr(cls, cls._fieldsname, value)\nsrc/hdmf/container.py:225:            msg = \"'%s' must be of type tuple\" % cls._fieldsname\nsrc/hdmf/testing/testcase.py:73:        for field in getattr(container1, type1._fieldsname):\n",
  "stderr": "",
  "exit_code": 0
}