hdmf-dev__hdmf-1270

{
  "status": "ok",
  "stdout": "src/hdmf/utils.py:796:def get_data_shape(data, strict_no_data_load=False):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "src/hdmf/backends/hdf5/h5_utils.py:120:class DatasetOfReferences(H5Dataset, ReferenceResolver, metaclass=ABCMeta):\nsrc/hdmf/data_utils.py:479:class DataChunkIterator(AbstractDataChunkIterator):\nsrc/hdmf/data_utils.py:763:class DataChunk:\nsrc/hdmf/data_utils.py:1005:class DataIO:\nsrc/hdmf/build/builders.py:320:class DatasetBuilder(BaseBuilder):\nsrc/hdmf/common/io/resources.py:12:           The DatasetBuilder for name is associated with data_type Data and container class Data,\nsrc/hdmf/spec/spec.py:659:class DatasetSpec(BaseStorageSpec):\nsrc/hdmf/monitor.py:19:class DataChunkProcessor(AbstractDataChunkIterator, metaclass=ABCMeta):\nsrc/hdmf/validate/validator.py:418:class DatasetValidator(BaseStorageValidator):\nsrc/hdmf/container.py:892:class Data(AbstractContainer):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/utils.py",
  "content": "        it = (getattr(cls, n) for n in dir(cls))\n        it = (a for a in it if hasattr(a, cls.__postinit))\n        for func in it:\n            func(name, bases, classdict)\n\n\ndef get_data_shape(data, strict_no_data_load=False):\n    \"\"\"\n    Helper function used to determine the shape of the given array.\n\n    In order to determine the shape of nested tuples, lists, and sets, this function\n    recursively inspects elements along the dimensions, assuming that the data has a regular,\n    rectangular shape. In the case of out-of-core iterators, this means that the first item\n    along each dimension would potentially be loaded into memory. Set strict_no_data_load=True\n    to enforce that this does not happen, at the cost that we may not be able to determine\n    the shape of the array.\n\n    :param data: Array for which we should determine the shape. Can be any object that supports __len__ or .shape.\n    :type data: List, numpy.ndarray, DataChunkIterator\n    :param strict_no_data_load: If True and data is an out-of-core iterator, None may be returned. If False (default),\n                                the first element of data may be loaded into memory.\n    :return: Tuple of ints indicating the size of known dimensions. Dimensions for which the size is unknown\n             will be set to None.\n    \"\"\"\n\n    def __get_shape_helper(local_data):\n        shape = list()\n        if hasattr(local_data, '__len__'):\n            shape.append(len(local_data))\n            if len(local_data):\n                el = next(iter(local_data))\n                if not isinstance(el, (str, bytes)):\n                    shape.extend(__get_shape_helper(el))\n        return tuple(shape)\n\n    # NOTE: data.maxshape will fail on empty h5py.Dataset without shape or maxshape. this will be fixed in h5py 3.0\n    if hasattr(data, 'maxshape'):\n        return data.maxshape\n    if hasattr(data, 'shape') and data.shape is not None:\n        return data.shape\n    if isinstance(data, dict):\n        return None\n    if hasattr(data, '__len__') and not isinstance(data, (str, bytes)):\n        if not strict_no_data_load or isinstance(data, (list, tuple, set)):\n            return __get_shape_helper(data)\n    return None\n\n\ndef pystr(s):\n    \"\"\"\n    Convert a string of characters to Python str object\n    \"\"\"\n    if isinstance(s, bytes):\n        return s.decode('utf-8')\n    else:\n        return s\n\n\ndef to_uint_array(arr):\n    \"\"\"\n    Convert a numpy array or array-like object to a numpy array of unsigned integers with the same dtype itemsize.",
  "start_line": 790,
  "end_line": 850,
  "total_lines": 1155,
  "is_truncated": false
}

{
  "status": "ok",
  "filepath": "src/hdmf/container.py",
  "content": "\n\nclass Data(AbstractContainer):\n    \"\"\"\n    A class for representing dataset containers\n    \"\"\"\n    @docval({'name': 'name', 'type': str, 'doc': 'the name of this container'},\n            {'name': 'data', 'type': ('scalar_data', 'array_data', 'data'), 'doc': 'the source of the data'})\n    def __init__(self, **kwargs):\n        data = popargs('data', kwargs)\n        super().__init__(**kwargs)\n\n        self._validate_new_data(data)\n        self.__data = data\n\n    @property\n    def data(self):\n        return self.__data\n\n    @property\n    def shape(self):\n        \"\"\"\n        Get the shape of the data represented by this container\n        :return: Shape tuple\n        :rtype: tuple of ints\n        \"\"\"\n        return get_data_shape(self.__data)\n\n    def set_data_io(\n        self,\n        data_io_class: Type[DataIO],\n        data_io_kwargs: dict,\n        data_chunk_iterator_class: Optional[Type[AbstractDataChunkIterator]] = None,\n        data_chunk_iterator_kwargs: dict = None,\n    ) -> None:\n        \"\"\"\n        Apply DataIO object to the data held by this Data object.\n\n        Parameters\n        ----------\n        data_io_class: Type[DataIO]\n            The DataIO to apply to the data held by this Data.\n        data_io_kwargs: dict\n            The keyword arguments to pass to the DataIO.\n        data_chunk_iterator_class: Type[AbstractDataChunkIterator]\n            The DataChunkIterator to use for the DataIO. If None, no DataChunkIterator is used.\n        data_chunk_iterator_kwargs: dict\n            The keyword arguments to pass to the DataChunkIterator.\n\n        Notes\n        -----\n        If data_chunk_iterator_class is not None, the data is wrapped in the DataChunkIterator before being wrapped in\n        the DataIO. This allows for rewriting the backend configuration of hdf5 datasets.\n        \"\"\"\n        data_chunk_iterator_kwargs = data_chunk_iterator_kwargs or dict()\n        data = self.__data\n        if data_chunk_iterator_class is not None:\n            data = data_chunk_iterator_class(data=data, **data_chunk_iterator_kwargs)\n        self.__data = data_io_class(data=data, **data_io_kwargs)\n\n    @docval({'name': 'func', 'type': types.FunctionType, 'doc': 'a function to transform *data*'})",
  "start_line": 890,
  "end_line": 950,
  "total_lines": 1619,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "src/hdmf/container.py:51:class AbstractContainer(metaclass=ExtenderMeta):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/container.py",
  "content": "             'doc': 'The external resources to be used for the container.'},)\n    def link_resources(self, **kwargs):\n        \"\"\"\n        Method to attach an instance of HERD in order to auto-add terms/references to data.\n        \"\"\"\n        self._herd = kwargs['herd']\n\n    def get_linked_resources(self):\n        return self._herd if hasattr(self, \"_herd\") else None\n\n\nclass AbstractContainer(metaclass=ExtenderMeta):\n    # The name of the class attribute that subclasses use to autogenerate properties\n    # This parameterization is supplied in case users would like to configure\n    # the class attribute name to something domain-specific\n\n    _experimental = False\n\n    _fieldsname = '__fields__'\n\n    _data_type_attr = 'data_type'\n\n    # Subclasses use this class attribute to add properties to autogenerate\n    # Autogenerated properties will store values in self.__field_values\n    __fields__ = tuple()\n\n    # This field is automatically set by __gather_fields before initialization.\n    # It holds all the values in __fields__ for this class and its parent classes.\n    __fieldsconf = tuple()\n\n    _pconf_allowed_keys = {'name', 'doc', 'settable'}\n\n    # Override the _setter factor function, so directives that apply to\n    # Container do not get used on Data\n    @classmethod\n    def _setter(cls, field):\n        \"\"\"\n        Make a setter function for creating a :py:func:`property`\n        \"\"\"\n        name = field['name']\n        if not field.get('settable', True):\n            return None\n\n        def setter(self, val):\n            if val is None:\n                return\n            if name in self.fields:\n                msg = \"can't set attribute '%s' -- already set\" % name\n                raise AttributeError(msg)\n            self.fields[name] = self._field_config(arg_name=name,\n                                                   val=val,\n                                                   type_map=self._get_type_map())\n        return setter\n\n    def _get_type_map(self):\n        from hdmf.common import get_type_map # circular import\n        return get_type_map()\n\n    @property\n    def data_type(self):\n        \"\"\"\n        Return the spec data type associated with this container.\n        \"\"\"\n        return getattr(self, self._data_type_attr)\n\n    def _field_config(self, arg_name, val, type_map):\n        \"\"\"\n        This method will be called in the setter. The termset configuration will be used (if loaded)\n        to check for a defined TermSet associated with the field. If found, the value of the field\n        will be wrapped with a TermSetWrapper.\n\n        Even though the path field in the configurator can be a list of paths, the config\n        itself is only one file. When a user loads custom configs, the config is appended/modified.\n        The modifications are not written to file, avoiding permanent modifications.\n        \"\"\"\n        configurator = type_map.type_config\n\n        if len(configurator.path)>0:\n            # The type_map has a config always set; however, when toggled off, the config path is empty.\n            CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0]))\n            termset_config = configurator.config\n        else:\n            return val\n\n        # If the val has been manually wrapped then skip checking the config for the attr\n        if isinstance(val, TermSetWrapper):\n            msg = \"Field value already wrapped with TermSetWrapper.\"\n            warn(msg)\n            return val\n\n        # check to see that the namespace for the container is in the config\n        if self.namespace not in termset_config['namespaces']:\n            msg = \"%s not found within loaded configuration.\" % self.namespace\n            warn(msg)\n            return val\n        else:\n            # check to see that the container type is in the config under the namespace\n            config_namespace = termset_config['namespaces'][self.namespace]\n            data_type = self.data_type\n\n            if data_type not in config_namespace['data_types']:\n                msg = '%s not found within the configuration for %s' % (data_type, self.namespace)\n                warn(msg)\n                return val\n            else:\n                # Get the ObjectMapper\n                obj_mapper = type_map.get_map(self)\n\n                # Get the spec for the constructor arg\n                spec = obj_mapper.get_carg_spec(arg_name)\n                if spec is None:",
  "start_line": 40,
  "end_line": 150,
  "total_lines": 1619,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "617:            if hasattr(v, '__len__'):\n970:    def __len__(self):\n1535:    def __len__(self):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/container.py",
  "content": "        return self\n\n    def __bool__(self):\n        if self.data is not None:\n            if isinstance(self.data, (np.ndarray, tuple, list)):\n                return len(self.data) != 0\n            if self.data:\n                return True\n        return False\n\n    def __len__(self):\n        return len(self.__data)\n\n    def __getitem__(self, args):\n        return self.get(args)\n\n    def get(self, args):\n        if isinstance(self.data, (tuple, list)) and isinstance(args, (tuple, list, np.ndarray)):\n            return [self.data[i] for i in args]\n        if isinstance(self.data, h5py.Dataset) and isinstance(args, np.ndarray):\n            # This is needed for h5py 2.9 compatibility\n            args = args.tolist()\n        return self.data[args]\n\n    def append(self, arg):\n        self._validate_new_data_element(arg)\n        self.__data = append_data(self.__data, arg)\n\n    def extend(self, arg):\n        \"\"\"\n        The extend_data method adds all the elements of the iterable arg to the\n        end of the data of this Data container.\n\n        :param arg: The iterable to add to the end of this VectorData\n        \"\"\"\n        self._validate_new_data(arg)\n        self.__data = extend_data(self.__data, arg)\n\n    def _validate_new_data(self, data):\n        \"\"\"Function to validate a new array that will be set or added to data. Raises an error if the data is invalid.\n",
  "start_line": 960,
  "end_line": 1000,
  "total_lines": 1619,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "Shape of list of Data objects: (2, 2)\nFAILURE: Expected (2,), got (2, 2)\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/utils.py",
  "content": "import collections\nimport copy as _copy\nimport re\nimport types\nimport warnings\nfrom abc import ABCMeta\nfrom enum import Enum\n\nimport h5py\nimport numpy as np\n\n\n__macros = {\n    'array_data': [np.ndarray, list, tuple, h5py.Dataset],\n    'scalar_data': [str, int, float, bytes, bool],\n    'data': []\n}\n\ntry:\n    # optionally accept zarr.Array as array data to support conversion of data from Zarr to HDMF\n    import zarr\n    __macros['array_data'].append(zarr.Array)\nexcept ImportError:\n    pass\n\n\n# code to signify how to handle positional arguments in docval\nAllowPositional = Enum('AllowPositional', 'ALLOWED WARNING ERROR')\n\n__supported_bool_types = (bool, np.bool_)\n__supported_uint_types = (np.uint8, np.uint16, np.uint32, np.uint64)\n__supported_int_types = (int, np.int8, np.int16, np.int32, np.int64)\n__supported_float_types = [float, np.float16, np.float32, np.float64]\nif hasattr(np, \"float128\"):  # pragma: no cover\n    __supported_float_types.append(np.float128)\nif hasattr(np, \"longdouble\"):  # pragma: no cover\n    # on windows python<=3.5, h5py floats resolve float64s as either np.float64 or np.longdouble\n    # non-deterministically. a future version of h5py will fix this. see #112\n    __supported_float_types.append(np.longdouble)\n__supported_float_types = tuple(__supported_float_types)\n__allowed_enum_types = (__supported_bool_types + __supported_uint_types + __supported_int_types\n                        + __supported_float_types + (str,))\n\n\ndef docval_macro(macro):\n    \"\"\"Class decorator to add the class to a list of types associated with the key macro in the __macros dict\n    \"\"\"\n\n    def _dec(cls):\n        if macro not in __macros:",
  "start_line": 1,
  "end_line": 50,
  "total_lines": 1155,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "Data object has _fieldsname: True\nNumpy array has _fieldsname: False\nList has _fieldsname: False\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "Shape: (2, 2)\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "Imports worked\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "src/hdmf/build/classgenerator.py:218:        :param classdict: The dict to update with __fields__ (or a different parent_cls._fieldsname).\nsrc/hdmf/build/classgenerator.py:240:        classdict.setdefault(parent_cls._fieldsname, list()).append(fields_conf)\nsrc/hdmf/build/classgenerator.py:283:        :param classdict: The class dictionary to convert with '__fields__' key (or a different bases[0]._fieldsname)\nsrc/hdmf/build/classgenerator.py:290:            fields = classdict.get(b._fieldsname)\nsrc/hdmf/build/classgenerator.py:292:                classdict[b._fieldsname] = tuple(fields)\nsrc/hdmf/container.py:58:    _fieldsname = '__fields__'\nsrc/hdmf/container.py:202:                       % (k, cls._fieldsname, field_conf['name'], cls.__name__))\nsrc/hdmf/container.py:207:        return getattr(cls, cls._fieldsname)\nsrc/hdmf/container.py:211:        return setattr(cls, cls._fieldsname, value)\nsrc/hdmf/container.py:225:            msg = \"'%s' must be of type tuple\" % cls._fieldsname\nsrc/hdmf/testing/testcase.py:73:        for field in getattr(container1, type1._fieldsname):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "74:       The difference between this function and :py:func:`isinstance` is that\n89:    if isinstance(argtype, str):\n105:    elif isinstance(argtype, type):\n112:        return isinstance(value, argtype)\n113:    elif isinstance(argtype, tuple) or isinstance(argtype, list):\n137:    return isinstance(value, __supported_uint_types)\n141:    return isinstance(value, __supported_int_types)\n145:    return isinstance(value, __supported_float_types)\n149:    return isinstance(value, __supported_bool_types)\n153:    if isinstance(argtype, str):\n155:    elif isinstance(argtype, type):\n157:    elif isinstance(argtype, tuple) or isinstance(argtype, list):\n185:    if isinstance(x, (list, tuple)):\n187:    if isinstance(x, str):\n282:                if isinstance(argval, TermSetWrapper):\n339:            if isinstance(argval, TermSetWrapper):\n419:    if isinstance(t, str):\n424:    elif isinstance(t, type):\n426:    elif isinstance(t, (list, tuple)):\n430:            if isinstance(resolved, tuple):\n442:    if isinstance(argtype, (list, tuple)):\n521:                if not isinstance(a['enum'], (list, tuple)):\n600:        if isinstance(default, str):\n613:        if isinstance(argtype, type):\n623:        elif isinstance(argtype, str):\n638:        if isinstance(type_arg, tuple) or isinstance(type_arg, list):\n687:    if not isinstance(argnames[-1], dict):\n715:    if not isinstance(argnames[-1], dict):\n821:                if not isinstance(el, (str, bytes)):\n830:    if isinstance(data, dict):\n832:    if hasattr(data, '__len__') and not isinstance(data, (str, bytes)):\n833:        if not strict_no_data_load or isinstance(data, (list, tuple, set)):\n842:    if isinstance(s, bytes):\n855:    if not isinstance(arr, np.ndarray):\n871:    if isinstance(data, (list, tuple)):\n872:        lengths = [len(sub_data) if isinstance(sub_data, (list, tuple)) else 1 for sub_data in data]\n",
  "stderr": "",
  "exit_code": 0
}