hdmf-dev__hdmf-1270

{
  "status": "ok",
  "stdout": "src/hdmf/utils.py:796:def get_data_shape(data, strict_no_data_load=False):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/utils.py",
  "content": "def get_data_shape(data, strict_no_data_load=False):\n    \"\"\"\n    Helper function used to determine the shape of the given array.\n\n    In order to determine the shape of nested tuples, lists, and sets, this function\n    recursively inspects elements along the dimensions, assuming that the data has a regular,\n    rectangular shape. In the case of out-of-core iterators, this means that the first item\n    along each dimension would potentially be loaded into memory. Set strict_no_data_load=True\n    to enforce that this does not happen, at the cost that we may not be able to determine\n    the shape of the array.\n\n    :param data: Array for which we should determine the shape. Can be any object that supports __len__ or .shape.\n    :type data: List, numpy.ndarray, DataChunkIterator\n    :param strict_no_data_load: If True and data is an out-of-core iterator, None may be returned. If False (default),\n                                the first element of data may be loaded into memory.\n    :return: Tuple of ints indicating the size of known dimensions. Dimensions for which the size is unknown\n             will be set to None.\n    \"\"\"\n\n    def __get_shape_helper(local_data):\n        shape = list()\n        if hasattr(local_data, '__len__'):\n            shape.append(len(local_data))\n            if len(local_data):\n                el = next(iter(local_data))\n                if not isinstance(el, (str, bytes)):\n                    shape.extend(__get_shape_helper(el))\n        return tuple(shape)\n\n    # NOTE: data.maxshape will fail on empty h5py.Dataset without shape or maxshape. this will be fixed in h5py 3.0\n    if hasattr(data, 'maxshape'):\n        return data.maxshape\n    if hasattr(data, 'shape') and data.shape is not None:\n        return data.shape\n    if isinstance(data, dict):\n        return None\n    if hasattr(data, '__len__') and not isinstance(data, (str, bytes)):\n        if not strict_no_data_load or isinstance(data, (list, tuple, set)):\n            return __get_shape_helper(data)\n    return None\n\n\ndef pystr(s):\n    \"\"\"\n    Convert a string of characters to Python str object\n    \"\"\"\n    if isinstance(s, bytes):\n        return s.decode('utf-8')\n    else:\n        return s\n\n\ndef to_uint_array(arr):\n    \"\"\"\n    Convert a numpy array or array-like object to a numpy array of unsigned integers with the same dtype itemsize.\n\n    For example, a list of int32 values is converted to a numpy array with dtype uint32.\n    :raises ValueError: if input array contains values that are not unsigned integers or non-negative integers.\n    \"\"\"\n    if not isinstance(arr, np.ndarray):\n        arr = np.array(arr)\n    if np.issubdtype(arr.dtype, np.unsignedinteger):\n        return arr\n    if np.issubdtype(arr.dtype, np.integer):\n        if (arr < 0).any():\n            raise ValueError('Cannot convert negative integer values to uint.')\n        dt = np.dtype('uint' + str(int(arr.dtype.itemsize*8)))  # keep precision\n        return arr.astype(dt)\n    raise ValueError('Cannot convert array of dtype %s to uint.' % arr.dtype)\n\n\ndef is_ragged(data):\n    \"\"\"\n    Test whether a list of lists or array is ragged / jagged\n    \"\"\"\n    if isinstance(data, (list, tuple)):\n        lengths = [len(sub_data) if isinstance(sub_data, (list, tuple)) else 1 for sub_data in data]\n        if len(set(lengths)) > 1:\n            return True  # ragged at this level\n\n        return any(is_ragged(sub_data) for sub_data in data)  # check next level\n\n    return False\n\ndef is_newer_version(version_a: str, version_b: str) -> bool:\n    # this method could be replaced by packaging.version if packaging is added as a dependency\n    version_a_match = re.match(r\"(\\d+\\.\\d+\\.\\d+)\", version_a)[0]  # trim off any non-numeric symbols at end\n    version_a_list = [int(i) for i in version_a_match.split(\".\")]\n\n    version_b_match = re.match(r\"(\\d+\\.\\d+\\.\\d+)\", version_b)[0]  # trim off any non-numeric symbols at end\n    version_b_list = [int(i) for i in version_b_match.split(\".\")]\n\n    for a, b in zip(version_a_list, version_b_list):\n        if a > b:\n            return True\n        elif a < b:\n            return False\n\n    return False\n\ndef get_basic_array_info(array):\n    def convert_bytes_to_str(bytes_size):\n        suffixes = ['bytes', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB']\n        i = 0\n        while bytes_size >= 1024 and i < len(suffixes)-1:\n            bytes_size /= 1024.\n            i += 1\n        return f\"{bytes_size:.2f} {suffixes[i]}\"\n\n    if hasattr(array, \"nbytes\"):  # TODO: Remove this after h5py minimal version is larger than 3.0\n        array_size_in_bytes = array.nbytes\n    else:\n        array_size_in_bytes = array.size * array.dtype.itemsize\n    array_size_repr = convert_bytes_to_str(array_size_in_bytes)\n    basic_array_info_dict = {\"Data type\": array.dtype, \"Shape\": array.shape, \"Array size\": array_size_repr}\n\n    return basic_array_info_dict\n\ndef generate_array_html_repr(array_info_dict, array, dataset_type=None):\n    def html_table(item_dicts) -> str:\n        \"\"\"\n        Generates an html table from a dictionary\n        \"\"\"\n        report = '<table class=\"data-info\">'\n        report += \"<tbody>\"\n        for k, v in item_dicts.items():\n            report += (\n                f\"<tr>\"\n                f'<th style=\"text-align: left\">{k}</th>'\n                f'<td style=\"text-align: left\">{v}</td>'\n                f\"</tr>\"\n            )\n        report += \"</tbody>\"\n        report += \"</table>\"\n        return report\n\n    array_info_html = html_table(array_info_dict)\n    repr_html = dataset_type + \"<br>\" + array_info_html if dataset_type is not None else array_info_html\n\n    # Array like might lack nbytes (h5py < 3.0) or size (DataIO object)\n    if hasattr(array, \"nbytes\"):\n        array_size_bytes = array.nbytes\n    else:\n        if hasattr(array, \"size\"):\n            array_size = array.size\n        else:\n            import math\n            array_size = math.prod(array.shape)\n        array_size_bytes = array_size * array.dtype.itemsize\n",
  "start_line": 796,
  "end_line": 945,
  "total_lines": 1155,
  "is_truncated": true
}

{
  "status": "ok",
  "stdout": "src/hdmf/backends/hdf5/h5_utils.py:120:class DatasetOfReferences(H5Dataset, ReferenceResolver, metaclass=ABCMeta):\nsrc/hdmf/data_utils.py:479:class DataChunkIterator(AbstractDataChunkIterator):\nsrc/hdmf/data_utils.py:763:class DataChunk:\nsrc/hdmf/data_utils.py:1005:class DataIO:\nsrc/hdmf/build/builders.py:320:class DatasetBuilder(BaseBuilder):\nsrc/hdmf/common/io/resources.py:12:           The DatasetBuilder for name is associated with data_type Data and container class Data,\nsrc/hdmf/spec/spec.py:659:class DatasetSpec(BaseStorageSpec):\nsrc/hdmf/monitor.py:19:class DataChunkProcessor(AbstractDataChunkIterator, metaclass=ABCMeta):\nsrc/hdmf/validate/validator.py:418:class DatasetValidator(BaseStorageValidator):\nsrc/hdmf/container.py:892:class Data(AbstractContainer):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/container.py",
  "content": "class Data(AbstractContainer):\n    \"\"\"\n    A class for representing dataset containers\n    \"\"\"\n    @docval({'name': 'name', 'type': str, 'doc': 'the name of this container'},\n            {'name': 'data', 'type': ('scalar_data', 'array_data', 'data'), 'doc': 'the source of the data'})\n    def __init__(self, **kwargs):\n        data = popargs('data', kwargs)\n        super().__init__(**kwargs)\n\n        self._validate_new_data(data)\n        self.__data = data\n\n    @property\n    def data(self):\n        return self.__data\n\n    @property\n    def shape(self):\n        \"\"\"\n        Get the shape of the data represented by this container\n        :return: Shape tuple\n        :rtype: tuple of ints\n        \"\"\"\n        return get_data_shape(self.__data)\n\n    def set_data_io(\n        self,\n        data_io_class: Type[DataIO],\n        data_io_kwargs: dict,\n        data_chunk_iterator_class: Optional[Type[AbstractDataChunkIterator]] = None,\n        data_chunk_iterator_kwargs: dict = None,\n    ) -> None:\n        \"\"\"\n        Apply DataIO object to the data held by this Data object.\n\n        Parameters\n        ----------\n        data_io_class: Type[DataIO]\n            The DataIO to apply to the data held by this Data.\n        data_io_kwargs: dict\n            The keyword arguments to pass to the DataIO.\n        data_chunk_iterator_class: Type[AbstractDataChunkIterator]\n            The DataChunkIterator to use for the DataIO. If None, no DataChunkIterator is used.\n        data_chunk_iterator_kwargs: dict\n            The keyword arguments to pass to the DataChunkIterator.\n\n        Notes\n        -----\n        If data_chunk_iterator_class is not None, the data is wrapped in the DataChunkIterator before being wrapped in\n        the DataIO. This allows for rewriting the backend configuration of hdf5 datasets.\n        \"\"\"\n        data_chunk_iterator_kwargs = data_chunk_iterator_kwargs or dict()\n        data = self.__data\n        if data_chunk_iterator_class is not None:\n            data = data_chunk_iterator_class(data=data, **data_chunk_iterator_kwargs)\n        self.__data = data_io_class(data=data, **data_io_kwargs)\n\n    @docval({'name': 'func', 'type': types.FunctionType, 'doc': 'a function to transform *data*'})\n    def transform(self, **kwargs):\n        \"\"\"\n        Transform data from the current underlying state.\n\n        This function can be used to permanently load data from disk, or convert to a different\n        representation, such as a torch.Tensor\n        \"\"\"\n        func = getargs('func', kwargs)\n        self.__data = func(self.__data)\n        return self\n\n    def __bool__(self):\n        if self.data is not None:\n            if isinstance(self.data, (np.ndarray, tuple, list)):\n                return len(self.data) != 0\n            if self.data:\n                return True\n        return False\n\n    def __len__(self):\n        return len(self.__data)\n\n    def __getitem__(self, args):\n        return self.get(args)\n\n    def get(self, args):\n        if isinstance(self.data, (tuple, list)) and isinstance(args, (tuple, list, np.ndarray)):\n            return [self.data[i] for i in args]\n        if isinstance(self.data, h5py.Dataset) and isinstance(args, np.ndarray):\n            # This is needed for h5py 2.9 compatibility\n            args = args.tolist()\n        return self.data[args]\n\n    def append(self, arg):\n        self._validate_new_data_element(arg)\n        self.__data = append_data(self.__data, arg)\n\n    def extend(self, arg):\n        \"\"\"\n        The extend_data method adds all the elements of the iterable arg to the\n        end of the data of this Data container.\n\n        :param arg: The iterable to add to the end of this VectorData\n        \"\"\"\n        self._validate_new_data(arg)\n        self.__data = extend_data(self.__data, arg)\n\n    def _validate_new_data(self, data):\n        \"\"\"Function to validate a new array that will be set or added to data. Raises an error if the data is invalid.\n\n        Subclasses should override this function to perform class-specific validation.\n        \"\"\"\n        pass\n\n    def _validate_new_data_element(self, arg):\n        \"\"\"Function to validate a new value that will be added to the data. Raises an error if the data is invalid.\n\n        Subclasses should override this function to perform class-specific validation.\n        \"\"\"\n        pass\n\n\nclass MultiContainerInterface(Container):\n    \"\"\"Class that dynamically defines methods to support a Container holding multiple Containers of the same type.\n\n    To use, extend this class and create a dictionary as a class attribute with any of the following keys:\n    * 'attr' to name the attribute that stores the Container instances\n    * 'type' to provide the Container object type (type or list/tuple of types, type can be a docval macro)\n    * 'add' to name the method for adding Container instances\n    * 'get' to name the method for getting Container instances\n    * 'create' to name the method for creating Container instances (only if a single type is specified)\n\n    If the attribute does not exist in the class, it will be generated. If it does exist, it should behave like a dict.\n\n    The keys 'attr', 'type', and 'add' are required.\n    \"\"\"\n\n    def __new__(cls, *args, **kwargs):\n        if cls is MultiContainerInterface:\n            raise TypeError(\"Can't instantiate class MultiContainerInterface.\")\n        if not hasattr(cls, '__clsconf__'):\n            raise TypeError(\"MultiContainerInterface subclass %s is missing __clsconf__ attribute. Please check that \"\n                            \"the class is properly defined.\" % cls.__name__)\n        return super().__new__(cls, *args, **kwargs)\n\n    @staticmethod\n    def __add_article(noun):\n        if isinstance(noun, tuple):\n            noun = noun[0]\n        if isinstance(noun, type):\n            noun = noun.__name__",
  "start_line": 892,
  "end_line": 1041,
  "total_lines": 1619,
  "is_truncated": true
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "",
  "details": {
    "stdout": "",
    "stderr": "",
    "exit_code": 1
  }
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "",
  "details": {
    "stdout": "",
    "stderr": "",
    "exit_code": 1
  }
}

{
  "status": "ok",
  "stdout": "13:from .data_utils import DataIO, append_data, extend_data, AbstractDataChunkIterator\n14:from .utils import (docval, get_docval, getargs, ExtenderMeta, get_data_shape, popargs, LabelledDict,\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "Traceback (most recent call last):\n  File \"<stdin>\", line 18, in <module>\nAssertionError: Expected (2,), got (2, 2)\n",
  "details": {
    "stdout": "List of Data: (2, 2)\nNested list: (2, 2)\n",
    "stderr": "Traceback (most recent call last):\n  File \"<stdin>\", line 18, in <module>\nAssertionError: Expected (2,), got (2, 2)\n",
    "exit_code": 1
  }
}

{
  "status": "ok",
  "stdout": "src/hdmf/container.py:51:class AbstractContainer(metaclass=ExtenderMeta):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "Data imported\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "src/hdmf/utils.py",
  "content": "def get_data_shape(data, strict_no_data_load=False):\n    \"\"\"\n    Helper function used to determine the shape of the given array.\n\n    In order to determine the shape of nested tuples, lists, and sets, this function\n    recursively inspects elements along the dimensions, assuming that the data has a regular,\n    rectangular shape. In the case of out-of-core iterators, this means that the first item\n    along each dimension would potentially be loaded into memory. Set strict_no_data_load=True\n    to enforce that this does not happen, at the cost that we may not be able to determine\n    the shape of the array.\n\n    :param data: Array for which we should determine the shape. Can be any object that supports __len__ or .shape.\n    :type data: List, numpy.ndarray, DataChunkIterator\n    :param strict_no_data_load: If True and data is an out-of-core iterator, None may be returned. If False (default),\n                                the first element of data may be loaded into memory.\n    :return: Tuple of ints indicating the size of known dimensions. Dimensions for which the size is unknown\n             will be set to None.\n    \"\"\"\n\n    def __get_shape_helper(local_data):\n        shape = list()\n        if hasattr(local_data, '__len__'):\n            shape.append(len(local_data))\n            if len(local_data):\n                el = next(iter(local_data))\n                if not isinstance(el, (str, bytes)):\n                    shape.extend(__get_shape_helper(el))\n        return tuple(shape)\n\n    # NOTE: data.maxshape will fail on empty h5py.Dataset without shape or maxshape. this will be fixed in h5py 3.0",
  "start_line": 796,
  "end_line": 825,
  "total_lines": 1155,
  "is_truncated": false
}

{
  "status": "ok",
  "filepath": "src/hdmf/utils.py",
  "occurrences": 1,
  "strategy": "unescaped",
  "diff": "--- a/src/hdmf/utils.py\n+++ b/src/hdmf/utils.py\n@@ -798,7 +798,7 @@\n     Helper function used to determine the shape of the given array.\n \n     In order to determine the shape of nested tuples, lists, and sets, this function\n-    recursively inspects elements along the dimensions, assuming that the data has a regular,\n+    recurs recursively inspects elements along the dimensions, assuming that the data has a regular,\n     rectangular shape. In the case of out-of-core iterators, this means that the first item\n     along each dimension would potentially be loaded into memory. Set strict_no_data_load=True\n     to enforce that this does not happen, at the cost that we may not be able to determine\n@@ -811,6 +811,7 @@\n     :return: Tuple of ints indicating the size of known dimensions. Dimensions for which the size is unknown\n              will be set to None.\n     \"\"\"\n+    from hdmf.container import Data\n \n     def __get_shape_helper(local_data):\n         shape = list()\n@@ -818,7 +819,7 @@\n             shape.append(len(local_data))\n             if len(local_data):\n                 el = next(iter(local_data))\n-                if not isinstance(el, (str, bytes)):\n+                if not isinstance(el, (str, bytes, Data)):\n                     shape.extend(__get_shape_helper(el))\n         return tuple(shape)\n \n",
  "is_truncated": false
}

{
  "status": "ok",
  "filepath": "src/hdmf/utils.py",
  "occurrences": 1,
  "strategy": "unescaped",
  "diff": "--- a/src/hdmf/utils.py\n+++ b/src/hdmf/utils.py\n@@ -798,7 +798,7 @@\n     Helper function used to determine the shape of the given array.\n \n     In order to determine the shape of nested tuples, lists, and sets, this function\n-    recurs recursively inspects elements along the dimensions, assuming that the data has a regular,\n+    recursively inspects elements along the dimensions, assuming that the data has a regular,\n     rectangular shape. In the case of out-of-core iterators, this means that the first item\n     along each dimension would potentially be loaded into memory. Set strict_no_data_load=True\n     to enforce that this does not happen, at the cost that we may not be able to determine\n",
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "List of Data: (2,)\nNested list: (2, 2)\nAll tests passed!\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "556:class Container(AbstractContainer):\n",
  "stderr": "",
  "exit_code": 0
}