ecmwf__earthkit-data-111

{
  "status": "ok",
  "stdout": "earthkit/data/core/statistics.py:71:    def to_pandas(self):\nearthkit/data/core/__init__.py:47:    def to_pandas(self, **kwargs):\nearthkit/data/mergers/pandas.py:21:    return pd.concat([s.to_pandas(**kwargs) for s in sources], **options)\nearthkit/data/mergers/__init__.py:17:FORWARDS = (\"to_xarray\", \"to_pandas\", \"to_tfdataset\")\nearthkit/data/mergers/__init__.py:74:    def to_pandas(self, **kwargs):\nearthkit/data/mergers/__init__.py:113:    def to_pandas(self, *args, **kwargs):\nearthkit/data/mergers/__init__.py:114:        return self.obj.to_pandas(self.paths_or_sources, **kwargs)\nearthkit/data/mergers/__init__.py:129:    to_pandas = _call_func\nearthkit/data/sources/multi.py:100:    def to_pandas(self, **kwargs):\nearthkit/data/sources/multi.py:101:        return make_merger(self.merger, self.sources).to_pandas(**kwargs)\nearthkit/data/sources/cds.py:130:    def to_pandas(self, **kwargs):\nearthkit/data/sources/cds.py:146:        return super().to_pandas(\nearthkit/data/sources/memory.py:41:    def to_pandas(self, **kwargs):\nearthkit/data/sources/memory.py:42:        return self._reader.to_pandas(**kwargs)\nearthkit/data/sources/ecmwf_api.py:99:    def to_pandas(self, **kwargs):\nearthkit/data/sources/ecmwf_api.py:117:        return super().to_pandas(\nearthkit/data/sources/file.py:105:    def to_pandas(self, **kwargs):\nearthkit/data/sources/file.py:106:        LOG.debug(\"Calling reader.to_pandas %s\", self)\nearthkit/data/sources/file.py:107:        return self._reader.to_pandas(**kwargs)\nearthkit/data/wrappers/xarray.py:71:    def to_pandas(self, *args, **kwargs):\nearthkit/data/readers/bufr.py:40:    def to_pandas(self, columns=COLUMNS, filters=None, **kwargs):\nearthkit/data/readers/grib/pandas.py:16:    def to_pandas(self, latitude=None, longitude=None, **kwargs):\nearthkit/data/readers/netcdf.py:191:    def to_pandas(self):\nearthkit/data/readers/netcdf.py:192:        return self._da.to_pandas()\nearthkit/data/readers/netcdf.py:377:    def to_pandas(self):\nearthkit/data/readers/netcdf.py:378:        return self.to_xarray().to_pandas()\nearthkit/data/readers/odb.py:18:    def to_pandas(self, **kwargs):\nearthkit/data/readers/csv.py:135:    def to_pandas(self, **kwargs):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "earthkit/data/readers/bufr.py",
  "content": "# (C) Copyright 2020 ECMWF.\n#\n# This software is licensed under the terms of the Apache Licence Version 2.0\n# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.\n# In applying this licence, ECMWF does not waive the privileges and immunities\n# granted to it by virtue of its status as an intergovernmental organisation\n# nor does it submit to any jurisdiction.\n#\n\n# See:\n# https://github.com/ecmwf/pdbufr\nfrom . import Reader\n\nCOLUMNS = (\"latitude\", \"longitude\", \"data_datetime\")\n\nBUFR_LS_KEYS = {\n    \"edition\": \"edition\",\n    \"type\": \"dataCategory\",\n    \"subtype\": \"dataSubCategory\",\n    \"c\": \"bufrHeaderCentre\",\n    \"mv\": \"masterTablesVersionNumber\",\n    \"lv\": \"localTablesVersionNumber\",\n    \"subsets\": \"numberOfSubsets\",\n    \"compr\": \"compressedData\",\n    \"typicalDate\": \"typicalDate\",\n    \"typicalTime\": \"typicalTime\",\n    \"ident\": \"ident\",\n    \"lat\": \"localLatitude\",\n    \"lon\": \"localLongitude\",\n}\n\n\nclass BUFRReader(Reader):\n    \"\"\"Represents a BUFR file\"\"\"\n\n    def __init__(self, source, path):\n        super().__init__(source, path)\n        self._num = None\n\n    def to_pandas(self, columns=COLUMNS, filters=None, **kwargs):\n        \"\"\"Extracts BUFR data into an pandas DataFranme using :xref:`pdbufr`.\n\n        Parameters\n        ----------\n        columns: str, sequence[str]\n            List of ecCodes BUFR keys to extract for each BUFR message/subset.\n            See: :xref:`read_bufr` for details.\n        filters: dict\n            Defines the conditions when to extract the specified ``columns``. See:\n            :xref:`read_bufr` for details.\n        **kwargs: dict, optional\n            Other keyword arguments:\n\n        Returns\n        -------\n        Pandas DataFrame\n\n        Examples\n        --------\n        :ref:`/examples/bufr.ipynb`\n\n        \"\"\"\n        import pdbufr\n\n        return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n\n    def __len__(self):\n        if self._num is None:\n            import eccodes\n\n            with open(self.path, \"rb\") as f:\n                self._num = eccodes.codes_count_in_file(f)\n        return self._num\n\n    def ls(self, *args, **kwargs):\n        from pdbufr.high_level_bufr.bufr import BufrFile\n\n        from earthkit.data.utils.summary import ls\n\n        b = BufrFile(self.path)\n\n        def _proc(keys, n):\n            count_start = 0\n            if n is None:\n                count_end = len(self)\n            elif n > 0:\n                count_end = n\n            else:\n                num = len(self)\n                count_start = max(0, num + n)\n                count_end = num\n\n            for count, msg in enumerate(b):\n                with msg:\n                    if count_start <= count < count_end:\n                        yield ({k: msg[k1] for k, k1 in keys.items()})\n                    elif count >= count_end:\n                        break\n\n        return ls(_proc, BUFR_LS_KEYS, *args, **kwargs)",
  "start_line": 1,
  "end_line": 100,
  "total_lines": 105,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "./docs/examples/temp_10.bufr\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "",
  "stderr": "Traceback (most recent call last):\n  File \"<stdin>\", line 6, in <module>\n  File \"/workspace/earthkit/data/sources/file.py\", line 107, in to_pandas\n    return self._reader.to_pandas(**kwargs)\n  File \"/workspace/earthkit/data/readers/bufr.py\", line 65, in to_pandas\n    return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n  File \"/usr/local/lib/python3.10/site-packages/pdbufr/bufr_read.py\", line 29, in read_bufr\n    return reader.execute()\n  File \"/usr/local/lib/python3.10/site-packages/pdbufr/readers/__init__.py\", line 45, in execute\n    return _read(bufr_obj)\n  File \"/usr/local/lib/python3.10/site-packages/pdbufr/readers/__init__.py\", line 39, in _read\n    df = pd.DataFrame.from_records(rows)\n  File \"/usr/local/lib/python3.10/site-packages/pandas/core/frame.py\", line 2459, in from_records\n    first_row = next(data)\n  File \"/usr/local/lib/python3.10/site-packages/pdbufr/readers/generic.py\", line 215, in read_records\n    filters = dict(filters)\nTypeError: 'NoneType' object is not iterable\n",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "Success!\n   latitude  longitude\n0     67.37      26.63\n1     56.90       3.35\n2     58.87       5.67\n3     70.93      -8.67\n4     67.28      14.45\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "earthkit/data/readers/bufr.py",
  "occurrences": 1,
  "strategy": "exact",
  "diff": "--- a/earthkit/data/readers/bufr.py\n+++ b/earthkit/data/readers/bufr.py\n@@ -61,6 +61,8 @@\n \n         \"\"\"\n         import pdbufr\n+        if filters is None:\n+            filters = {}\n \n         return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n \n",
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "Success!\n   latitude  longitude\n0     67.37      26.63\n1     56.90       3.35\n2     58.87       5.67\n3     70.93      -8.67\n4     67.28      14.45\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "earthkit/data/readers/bufr.py:67:        return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "============================= test session starts ==============================\ncollected 351 items / 346 deselected / 5 selected\n\ntests/bufr/test_bufr_summary.py ....\ntests/documentation/test_notebooks.py s\n\n=============================== warnings summary ===============================\ntests/sources/test_cds.py:19\n  /workspace/tests/sources/test_cds.py:19: PytestUnknownMarkWarning: Unknown pytest.mark.download - is this a typo?  You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html\n    @pytest.mark.download\n\ntests/sources/test_cds.py:35\n  /workspace/tests/sources/test_cds.py:35: PytestUnknownMarkWarning: Unknown pytest.mark.download - is this a typo?  You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html\n    @pytest.mark.download\n\ntests/sources/test_cds.py:52\n  /workspace/tests/sources/test_cds.py:52: PytestUnknownMarkWarning: Unknown pytest.mark.download - is this a typo?  You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html\n    @pytest.mark.download\n\ntests/sources/test_cds.py:68\n  /workspace/tests/sources/test_cds.py:68: PytestUnknownMarkWarning: Unknown pytest.mark.download - is this a typo?  You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html\n    @pytest.mark.download\n\ntests/sources/test_mars.py:19\n  /workspace/tests/sources/test_mars.py:19: PytestUnknownMarkWarning: Unknown pytest.mark.download - is this a typo?  You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html\n    @pytest.mark.download\n\ntests/sources/test_mars.py:34\n  /workspace/tests/sources/test_mars.py:34: PytestUnknownMarkWarning: Unknown pytest.mark.download - is this a typo?  You can register custom marks to avoid this warning - for details, see https://docs.pytest.org/en/stable/how-to/mark.html\n    @pytest.mark.download\n\n-- Docs: https://docs.pytest.org/en/stable/how-to/capture-warnings.html\n=========== 4 passed, 1 skipped, 346 deselected, 6 warnings in 0.70s ===========\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "test_bufr_summary.py\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "tests/bufr\ntests/bufr/test_bufr_summary.py\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "",
  "stderr": "Traceback (most recent call last):\n  File \"<stdin>\", line 6, in <module>\n  File \"/workspace/earthkit/data/sources/file.py\", line 107, in to_pandas\n    return self._reader.to_pandas(**kwargs)\n  File \"/workspace/earthkit/data/readers/bufr.py\", line 67, in to_pandas\n    return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n  File \"/usr/local/lib/python3.10/site-packages/pdbufr/bufr_read.py\", line 29, in read_bufr\n    return reader.execute()\n  File \"/usr/local/lib/python3.10/site-packages/pdbufr/readers/__init__.py\", line 45, in execute\n    return _read(bufr_obj)\n  File \"/usr/local/lib/python3.10/site-packages/pdbufr/readers/__init__.py\", line 39, in _read\n    df = pd.DataFrame.from_records(rows)\n  File \"/usr/local/lib/python3.10/site-packages/pandas/core/frame.py\", line 2459, in from_records\n    first_row = next(data)\n  File \"/usr/local/lib/python3.10/site-packages/pdbufr/readers/generic.py\", line 207, in read_records\n    required_columns = set(columns)\nTypeError: 'NoneType' object is not iterable\n",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": " M earthkit/data/readers/bufr.py\ndiff --git a/earthkit/data/readers/bufr.py b/earthkit/data/readers/bufr.py\nindex 59c7672..14e9aed 100644\n--- a/earthkit/data/readers/bufr.py\n+++ b/earthkit/data/readers/bufr.py\n@@ -61,6 +61,8 @@ class BUFRReader(Reader):\n \n         \"\"\"\n         import pdbufr\n+        if filters is None:\n+            filters = {}\n \n         return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n \n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "patch_size": 396,
  "files_changed": 1
}