ecmwf__earthkit-data-111

{
  "status": "ok",
  "stdout": "./earthkit/data/core/statistics.py:71:    def to_pandas(self):\n./earthkit/data/core/__init__.py:47:    def to_pandas(self, **kwargs):\n./earthkit/data/mergers/__init__.py:74:    def to_pandas(self, **kwargs):\n./earthkit/data/mergers/__init__.py:113:    def to_pandas(self, *args, **kwargs):\n./earthkit/data/sources/multi.py:100:    def to_pandas(self, **kwargs):\n./earthkit/data/sources/cds.py:130:    def to_pandas(self, **kwargs):\n./earthkit/data/sources/memory.py:41:    def to_pandas(self, **kwargs):\n./earthkit/data/sources/ecmwf_api.py:99:    def to_pandas(self, **kwargs):\n./earthkit/data/sources/file.py:105:    def to_pandas(self, **kwargs):\n./earthkit/data/wrappers/xarray.py:71:    def to_pandas(self, *args, **kwargs):\n./earthkit/data/readers/bufr.py:40:    def to_pandas(self, columns=COLUMNS, filters=None, **kwargs):\n./earthkit/data/readers/grib/pandas.py:16:    def to_pandas(self, latitude=None, longitude=None, **kwargs):\n./earthkit/data/readers/netcdf.py:191:    def to_pandas(self):\n./earthkit/data/readers/netcdf.py:377:    def to_pandas(self):\n./earthkit/data/readers/odb.py:18:    def to_pandas(self, **kwargs):\n./earthkit/data/readers/csv.py:135:    def to_pandas(self, **kwargs):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "earthkit/data/readers/bufr.py",
  "content": "# (C) Copyright 2020 ECMWF.\n#\n# This software is licensed under the terms of the Apache Licence Version 2.0\n# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.\n# In applying this licence, ECMWF does not waive the privileges and immunities\n# granted to it by virtue of its status as an intergovernmental organisation\n# nor does it submit to any jurisdiction.\n#\n\n# See:\n# https://github.com/ecmwf/pdbufr\nfrom . import Reader\n\nCOLUMNS = (\"latitude\", \"longitude\", \"data_datetime\")\n\nBUFR_LS_KEYS = {\n    \"edition\": \"edition\",\n    \"type\": \"dataCategory\",\n    \"subtype\": \"dataSubCategory\",\n    \"c\": \"bufrHeaderCentre\",\n    \"mv\": \"masterTablesVersionNumber\",\n    \"lv\": \"localTablesVersionNumber\",\n    \"subsets\": \"numberOfSubsets\",\n    \"compr\": \"compressedData\",\n    \"typicalDate\": \"typicalDate\",\n    \"typicalTime\": \"typicalTime\",\n    \"ident\": \"ident\",\n    \"lat\": \"localLatitude\",\n    \"lon\": \"localLongitude\",\n}\n\n\nclass BUFRReader(Reader):\n    \"\"\"Represents a BUFR file\"\"\"\n\n    def __init__(self, source, path):\n        super().__init__(source, path)\n        self._num = None\n\n    def to_pandas(self, columns=COLUMNS, filters=None, **kwargs):\n        \"\"\"Extracts BUFR data into an pandas DataFranme using :xref:`pdbufr`.\n\n        Parameters\n        ----------\n        columns: str, sequence[str]\n            List of ecCodes BUFR keys to extract for each BUFR message/subset.\n            See: :xref:`read_bufr` for details.\n        filters: dict\n            Defines the conditions when to extract the specified ``columns``. See:\n            :xref:`read_bufr` for details.\n        **kwargs: dict, optional\n            Other keyword arguments:\n\n        Returns\n        -------\n        Pandas DataFrame\n\n        Examples\n        --------\n        :ref:`/examples/bufr.ipynb`\n\n        \"\"\"\n        import pdbufr\n\n        return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n\n    def __len__(self):\n        if self._num is None:\n            import eccodes\n\n            with open(self.path, \"rb\") as f:\n                self._num = eccodes.codes_count_in_file(f)\n        return self._num\n\n    def ls(self, *args, **kwargs):\n        from pdbufr.high_level_bufr.bufr import BufrFile\n\n        from earthkit.data.utils.summary import ls\n\n        b = BufrFile(self.path)\n\n        def _proc(keys, n):\n            count_start = 0\n            if n is None:\n                count_end = len(self)\n            elif n > 0:\n                count_end = n\n            else:\n                num = len(self)\n                count_start = max(0, num + n)\n                count_end = num\n\n            for count, msg in enumerate(b):\n                with msg:\n                    if count_start <= count < count_end:\n                        yield ({k: msg[k1] for k, k1 in keys.items()})\n                    elif count >= count_end:\n                        break\n\n        return ls(_proc, BUFR_LS_KEYS, *args, **kwargs)\n\n\ndef reader(source, path, magic=None, deeper_check=False):\n    if magic is None or magic[:4] == b\"BUFR\":\n        return BUFRReader(source, path)",
  "start_line": 1,
  "end_line": 105,
  "total_lines": 105,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "/workspace/docs/examples/temp_10.bufr\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "",
  "stderr": "Traceback (most recent call last):\n  File \"<stdin>\", line 4, in <module>\n  File \"/workspace/earthkit/data/sources/file.py\", line 107, in to_pandas\n    return self._reader.to_pandas(**kwargs)\n  File \"/workspace/earthkit/data/readers/bufr.py\", line 65, in to_pandas\n    return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n  File \"/usr/local/lib/python3.10/site-packages/pdbufr/bufr_read.py\", line 29, in read_bufr\n    return reader.execute()\n  File \"/usr/local/lib/python3.10/site-packages/pdbufr/readers/__init__.py\", line 45, in execute\n    return _read(bufr_obj)\n  File \"/usr/local/lib/python3.10/site-packages/pdbufr/readers/__init__.py\", line 39, in _read\n    df = pd.DataFrame.from_records(rows)\n  File \"/usr/local/lib/python3.10/site-packages/pandas/core/frame.py\", line 2459, in from_records\n    first_row = next(data)\n  File \"/usr/local/lib/python3.10/site-packages/pdbufr/readers/generic.py\", line 215, in read_records\n    filters = dict(filters)\nTypeError: 'NoneType' object is not iterable\n",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "earthkit/data/readers/bufr.py",
  "occurrences": 1,
  "strategy": "exact",
  "diff": "--- a/earthkit/data/readers/bufr.py\n+++ b/earthkit/data/readers/bufr.py\n@@ -62,6 +62,9 @@\n         \"\"\"\n         import pdbufr\n \n+        if filters is None:\n+            filters = {}\n+\n         return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n \n     def __len__(self):\n",
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "Success!\n   latitude  longitude\n0     67.37      26.63\n1     56.90       3.35\n2     58.87       5.67\n3     70.93      -8.67\n4     67.28      14.45\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "./setup.cfg:32:    pdbufr\n./environment.yml:15:- pdbufr\n./earthkit/data/readers/bufr.py:11:# https://github.com/ecmwf/pdbufr\n./earthkit/data/readers/bufr.py:41:        \"\"\"Extracts BUFR data into an pandas DataFranme using :xref:`pdbufr`.\n./earthkit/data/readers/bufr.py:63:        import pdbufr\n./earthkit/data/readers/bufr.py:68:        return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n./earthkit/data/readers/bufr.py:79:        from pdbufr.high_level_bufr.bufr import BufrFile\n./docs/examples/bufr.ipynb:226:    \"to_pandas() extracts data into a Pandas dataframe (uses [pdbufr](\\\"https://github.com/ecmwf/pdbufr\\\") under the hood). The following example shows how the get the temperature profile for a given station:\"\n./docs/conf.py:101:    \"pdbufr\": (\"pdbufr\", \"https://github.com/ecmwf/pdbufr\"),\n./docs/conf.py:103:        \"pdbufr.read_bufr()\",\n./docs/conf.py:104:        \"https://pdbufr.readthedocs.io/en/latest/read_bufr.html\",\n./docs/guide/data_format/bufr.rst:12:BUFR is a message based format and can contain both forecasts and observations. The structure of a BUFR message is typically hierarchical and can be rather complex, so the recommended way is to extract the required data with ``to_pandas()`` into a pandas DataFrame, which is much easier to work with. The BUFR data extraction in ``to_pandas()`` is implemented by :xref:`pdbufr`.\n./tests/environment-unit-tests.yml:16:- pdbufr\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "",
  "details": {
    "stdout": "",
    "stderr": "",
    "exit_code": 1
  }
}

{
  "status": "ok",
  "stdout": "tests/sources/test_url.py:59:        \"https://get.ecmwf.int/repository/test-data/earthkit-data/test-data/temp.bufr\",\ntests/sources/test_url.py:81:        \"https://get.ecmwf.int/repository/test-data/earthkit-data/test-data/temp.bufr\",\ntests/sources/test_url.py:86:        \"https://get.ecmwf.int/repository/test-data/earthkit-data/test-data/temp.bufr\",\ntests/sources/test_url.py:97:        \"https://get.ecmwf.int/repository/test-data/earthkit-data/test-data/temp.bufr\",\ntests/environment-unit-tests.yml:16:- pdbufr\ntests/bufr/test_bufr_summary.py:17:def test_bufr_ls_invalid_num():\ntests/bufr/test_bufr_summary.py:18:    f = from_source(\"file\", earthkit_examples_file(\"temp_10.bufr\"))\ntests/bufr/test_bufr_summary.py:26:def test_bufr_ls_num():\ntests/bufr/test_bufr_summary.py:27:    f = from_source(\"file\", earthkit_examples_file(\"temp_10.bufr\"))\ntests/bufr/test_bufr_summary.py:75:def test_bufr_head_num():\ntests/bufr/test_bufr_summary.py:76:    f = from_source(\"file\", earthkit_examples_file(\"temp_10.bufr\"))\ntests/bufr/test_bufr_summary.py:102:def test_bufr_tail_num():\ntests/bufr/test_bufr_summary.py:103:    f = from_source(\"file\", earthkit_examples_file(\"temp_10.bufr\"))\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "tests/bufr/test_bufr_summary.py",
  "content": "#!/usr/bin/env python3\n\n# (C) Copyright 2022- ECMWF.\n#\n# This software is licensed under the terms of the Apache Licence Version 2.0\n# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.\n# In applying this licence, ECMWF does not waive the privileges and immunities\n# granted to it by virtue of its status as an intergovernmental organisation\n# nor does it submit to any jurisdiction.\n\nimport pytest\n\nfrom earthkit.data import from_source\nfrom earthkit.data.testing import earthkit_examples_file\n\n\ndef test_bufr_ls_invalid_num():\n    f = from_source(\"file\", earthkit_examples_file(\"temp_10.bufr\"))\n    with pytest.raises(ValueError):\n        f.ls(n=0, print=False)\n\n    with pytest.raises(ValueError):\n        f.ls(0, print=False)\n\n\ndef test_bufr_ls_num():\n    f = from_source(\"file\", earthkit_examples_file(\"temp_10.bufr\"))\n\n    # default keys\n\n    # head\n    df = f.ls(n=2, print=False)\n    ref = {\n        \"edition\": {0: 3, 1: 3},\n        \"type\": {0: 2, 1: 2},\n        \"subtype\": {0: 101, 1: 101},\n        \"c\": {0: 98, 1: 98},\n        \"mv\": {0: 13, 1: 13},\n        \"lv\": {0: 1, 1: 1},\n        \"subsets\": {0: 1, 1: 1},\n        \"compr\": {0: 0, 1: 0},\n        \"typicalDate\": {0: \"20081208\", 1: \"20081208\"},\n        \"typicalTime\": {0: \"120000\", 1: \"120000\"},\n        \"ident\": {0: \"02836\", 1: \"01400\"},\n        \"lat\": {0: 67.37, 1: 56.9},\n        \"lon\": {0: 26.63, 1: 3.35},\n    }\n\n    assert ref == df.to_dict()\n\n    # t tail\n    df = f.ls(-2, print=False)\n    ref = {\n        \"edition\": {0: 3, 1: 3},\n        \"type\": {0: 2, 1: 2},\n        \"subtype\": {0: 101, 1: 101},\n        \"c\": {0: 98, 1: 98},\n        \"mv\": {0: 13, 1: 13},\n        \"lv\": {0: 1, 1: 1},\n        \"subsets\": {0: 1, 1: 1},\n        \"compr\": {0: 0, 1: 0},\n        \"typicalDate\": {0: \"20081208\", 1: \"20081208\"},\n        \"typicalTime\": {0: \"120000\", 1: \"120000\"},\n        \"ident\": {0: \"11035\", 1: \"02963\"},\n        \"lat\": {0: 48.25, 1: 60.82},\n        \"lon\": {0: 16.37, 1: 23.5},\n    }\n\n    assert ref == df.to_dict()\n\n    df = f.ls(-2, print=False)\n    assert ref == df.to_dict()\n\n\ndef test_bufr_head_num():\n    f = from_source(\"file\", earthkit_examples_file(\"temp_10.bufr\"))\n\n    # default keys\n    df = f.head(n=2, print=False)\n    ref = {\n        \"edition\": {0: 3, 1: 3},\n        \"type\": {0: 2, 1: 2},\n        \"subtype\": {0: 101, 1: 101},\n        \"c\": {0: 98, 1: 98},\n        \"mv\": {0: 13, 1: 13},\n        \"lv\": {0: 1, 1: 1},\n        \"subsets\": {0: 1, 1: 1},\n        \"compr\": {0: 0, 1: 0},\n        \"typicalDate\": {0: \"20081208\", 1: \"20081208\"},\n        \"typicalTime\": {0: \"120000\", 1: \"120000\"},\n        \"ident\": {0: \"02836\", 1: \"01400\"},\n        \"lat\": {0: 67.37, 1: 56.9},\n        \"lon\": {0: 26.63, 1: 3.35},\n    }\n\n    assert ref == df.to_dict()\n\n    df = f.head(2, print=False)\n    assert ref == df.to_dict()\n\n\ndef test_bufr_tail_num():\n    f = from_source(\"file\", earthkit_examples_file(\"temp_10.bufr\"))\n\n    # default keys\n    df = f.tail(n=2, print=False)\n    ref = {\n        \"edition\": {0: 3, 1: 3},\n        \"type\": {0: 2, 1: 2},\n        \"subtype\": {0: 101, 1: 101},\n        \"c\": {0: 98, 1: 98},\n        \"mv\": {0: 13, 1: 13},\n        \"lv\": {0: 1, 1: 1},\n        \"subsets\": {0: 1, 1: 1},\n        \"compr\": {0: 0, 1: 0},\n        \"typicalDate\": {0: \"20081208\", 1: \"20081208\"},\n        \"typicalTime\": {0: \"120000\", 1: \"120000\"},\n        \"ident\": {0: \"11035\", 1: \"02963\"},\n        \"lat\": {0: 48.25, 1: 60.82},\n        \"lon\": {0: 16.37, 1: 23.5},\n    }\n\n    assert ref == df.to_dict()\n\n    df = f.tail(2, print=False)\n    assert ref == df.to_dict()",
  "start_line": 1,
  "end_line": 126,
  "total_lines": 126,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "Success!\n   latitude  longitude\n0     67.37      26.63\n1     56.90       3.35\n2     58.87       5.67\n3     70.93      -8.67\n4     67.28      14.45\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": " M earthkit/data/readers/bufr.py\ndiff --git a/earthkit/data/readers/bufr.py b/earthkit/data/readers/bufr.py\nindex 59c7672..02882e4 100644\n--- a/earthkit/data/readers/bufr.py\n+++ b/earthkit/data/readers/bufr.py\n@@ -62,6 +62,9 @@ class BUFRReader(Reader):\n         \"\"\"\n         import pdbufr\n \n+        if filters is None:\n+            filters = {}\n+\n         return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n \n     def __len__(self):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "patch_size": 420,
  "files_changed": 1
}