ecmwf__earthkit-data-111

{
  "status": "ok",
  "stdout": "earthkit/data/core/statistics.py:71:    def to_pandas(self):\nearthkit/data/core/__init__.py:47:    def to_pandas(self, **kwargs):\nearthkit/data/mergers/__init__.py:74:    def to_pandas(self, **kwargs):\nearthkit/data/mergers/__init__.py:113:    def to_pandas(self, *args, **kwargs):\nearthkit/data/sources/multi.py:100:    def to_pandas(self, **kwargs):\nearthkit/data/sources/cds.py:130:    def to_pandas(self, **kwargs):\nearthkit/data/sources/memory.py:41:    def to_pandas(self, **kwargs):\nearthkit/data/sources/ecmwf_api.py:99:    def to_pandas(self, **kwargs):\nearthkit/data/sources/file.py:105:    def to_pandas(self, **kwargs):\nearthkit/data/wrappers/xarray.py:71:    def to_pandas(self, *args, **kwargs):\nearthkit/data/readers/bufr.py:40:    def to_pandas(self, columns=COLUMNS, filters=None, **kwargs):\nearthkit/data/readers/grib/pandas.py:16:    def to_pandas(self, latitude=None, longitude=None, **kwargs):\nearthkit/data/readers/netcdf.py:191:    def to_pandas(self):\nearthkit/data/readers/netcdf.py:377:    def to_pandas(self):\nearthkit/data/readers/odb.py:18:    def to_pandas(self, **kwargs):\nearthkit/data/readers/csv.py:135:    def to_pandas(self, **kwargs):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "earthkit/data/readers/bufr.py",
  "content": "# (C) Copyright 2020 ECMWF.\n#\n# This software is licensed under the terms of the Apache Licence Version 2.0\n# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.\n# In applying this licence, ECMWF does not waive the privileges and immunities\n# granted to it by virtue of its status as an intergovernmental organisation\n# nor does it submit to any jurisdiction.\n#\n\n# See:\n# https://github.com/ecmwf/pdbufr\nfrom . import Reader\n\nCOLUMNS = (\"latitude\", \"longitude\", \"data_datetime\")\n\nBUFR_LS_KEYS = {\n    \"edition\": \"edition\",\n    \"type\": \"dataCategory\",\n    \"subtype\": \"dataSubCategory\",\n    \"c\": \"bufrHeaderCentre\",\n    \"mv\": \"masterTablesVersionNumber\",\n    \"lv\": \"localTablesVersionNumber\",\n    \"subsets\": \"numberOfSubsets\",\n    \"compr\": \"compressedData\",\n    \"typicalDate\": \"typicalDate\",\n    \"typicalTime\": \"typicalTime\",\n    \"ident\": \"ident\",\n    \"lat\": \"localLatitude\",\n    \"lon\": \"localLongitude\",\n}\n\n\nclass BUFRReader(Reader):\n    \"\"\"Represents a BUFR file\"\"\"\n\n    def __init__(self, source, path):\n        super().__init__(source, path)\n        self._num = None\n\n    def to_pandas(self, columns=COLUMNS, filters=None, **kwargs):\n        \"\"\"Extracts BUFR data into an pandas DataFranme using :xref:`pdbufr`.\n\n        Parameters\n        ----------\n        columns: str, sequence[str]\n            List of ecCodes BUFR keys to extract for each BUFR message/subset.\n            See: :xref:`read_bufr` for details.\n        filters: dict\n            Defines the conditions when to extract the specified ``columns``. See:\n            :xref:`read_bufr` for details.\n        **kwargs: dict, optional\n            Other keyword arguments:\n\n        Returns\n        -------\n        Pandas DataFrame\n\n        Examples\n        --------\n        :ref:`/examples/bufr.ipynb`\n\n        \"\"\"\n        import pdbufr\n\n        return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n\n    def __len__(self):\n        if self._num is None:\n            import eccodes\n\n            with open(self.path, \"rb\") as f:\n                self._num = eccodes.codes_count_in_file(f)\n        return self._num\n\n    def ls(self, *args, **kwargs):\n        from pdbufr.high_level_bufr.bufr import BufrFile\n\n        from earthkit.data.utils.summary import ls\n\n        b = BufrFile(self.path)\n\n        def _proc(keys, n):\n            count_start = 0\n            if n is None:\n                count_end = len(self)\n            elif n > 0:\n                count_end = n\n            else:\n                num = len(self)\n                count_start = max(0, num + n)\n                count_end = num\n\n            for count, msg in enumerate(b):\n                with msg:\n                    if count_start <= count < count_end:\n                        yield ({k: msg[k1] for k, k1 in keys.items()})\n                    elif count >= count_end:\n                        break\n\n        return ls(_proc, BUFR_LS_KEYS, *args, **kwargs)",
  "start_line": 1,
  "end_line": 100,
  "total_lines": 105,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "earthkit/data/readers/bufr.py:11:# https://github.com/ecmwf/pdbufr\nearthkit/data/readers/bufr.py:41:        \"\"\"Extracts BUFR data into an pandas DataFranme using :xref:`pdbufr`.\nearthkit/data/readers/bufr.py:63:        import pdbufr\nearthkit/data/readers/bufr.py:65:        return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\nearthkit/data/readers/bufr.py:76:        from pdbufr.high_level_bufr.bufr import BufrFile\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "",
  "details": {
    "stdout": "",
    "stderr": "",
    "exit_code": 1
  }
}

{
  "status": "ok",
  "stdout": "tests/grib/test_grib_convert.py:29:def test_grib_to_pandas():\ntests/grib/test_grib_convert.py:33:    df = f.to_pandas()\ntests/grib/test_grib_convert.py:57:    df = f.to_pandas(latitude=90, longitude=30)\ntests/readers/test_csv_reader.py:31:    print(s.to_pandas())\ntests/readers/test_csv_reader.py:46:    print(s.to_pandas())\ntests/readers/test_csv_reader.py:61:    print(s.to_pandas())\ntests/readers/test_csv_reader.py:77:    print(s.to_pandas())\ntests/readers/test_csv_reader.py:113:    print(s.to_pandas())\ntests/readers/test_odb_reader.py:17:def test_odb_to_pandas():\ntests/readers/test_odb_reader.py:19:    df = ds.to_pandas()\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "earthkit/data/readers/grib/pandas.py",
  "content": "# (C) Copyright 2020 ECMWF.\n#\n# This software is licensed under the terms of the Apache Licence Version 2.0\n# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.\n# In applying this licence, ECMWF does not waive the privileges and immunities\n# granted to it by virtue of its status as an intergovernmental organisation\n# nor does it submit to any jurisdiction.\n#\n\nimport logging\n\nLOG = logging.getLogger(__name__)\n\n\nclass PandasMixIn:\n    def to_pandas(self, latitude=None, longitude=None, **kwargs):\n        import numpy as np\n        import pandas as pd\n\n        def ident(x):\n            return x\n\n        filter = ident\n\n        def select_point(d):\n            idx = np.where((d[\"lat\"] == latitude) & (d[\"lon\"] == longitude))\n            return dict(lat=d[\"lat\"][idx], lon=d[\"lon\"][idx], value=d[\"value\"][idx])\n\n        if latitude is not None or longitude is not None:\n            filter = select_point\n\n        columns = (\"lat\", \"lon\", \"value\")\n        frames = []\n        for s in self:\n            df = pd.DataFrame.from_dict(\n                filter(dict(zip(columns, s.data(columns, flatten=True))))\n            )\n            df[\"datetime\"] = s.datetime()[\"valid_time\"]\n            for k, v in s.metadata(namespace=\"mars\").items():\n                df[k] = v\n            frames.append(df)\n        return pd.concat(frames)",
  "start_line": 1,
  "end_line": 42,
  "total_lines": 42,
  "is_truncated": false
}

{
  "status": "ok",
  "filepath": "earthkit/data/readers/bufr.py",
  "occurrences": 1,
  "strategy": "exact",
  "diff": "--- a/earthkit/data/readers/bufr.py\n+++ b/earthkit/data/readers/bufr.py\n@@ -62,6 +62,8 @@\n         \"\"\"\n         import pdbufr\n \n+        if filters is None:\n+            filters = {}\n         return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n \n     def __len__(self):\n",
  "is_truncated": false
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "EE\n======================================================================\nERROR: test_to_pandas_filters_none (__main__.TestBUFRReader)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n  File \"/usr/local/lib/python3.10/unittest/mock.py\", line 1379, in patched\n    return func(*newargs, **newkeywargs)\n  File \"<stdin>\", line 8, in test_to_pandas_filters_none\n  File \"/workspace/earthkit/data/core/__init__.py\", line 21, in __call__\n    obj.__init__(*args, **kwargs)\n  File \"/workspace/earthkit/data/readers/bufr.py\", line 37, in __init__\n    super().__init__(source, path)\n  File \"/workspace/earthkit/data/readers/__init__.py\", line 32, in __init__\n    self._source = weakref.ref(source)\nTypeError: cannot create weak reference to 'NoneType' object\n\n======================================================================\nERROR: test_to_pandas_filters_provided (__main__.TestBUFRReader)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n  File \"/usr/local/lib/python3.10/unittest/mock.py\", line 1379, in patched\n    return func(*newargs, **newkeywargs)\n  File \"<stdin>\", line 18, in test_to_pandas_filters_provided\n  File \"/workspace/earthkit/data/core/__init__.py\", line 21, in __call__\n    obj.__init__(*args, **kwargs)\n  File \"/workspace/earthkit/data/readers/bufr.py\", line 37, in __init__\n    super().__init__(source, path)\n  File \"/workspace/earthkit/data/readers/__init__.py\", line 32, in __init__\n    self._source = weakref.ref(source)\nTypeError: cannot create weak reference to 'NoneType' object\n\n----------------------------------------------------------------------\nRan 2 tests in 0.122s\n\nFAILED (errors=2)\n",
  "details": {
    "stdout": "",
    "stderr": "EE\n======================================================================\nERROR: test_to_pandas_filters_none (__main__.TestBUFRReader)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n  File \"/usr/local/lib/python3.10/unittest/mock.py\", line 1379, in patched\n    return func(*newargs, **newkeywargs)\n  File \"<stdin>\", line 8, in test_to_pandas_filters_none\n  File \"/workspace/earthkit/data/core/__init__.py\", line 21, in __call__\n    obj.__init__(*args, **kwargs)\n  File \"/workspace/earthkit/data/readers/bufr.py\", line 37, in __init__\n    super().__init__(source, path)\n  File \"/workspace/earthkit/data/readers/__init__.py\", line 32, in __init__\n    self._source = weakref.ref(source)\nTypeError: cannot create weak reference to 'NoneType' object\n\n======================================================================\nERROR: test_to_pandas_filters_provided (__main__.TestBUFRReader)\n----------------------------------------------------------------------\nTraceback (most recent call last):\n  File \"/usr/local/lib/python3.10/unittest/mock.py\", line 1379, in patched\n    return func(*newargs, **newkeywargs)\n  File \"<stdin>\", line 18, in test_to_pandas_filters_provided\n  File \"/workspace/earthkit/data/core/__init__.py\", line 21, in __call__\n    obj.__init__(*args, **kwargs)\n  File \"/workspace/earthkit/data/readers/bufr.py\", line 37, in __init__\n    super().__init__(source, path)\n  File \"/workspace/earthkit/data/readers/__init__.py\", line 32, in __init__\n    self._source = weakref.ref(source)\nTypeError: cannot create weak reference to 'NoneType' object\n\n----------------------------------------------------------------------\nRan 2 tests in 0.122s\n\nFAILED (errors=2)\n",
    "exit_code": 1
  }
}

{
  "status": "ok",
  "stdout": "",
  "stderr": "..\n----------------------------------------------------------------------\nRan 2 tests in 0.060s\n\nOK\n",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": " M earthkit/data/readers/bufr.py\ndiff --git a/earthkit/data/readers/bufr.py b/earthkit/data/readers/bufr.py\nindex 59c7672..14dcc4d 100644\n--- a/earthkit/data/readers/bufr.py\n+++ b/earthkit/data/readers/bufr.py\n@@ -62,6 +62,8 @@ class BUFRReader(Reader):\n         \"\"\"\n         import pdbufr\n \n+        if filters is None:\n+            filters = {}\n         return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n \n     def __len__(self):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "patch_size": 418,
  "files_changed": 1
}