ecmwf__earthkit-data-111

{
  "status": "ok",
  "stdout": "earthkit/data/core/statistics.py:71:    def to_pandas(self):\nearthkit/data/core/__init__.py:47:    def to_pandas(self, **kwargs):\nearthkit/data/mergers/pandas.py:21:    return pd.concat([s.to_pandas(**kwargs) for s in sources], **options)\nearthkit/data/mergers/__init__.py:17:FORWARDS = (\"to_xarray\", \"to_pandas\", \"to_tfdataset\")\nearthkit/data/mergers/__init__.py:74:    def to_pandas(self, **kwargs):\nearthkit/data/mergers/__init__.py:113:    def to_pandas(self, *args, **kwargs):\nearthkit/data/mergers/__init__.py:114:        return self.obj.to_pandas(self.paths_or_sources, **kwargs)\nearthkit/data/mergers/__init__.py:129:    to_pandas = _call_func\nearthkit/data/sources/multi.py:100:    def to_pandas(self, **kwargs):\nearthkit/data/sources/multi.py:101:        return make_merger(self.merger, self.sources).to_pandas(**kwargs)\nearthkit/data/sources/cds.py:130:    def to_pandas(self, **kwargs):\nearthkit/data/sources/cds.py:146:        return super().to_pandas(\nearthkit/data/sources/memory.py:41:    def to_pandas(self, **kwargs):\nearthkit/data/sources/memory.py:42:        return self._reader.to_pandas(**kwargs)\nearthkit/data/sources/ecmwf_api.py:99:    def to_pandas(self, **kwargs):\nearthkit/data/sources/ecmwf_api.py:117:        return super().to_pandas(\nearthkit/data/sources/file.py:105:    def to_pandas(self, **kwargs):\nearthkit/data/sources/file.py:106:        LOG.debug(\"Calling reader.to_pandas %s\", self)\nearthkit/data/sources/file.py:107:        return self._reader.to_pandas(**kwargs)\nearthkit/data/wrappers/xarray.py:71:    def to_pandas(self, *args, **kwargs):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "earthkit/data/readers/bufr.py:11:# https://github.com/ecmwf/pdbufr\nearthkit/data/readers/bufr.py:20:    \"c\": \"bufrHeaderCentre\",\nearthkit/data/readers/bufr.py:41:        \"\"\"Extracts BUFR data into an pandas DataFranme using :xref:`pdbufr`.\nearthkit/data/readers/bufr.py:47:            See: :xref:`read_bufr` for details.\nearthkit/data/readers/bufr.py:50:            :xref:`read_bufr` for details.\nearthkit/data/readers/bufr.py:60:        :ref:`/examples/bufr.ipynb`\nearthkit/data/readers/bufr.py:63:        import pdbufr\nearthkit/data/readers/bufr.py:65:        return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\nearthkit/data/readers/bufr.py:76:        from pdbufr.high_level_bufr.bufr import BufrFile\nearthkit/data/readers/__init__.py:115:def _find_reader(method_name, source, path_or_bufr_or_stream, magic):\nearthkit/data/readers/__init__.py:124:            reader = r(source, path_or_bufr_or_stream, magic, deeper_check)\nearthkit/data/readers/__init__.py:131:        source, path_or_bufr_or_stream if method_name == \"reader\" else \"\", magic\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "earthkit/data/readers/bufr.py",
  "content": "# (C) Copyright 2020 ECMWF.\n#\n# This software is licensed under the terms of the Apache Licence Version 2.0\n# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.\n# In applying this licence, ECMWF does not waive the privileges and immunities\n# granted to it by virtue of its status as an intergovernmental organisation\n# nor does it submit to any jurisdiction.\n#\n\n# See:\n# https://github.com/ecmwf/pdbufr\nfrom . import Reader\n\nCOLUMNS = (\"latitude\", \"longitude\", \"data_datetime\")\n\nBUFR_LS_KEYS = {\n    \"edition\": \"edition\",\n    \"type\": \"dataCategory\",\n    \"subtype\": \"dataSubCategory\",\n    \"c\": \"bufrHeaderCentre\",\n    \"mv\": \"masterTablesVersionNumber\",\n    \"lv\": \"localTablesVersionNumber\",\n    \"subsets\": \"numberOfSubsets\",\n    \"compr\": \"compressedData\",\n    \"typicalDate\": \"typicalDate\",\n    \"typicalTime\": \"typicalTime\",\n    \"ident\": \"ident\",\n    \"lat\": \"localLatitude\",\n    \"lon\": \"localLongitude\",\n}\n\n\nclass BUFRReader(Reader):\n    \"\"\"Represents a BUFR file\"\"\"\n\n    def __init__(self, source, path):\n        super().__init__(source, path)\n        self._num = None\n\n    def to_pandas(self, columns=COLUMNS, filters=None, **kwargs):\n        \"\"\"Extracts BUFR data into an pandas DataFranme using :xref:`pdbufr`.\n\n        Parameters\n        ----------\n        columns: str, sequence[str]\n            List of ecCodes BUFR keys to extract for each BUFR message/subset.\n            See: :xref:`read_bufr` for details.\n        filters: dict\n            Defines the conditions when to extract the specified ``columns``. See:\n            :xref:`read_bufr` for details.\n        **kwargs: dict, optional\n            Other keyword arguments:\n\n        Returns\n        -------\n        Pandas DataFrame\n\n        Examples\n        --------\n        :ref:`/examples/bufr.ipynb`\n\n        \"\"\"\n        import pdbufr\n\n        return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n\n    def __len__(self):\n        if self._num is None:\n            import eccodes\n\n            with open(self.path, \"rb\") as f:\n                self._num = eccodes.codes_count_in_file(f)\n        return self._num\n\n    def ls(self, *args, **kwargs):\n        from pdbufr.high_level_bufr.bufr import BufrFile\n\n        from earthkit.data.utils.summary import ls\n\n        b = BufrFile(self.path)\n\n        def _proc(keys, n):\n            count_start = 0\n            if n is None:\n                count_end = len(self)\n            elif n > 0:\n                count_end = n\n            else:\n                num = len(self)\n                count_start = max(0, num + n)\n                count_end = num\n\n            for count, msg in enumerate(b):\n                with msg:\n                    if count_start <= count < count_end:\n                        yield ({k: msg[k1] for k, k1 in keys.items()})\n                    elif count >= count_end:\n                        break\n\n        return ls(_proc, BUFR_LS_KEYS, *args, **kwargs)\n\n\ndef reader(source, path, magic=None, deeper_check=False):\n    if magic is None or magic[:4] == b\"BUFR\":\n        return BUFRReader(source, path)",
  "start_line": 1,
  "end_line": 105,
  "total_lines": 105,
  "is_truncated": false
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "",
  "details": {
    "stdout": "",
    "stderr": "",
    "exit_code": 1
  }
}

{
  "status": "ok",
  "stdout": "tests/bufr\ntests/bufr/test_bufr_summary.py\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "",
  "details": {
    "stdout": "",
    "stderr": "",
    "exit_code": 1
  }
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "Traceback (most recent call last):\n  File \"<stdin>\", line 10, in <module>\n  File \"/workspace/earthkit/data/core/__init__.py\", line 21, in __call__\n    obj.__init__(*args, **kwargs)\n  File \"/workspace/earthkit/data/readers/bufr.py\", line 37, in __init__\n    super().__init__(source, path)\n  File \"/workspace/earthkit/data/readers/__init__.py\", line 32, in __init__\n    self._source = weakref.ref(source)\nTypeError: cannot create weak reference to 'NoneType' object\n",
  "details": {
    "stdout": "",
    "stderr": "Traceback (most recent call last):\n  File \"<stdin>\", line 10, in <module>\n  File \"/workspace/earthkit/data/core/__init__.py\", line 21, in __call__\n    obj.__init__(*args, **kwargs)\n  File \"/workspace/earthkit/data/readers/bufr.py\", line 37, in __init__\n    super().__init__(source, path)\n  File \"/workspace/earthkit/data/readers/__init__.py\", line 32, in __init__\n    self._source = weakref.ref(source)\nTypeError: cannot create weak reference to 'NoneType' object\n",
    "exit_code": 1
  }
}

{
  "status": "ok",
  "stdout": "Arguments: ('dummy.bufr',)\nKwargs: {'columns': ['lat', 'lon'], 'filters': None}\nReproduction successful: filters is None\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "earthkit/data/readers/bufr.py",
  "occurrences": 1,
  "strategy": "exact",
  "diff": "--- a/earthkit/data/readers/bufr.py\n+++ b/earthkit/data/readers/bufr.py\n@@ -62,7 +62,10 @@\n         \"\"\"\n         import pdbufr\n \n-        return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n+        if filters is None:\n+            filters = {}\n+\n+        return pdbufr.read_bufr(self.path, columns=columns, filters=filters, **kwargs)\n \n     def __len__(self):\n         if self._num is None:\n",
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "Test 1 Kwargs: {'columns': ['lat', 'lon'], 'filters': {}}\nTest 1 successful\nTest 2 Kwargs: {'columns': ['lat', 'lon'], 'filters': {}}\nTest 2 successful\nTest 3 Kwargs: {'columns': ['lat', 'lon'], 'filters': {'key': 'value'}}\nTest 3 successful\nTest 4 Kwargs: {'columns': ['lat', 'lon'], 'filters': {}, 'extra_arg': 'extra_value'}\nTest 4 successful\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "__init__.py\narchive.py\nbufr.py\ncsv.py\ndirectory.py\ngrib\nnetcdf.py\nnumpy.py\nodb.py\ntar.py\ntext.py\nunknown.py\nzip.py\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "earthkit/data/readers/bufr.py:40:    def to_pandas(self, columns=COLUMNS, filters=None, **kwargs):\nearthkit/data/readers/grib/pandas.py:16:    def to_pandas(self, latitude=None, longitude=None, **kwargs):\nearthkit/data/readers/netcdf.py:191:    def to_pandas(self):\nearthkit/data/readers/netcdf.py:377:    def to_pandas(self):\nearthkit/data/readers/odb.py:18:    def to_pandas(self, **kwargs):\nearthkit/data/readers/csv.py:135:    def to_pandas(self, **kwargs):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "earthkit/data/readers/grib/pandas.py",
  "content": "import logging\n\nLOG = logging.getLogger(__name__)\n\n\nclass PandasMixIn:\n    def to_pandas(self, latitude=None, longitude=None, **kwargs):\n        import numpy as np\n        import pandas as pd\n\n        def ident(x):\n            return x\n\n        filter = ident\n\n        def select_point(d):\n            idx = np.where((d[\"lat\"] == latitude) & (d[\"lon\"] == longitude))\n            return dict(lat=d[\"lat\"][idx], lon=d[\"lon\"][idx], value=d[\"value\"][idx])\n\n        if latitude is not None or longitude is not None:\n            filter = select_point\n\n        columns = (\"lat\", \"lon\", \"value\")\n        frames = []\n        for s in self:\n            df = pd.DataFrame.from_dict(\n                filter(dict(zip(columns, s.data(columns, flatten=True))))\n            )\n            df[\"datetime\"] = s.datetime()[\"valid_time\"]\n            for k, v in s.metadata(namespace=\"mars\").items():\n                df[k] = v",
  "start_line": 10,
  "end_line": 40,
  "total_lines": 42,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": " M earthkit/data/readers/bufr.py\ndiff --git a/earthkit/data/readers/bufr.py b/earthkit/data/readers/bufr.py\nindex 59c7672..bbf2808 100644\n--- a/earthkit/data/readers/bufr.py\n+++ b/earthkit/data/readers/bufr.py\n@@ -62,7 +62,10 @@ class BUFRReader(Reader):\n         \"\"\"\n         import pdbufr\n \n-        return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n+        if filters is None:\n+            filters = {}\n+\n+        return pdbufr.read_bufr(self.path, columns=columns, filters=filters, **kwargs)\n \n     def __len__(self):\n         if self._num is None:\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "patch_size": 540,
  "files_changed": 1
}