ecmwf__earthkit-data-111

{
  "status": "ok",
  "stdout": "earthkit/data/core/statistics.py:71:    def to_pandas(self):\nearthkit/data/core/__init__.py:47:    def to_pandas(self, **kwargs):\nearthkit/data/mergers/__init__.py:74:    def to_pandas(self, **kwargs):\nearthkit/data/mergers/__init__.py:113:    def to_pandas(self, *args, **kwargs):\nearthkit/data/sources/multi.py:100:    def to_pandas(self, **kwargs):\nearthkit/data/sources/cds.py:130:    def to_pandas(self, **kwargs):\nearthkit/data/sources/memory.py:41:    def to_pandas(self, **kwargs):\nearthkit/data/sources/ecmwf_api.py:99:    def to_pandas(self, **kwargs):\nearthkit/data/sources/file.py:105:    def to_pandas(self, **kwargs):\nearthkit/data/wrappers/xarray.py:71:    def to_pandas(self, *args, **kwargs):\nearthkit/data/readers/bufr.py:40:    def to_pandas(self, columns=COLUMNS, filters=None, **kwargs):\nearthkit/data/readers/grib/pandas.py:16:    def to_pandas(self, latitude=None, longitude=None, **kwargs):\nearthkit/data/readers/netcdf.py:191:    def to_pandas(self):\nearthkit/data/readers/netcdf.py:377:    def to_pandas(self):\nearthkit/data/readers/odb.py:18:    def to_pandas(self, **kwargs):\nearthkit/data/readers/csv.py:135:    def to_pandas(self, **kwargs):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "earthkit/data/readers/bufr.py",
  "content": "# (C) Copyright 2020 ECMWF.\n#\n# This software is licensed under the terms of the Apache Licence Version 2.0\n# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.\n# In applying this licence, ECMWF does not waive the privileges and immunities\n# granted to it by virtue of its status as an intergovernmental organisation\n# nor does it submit to any jurisdiction.\n#\n\n# See:\n# https://github.com/ecmwf/pdbufr\nfrom . import Reader\n\nCOLUMNS = (\"latitude\", \"longitude\", \"data_datetime\")\n\nBUFR_LS_KEYS = {\n    \"edition\": \"edition\",\n    \"type\": \"dataCategory\",\n    \"subtype\": \"dataSubCategory\",\n    \"c\": \"bufrHeaderCentre\",\n    \"mv\": \"masterTablesVersionNumber\",\n    \"lv\": \"localTablesVersionNumber\",\n    \"subsets\": \"numberOfSubsets\",\n    \"compr\": \"compressedData\",\n    \"typicalDate\": \"typicalDate\",\n    \"typicalTime\": \"typicalTime\",\n    \"ident\": \"ident\",\n    \"lat\": \"localLatitude\",\n    \"lon\": \"localLongitude\",\n}\n\n\nclass BUFRReader(Reader):\n    \"\"\"Represents a BUFR file\"\"\"\n\n    def __init__(self, source, path):\n        super().__init__(source, path)\n        self._num = None\n\n    def to_pandas(self, columns=COLUMNS, filters=None, **kwargs):\n        \"\"\"Extracts BUFR data into an pandas DataFranme using :xref:`pdbufr`.\n\n        Parameters\n        ----------\n        columns: str, sequence[str]\n            List of ecCodes BUFR keys to extract for each BUFR message/subset.\n            See: :xref:`read_bufr` for details.\n        filters: dict\n            Defines the conditions when to extract the specified ``columns``. See:\n            :xref:`read_bufr` for details.\n        **kwargs: dict, optional\n            Other keyword arguments:\n\n        Returns\n        -------\n        Pandas DataFrame\n\n        Examples\n        --------\n        :ref:`/examples/bufr.ipynb`\n\n        \"\"\"\n        import pdbufr\n\n        return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n\n    def __len__(self):\n        if self._num is None:\n            import eccodes\n\n            with open(self.path, \"rb\") as f:\n                self._num = eccodes.codes_count_in_file(f)\n        return self._num\n\n    def ls(self, *args, **kwargs):\n        from pdbufr.high_level_bufr.bufr import BufrFile\n\n        from earthkit.data.utils.summary import ls\n\n        b = BufrFile(self.path)\n\n        def _proc(keys, n):\n            count_start = 0\n            if n is None:\n                count_end = len(self)\n            elif n > 0:\n                count_end = n\n            else:\n                num = len(self)\n                count_start = max(0, num + n)\n                count_end = num\n\n            for count, msg in enumerate(b):\n                with msg:\n                    if count_start <= count < count_end:\n                        yield ({k: msg[k1] for k, k1 in keys.items()})\n                    elif count >= count_end:\n                        break\n\n        return ls(_proc, BUFR_LS_KEYS, *args, **kwargs)\n\n\ndef reader(source, path, magic=None, deeper_check=False):\n    if magic is None or magic[:4] == b\"BUFR\":\n        return BUFRReader(source, path)",
  "start_line": 1,
  "end_line": 105,
  "total_lines": 105,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "earthkit/data/readers/bufr.py:11:# https://github.com/ecmwf/pdbufr\nearthkit/data/readers/bufr.py:41:        \"\"\"Extracts BUFR data into an pandas DataFranme using :xref:`pdbufr`.\nearthkit/data/readers/bufr.py:63:        import pdbufr\nearthkit/data/readers/bufr.py:65:        return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\nearthkit/data/readers/bufr.py:76:        from pdbufr.high_level_bufr.bufr import BufrFile\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "earthkit/data/sources/file.py",
  "content": "# (C) Copyright 2020 ECMWF.\n#\n# This software is licensed under the terms of the Apache Licence Version 2.0\n# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.\n# In applying this licence, ECMWF does not waive the privileges and immunities\n# granted to it by virtue of its status as an intergovernmental organisation\n# nor does it submit to any jurisdiction.\n#\n\n\nimport glob\nimport logging\nimport os\n\nfrom earthkit.data import from_source\nfrom earthkit.data.core.settings import SETTINGS\nfrom earthkit.data.readers import reader\nfrom earthkit.data.sources.file_indexed import FileIndexedSource\n\nfrom . import Source\n\nLOG = logging.getLogger(__name__)\n\n\nclass FileSourceMeta(type(Source), type(os.PathLike)):\n    def patch(cls, obj, *args, **kwargs):\n        if \"reader\" in kwargs:\n            setattr(obj, \"reader\", kwargs.pop(\"reader\"))\n\n        return super().patch(obj, *args, **kwargs)\n\n\nclass FileSource(Source, os.PathLike, metaclass=FileSourceMeta):\n    _reader_ = None\n\n    def __init__(self, path=None, filter=None, merger=None, **kwargs):\n        Source.__init__(self, **kwargs)\n        self.path = path\n        self.filter = filter\n        self.merger = merger\n\n    def mutate(self):\n        if isinstance(self.path, (list, tuple)):\n            if len(self.path) == 1:\n                self.path = self.path[0]\n            else:\n                return from_source(\n                    \"multi\",\n                    [from_source(\"file\", p, **self._kwargs) for p in self.path],\n                    filter=self.filter,\n                    merger=self.merger,\n                )\n\n        # here we must have a file or a directory\n        if self._kwargs.get(\"indexing\", False):\n            kw = dict(self._kwargs)\n            kw.pop(\"indexing\", None)\n            return FileIndexedSource(self.path, filter=filter, merger=self.merger, **kw)\n\n        # Give a chance to directories and zip files\n        # to return a multi-source\n        source = self._reader.mutate_source()\n        if source not in (None, self):\n            source._parent = self\n            return source\n        return self\n\n    def ignore(self):\n        return self._reader.ignore()\n\n    @classmethod\n    def merge(cls, sources):\n        from earthkit.data.mergers import merge_by_class\n\n        assert all(isinstance(s, FileSource) for s in sources)\n        return merge_by_class([s._reader for s in sources])\n\n    @property\n    def _reader(self):\n        if self._reader_ is None:\n            self._reader_ = reader(self, self.path)\n        return self._reader_\n\n    def __iter__(self):\n        return iter(self._reader)\n\n    def __len__(self):\n        return len(self._reader)\n\n    def __getitem__(self, n):\n        return self._reader[n]\n\n    def sel(self, *args, **kwargs):\n        return self._reader.sel(*args, **kwargs)\n\n    def isel(self, *args, **kwargs):\n        return self._reader.isel(*args, **kwargs)\n\n    def order_by(self, *args, **kwargs):\n        return self._reader.order_by(*args, **kwargs)\n\n    def to_xarray(self, **kwargs):\n        return self._reader.to_xarray(**kwargs)\n\n    def to_pandas(self, **kwargs):\n        LOG.debug(\"Calling reader.to_pandas %s\", self)\n        return self._reader.to_pandas(**kwargs)\n\n    def to_numpy(self, **kwargs):\n        return self._reader.to_numpy(**kwargs)\n\n    @property\n    def values(self):\n        return self._reader.values\n\n    def save(self, path):\n        return self._reader.save(path)\n\n    def write(self, f):\n        return self._reader.write(f)\n\n    def scaled(self, *args, **kwargs):\n        return self._reader.scaled(*args, **kwargs)\n\n    def _attributes(self, names):\n        return self._reader._attributes(names)\n\n    def __repr__(self):\n        cache_dir = SETTINGS.get(\"cache-directory\")\n        path = getattr(self, \"path\", None)\n        if isinstance(path, str):\n            path = path.replace(cache_dir, \"CACHE:\")\n        try:\n            reader_class_name = str(self._reader.__class__.__name__)\n        except AttributeError as e:\n            reader_class_name = str(e)\n        except:  # noqa: E722\n            reader_class_name = \"Unknown\"\n        return f\"{self.__class__.__name__}({path},{reader_class_name})\"\n\n    def __fspath__(self):\n        return self.path\n\n    def metadata(self, *args, **kwargs):\n        return self._reader.metadata(*args, **kwargs)\n\n    def indices(self, *args, **kwargs):\n        return self._reader.indices(*args, **kwargs)\n\n    def index(self, *args, **kwargs):",
  "start_line": 1,
  "end_line": 150,
  "total_lines": 213,
  "is_truncated": true
}

{
  "status": "ok",
  "stdout": "",
  "stderr": ".\n----------------------------------------------------------------------\nRan 1 test in 0.126s\n\nOK\n",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "earthkit/data/readers/bufr.py",
  "occurrences": 1,
  "strategy": "exact",
  "diff": "--- a/earthkit/data/readers/bufr.py\n+++ b/earthkit/data/readers/bufr.py\n@@ -62,6 +62,9 @@\n         \"\"\"\n         import pdbufr\n \n+        if filters is None:\n+            filters = {}\n+\n         return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n \n     def __len__(self):\n",
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "",
  "stderr": ".\n----------------------------------------------------------------------\nRan 1 test in 0.061s\n\nOK\n",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "earthkit/data/readers/grib/pandas.py",
  "content": "# (C) Copyright 2020 ECMWF.\n#\n# This software is licensed under the terms of the Apache Licence Version 2.0\n# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.\n# In applying this licence, ECMWF does not waive the privileges and immunities\n# granted to it by virtue of its status as an intergovernmental organisation\n# nor does it submit to any jurisdiction.\n#\n\nimport logging\n\nLOG = logging.getLogger(__name__)\n\n\nclass PandasMixIn:\n    def to_pandas(self, latitude=None, longitude=None, **kwargs):\n        import numpy as np\n        import pandas as pd\n\n        def ident(x):\n            return x\n\n        filter = ident\n\n        def select_point(d):\n            idx = np.where((d[\"lat\"] == latitude) & (d[\"lon\"] == longitude))\n            return dict(lat=d[\"lat\"][idx], lon=d[\"lon\"][idx], value=d[\"value\"][idx])\n\n        if latitude is not None or longitude is not None:\n            filter = select_point\n\n        columns = (\"lat\", \"lon\", \"value\")\n        frames = []\n        for s in self:\n            df = pd.DataFrame.from_dict(\n                filter(dict(zip(columns, s.data(columns, flatten=True))))\n            )\n            df[\"datetime\"] = s.datetime()[\"valid_time\"]\n            for k, v in s.metadata(namespace=\"mars\").items():\n                df[k] = v\n            frames.append(df)\n        return pd.concat(frames)",
  "start_line": 1,
  "end_line": 42,
  "total_lines": 42,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "earthkit/data/readers/bufr.py:68:        return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": " M earthkit/data/readers/bufr.py\ndiff --git a/earthkit/data/readers/bufr.py b/earthkit/data/readers/bufr.py\nindex 59c7672..02882e4 100644\n--- a/earthkit/data/readers/bufr.py\n+++ b/earthkit/data/readers/bufr.py\n@@ -62,6 +62,9 @@ class BUFRReader(Reader):\n         \"\"\"\n         import pdbufr\n \n+        if filters is None:\n+            filters = {}\n+\n         return pdbufr.read_bufr(self.path, columns=columns, filters=filters)\n \n     def __len__(self):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "patch_size": 420,
  "files_changed": 1
}