facebookresearch__hydra-273

{
  "status": "ok",
  "stdout": "18:from ..errors import MissingConfigException\n299:                    raise MissingConfigException(msg, new_cfg)\n308:                    raise MissingConfigException(msg, new_cfg, options)\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "hydra/_internal/config_loader.py",
  "content": "                    \"{}/{}\".format(path, group_name)\n                )\n                if self._exists(is_pkg, path) and resource_isdir(\n                    module_name, resource_name\n                ):\n                    all_files = resource_listdir(module_name, resource_name)\n                    files = []\n                    for file in all_files:\n                        if (\n                            file_type == \"dir\"\n                            and resource_isdir(\n                                module_name, os.path.join(group_name, file)\n                            )\n                            and file != \"__pycache__\"\n                        ):\n                            files.append(file)\n                        elif file_type == \"file\" and file.endswith(\".yaml\"):\n                            files.append(file[0 : -len(\".yaml\")])\n            else:\n                group_path = \"{}/{}\".format(path, group_name)\n                if os.path.isdir(group_path):\n                    all_files = os.listdir(group_path)\n                    files = []\n                    for file in all_files:\n                        full_path = os.path.join(group_path, group_name, file)\n                        if (\n                            file_type == \"dir\"\n                            and os.path.isdir(full_path)\n                            and file != \"__pycache__\"\n                        ):\n                            files.append(file)\n                        elif file_type == \"file\" and file.endswith(\".yaml\"):\n                            files.append(file[0 : -len(\".yaml\")])\n\n            options.extend(files)\n        return options\n\n    def _merge_config(self, cfg, family, name, required):\n\n        if family != \"\":\n            new_cfg = \"{}/{}\".format(family, name)\n        else:\n            new_cfg = name\n\n        loaded_cfg = self._load_config_impl(new_cfg)\n        if loaded_cfg is None:\n            if required:\n                if family == \"\":\n                    msg = \"Could not load {}\".format(new_cfg)\n                    raise MissingConfigException(msg, new_cfg)\n                else:\n                    options = self.get_group_options(family)\n                    if options:\n                        msg = \"Could not load {}, available options:\\n{}:\\n\\t{}\".format(\n                            new_cfg, family, \"\\n\\t\".join(options)\n                        )\n                    else:\n                        msg = \"Could not load {}\".format(new_cfg)\n                    raise MissingConfigException(msg, new_cfg, options)\n            else:\n                return cfg\n\n        else:\n            return OmegaConf.merge(cfg, loaded_cfg)\n\n        assert False\n\n    @staticmethod\n    def _split_module_and_resource(filename):\n        sep = filename.find(\"/\")\n        if sep == -1:\n            module_name = filename\n            resource_name = \"\"\n        else:\n            module_name = filename[0:sep]\n            resource_name = filename[sep + 1 :]\n        if module_name == \"\":\n            # if we have a module a module only, dirname would return nothing and basename would return the module.\n            module_name = resource_name\n            resource_name = \"\"\n\n        return module_name, resource_name\n\n    @staticmethod\n    def _exists(is_pkg, filename):\n        if is_pkg:\n            module_name, resource_name = ConfigLoader._split_module_and_resource(\n                filename\n            )\n            try:\n                ret = resource_exists(module_name, resource_name)\n            except ImportError:\n                ret = False\n            except ValueError:  # Python 2.7 throws ValueError empty module name sometimes.\n                ret = False\n            except NotImplementedError:\n                raise NotImplementedError(\n                    \"Unable to load {}/{}, are you missing an __init__.py?\".format(\n                        module_name, resource_name\n                    )\n                )",
  "start_line": 250,
  "end_line": 350,
  "total_lines": 453,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "201:    def _load_config_impl(self, input_file, record_load=True):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "hydra/_internal/config_loader.py",
  "content": "            key1 = next(keys_iter)\n            if key1 == \"optional\":\n                key1 = next(keys_iter)\n            return key1\n\n        key_to_idx = {}\n        for idx, d in enumerate(primary):\n            if isinstance(d, (dict, DictConfig)):\n                key = get_key(d)\n                key_to_idx[key] = idx\n        for d in copy.deepcopy(merged_list):\n            if isinstance(d, (dict, DictConfig)):\n                key = get_key(d)\n                if key in key_to_idx.keys():\n                    idx = key_to_idx[key]\n                    primary[idx] = d\n                    merged_list.remove(d)\n\n        for d in merged_list:\n            primary.append(d)\n\n    def _load_config_impl(self, input_file, record_load=True):\n        \"\"\"\n        :param input_file:\n        :param record_load:\n        :return: the loaded config or None if it was not found\n        \"\"\"\n        loaded_cfg = None\n        filename, search_path = self._find_config(input_file)\n        if search_path is None and record_load:\n            assert search_path is None\n            self.all_config_checked.append((filename, None, None))\n\n        if search_path is not None:\n            fullpath = \"{}/{}\".format(search_path.path, filename)\n            is_pkg = search_path.path.startswith(\"pkg://\")\n            if is_pkg:\n                fullpath = fullpath[len(\"pkg://\") :]\n                module_name, resource_name = ConfigLoader._split_module_and_resource(\n                    fullpath\n                )\n                with resource_stream(module_name, resource_name) as stream:\n                    loaded_cfg = OmegaConf.load(stream)\n                if record_load:\n                    self.all_config_checked.append(\n                        (filename, search_path.path, search_path.provider)\n                    )\n            elif os.path.exists(fullpath):\n                loaded_cfg = OmegaConf.load(fullpath)\n                if record_load:\n                    self.all_config_checked.append(",
  "start_line": 180,
  "end_line": 230,
  "total_lines": 453,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "117:    def _find_config(self, filepath):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "hydra/_internal/config_loader.py",
  "content": "    def _split_path(path):\n        prefix = \"pkg://\"\n        is_pkg = path.startswith(prefix)\n        if is_pkg:\n            path = path[len(prefix) :]\n        return is_pkg, path\n\n    def _find_config(self, filepath):\n        found_search_path = None\n        for search_path in self.config_search_path.config_search_path:\n            is_pkg, path = self._split_path(search_path.path)\n            config_file = \"{}/{}\".format(path, filepath)\n            if self._exists(is_pkg, config_file):\n                found_search_path = search_path\n                break\n        return filepath, found_search_path\n\n    @staticmethod\n    def _split_key_val(s):\n        assert \"=\" in s, \"'{}' not a valid override, expecting key=value format\".format(\n            s\n        )\n\n        idx = s.find(\"=\")\n        assert idx != -1\n        return s[0:idx], s[idx + 1 :]\n\n    @staticmethod\n    def _apply_defaults_overrides(overrides, defaults):\n        consumed = []\n        key_to_idx = {}\n        for idx, d in enumerate(defaults):\n            if isinstance(d, DictConfig):\n                key = next(iter(d.keys()))\n                key_to_idx[key] = idx\n        for override in copy.deepcopy(overrides):\n            key, value = ConfigLoader._split_key_val(override)\n            if key in key_to_idx:\n                if \",\" in value:\n                    # If this is a multirun config (comma separated list), flag the default to prevent it from being\n                    # loaded until we are constructing the config for individual jobs.\n                    value = \"_SKIP_\"\n\n                if value == \"null\":\n                    del defaults[key_to_idx[key]]\n                else:\n                    defaults[key_to_idx[key]][key] = value\n\n                overrides.remove(override)\n                consumed.append(override)\n        return consumed\n\n    def _apply_free_defaults(self, defaults, overrides):\n        consumed = []\n        for override in overrides:\n            key, value = ConfigLoader._split_key_val(override)\n            if self.exists_in_search_path(key):\n                # Do not add multirun configs into defaults, those will be added to the defaults\n                # during the runs after list is broken into items\n                if \",\" not in value:\n                    defaults.append({key: value})\n                overrides.remove(override)\n                consumed.append(override)\n\n        return consumed\n",
  "start_line": 110,
  "end_line": 175,
  "total_lines": 453,
  "is_truncated": false
}

{
  "status": "ok",
  "filepath": "hydra/_internal/config_loader.py",
  "content": "                ret = resource_exists(module_name, resource_name)\n            except ImportError:\n                ret = False\n            except ValueError:  # Python 2.7 throws ValueError empty module name sometimes.\n                ret = False\n            except NotImplementedError:\n                raise NotImplementedError(\n                    \"Unable to load {}/{}, are you missing an __init__.py?\".format(\n                        module_name, resource_name\n                    )\n                )\n            # Uncomment to debug issues with configs not getting loaded.\n            # WARNING: breaks integration tests that relies on specified output from the process.\n            # print(\"_exists? {}, {} == {}\".format(module_name, resource_name, ret))\n            return ret\n        else:\n            return os.path.exists(filename)\n\n    def _merge_defaults(self, cfg, defaults, split_at):\n        def get_filename(config_name):\n            filename, ext = os.path.splitext(config_name)\n            if ext == \"\":\n                ext = \".yaml\"\n            return \"{}{}\".format(filename, ext)\n\n        def merge_defaults(merged_cfg, def_list):",
  "start_line": 340,
  "end_line": 365,
  "total_lines": 453,
  "is_truncated": false
}

{
  "status": "ok",
  "filepath": "hydra/_internal/config_loader.py",
  "content": "                        module_name, resource_name\n                    )\n                )\n            # Uncomment to debug issues with configs not getting loaded.\n            # WARNING: breaks integration tests that relies on specified output from the process.\n            # print(\"_exists? {}, {} == {}\".format(module_name, resource_name, ret))\n            return ret\n        else:\n            return os.path.exists(filename)\n\n    def _merge_defaults(self, cfg, defaults, split_at):\n        def get_filename(config_name):\n            filename, ext = os.path.splitext(config_name)\n            if ext == \"\":\n                ext = \".yaml\"\n            return \"{}{}\".format(filename, ext)\n\n        def merge_defaults(merged_cfg, def_list):\n            cfg_with_list = OmegaConf.create(dict(defaults=def_list))\n            for default1 in cfg_with_list.defaults:\n                if isinstance(default1, DictConfig):\n                    is_optional = False\n                    if default1.optional is not None:\n                        is_optional = default1.optional\n                        del default1[\"optional\"]\n                    family = next(iter(default1.keys()))\n                    name = default1[family]\n                    # Name is none if default value is removed",
  "start_line": 348,
  "end_line": 375,
  "total_lines": 453,
  "is_truncated": false
}

{
  "status": "ok",
  "filepath": "hydra/_internal/config_loader.py",
  "content": "                    if name is not None and \"_SKIP_\" not in name:\n                        merged_cfg = self._merge_config(\n                            cfg=merged_cfg,\n                            family=family,\n                            name=get_filename(name),\n                            required=not is_optional,\n                        )\n                else:\n                    assert isinstance(default1, str)\n                    if \"_SKIP_\" not in default1:\n                        merged_cfg = self._merge_config(\n                            cfg=merged_cfg,\n                            family=\"\",\n                            name=get_filename(default1),\n                            required=True,\n                        )\n            return merged_cfg\n\n        system_list = []\n        user_list = []\n        for default in defaults:\n            if len(system_list) < split_at:\n                system_list.append(default)\n            else:\n                user_list.append(default)",
  "start_line": 376,
  "end_line": 400,
  "total_lines": 453,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "('0', '.1')\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "hydra/_internal/config_loader.py",
  "content": "\n    def _load_config_impl(self, input_file, record_load=True):\n        \"\"\"\n        :param input_file:\n        :param record_load:\n        :return: the loaded config or None if it was not found\n        \"\"\"\n        loaded_cfg = None\n        filename, search_path = self._find_config(input_file)\n        if search_path is None and record_load:\n            assert search_path is None\n            self.all_config_checked.append((filename, None, None))\n\n        if search_path is not None:\n            fullpath = \"{}/{}\".format(search_path.path, filename)\n            is_pkg = search_path.path.startswith(\"pkg://\")\n            if is_pkg:\n                fullpath = fullpath[len(\"pkg://\") :]\n                module_name, resource_name = ConfigLoader._split_module_and_resource(\n                    fullpath\n                )\n                with resource_stream(module_name, resource_name) as stream:\n                    loaded_cfg = OmegaConf.load(stream)\n                if record_load:\n                    self.all_config_checked.append(\n                        (filename, search_path.path, search_path.provider)\n                    )\n            elif os.path.exists(fullpath):\n                loaded_cfg = OmegaConf.load(fullpath)\n                if record_load:\n                    self.all_config_checked.append(\n                        (filename, search_path.path, search_path.provider)\n                    )\n            else:\n                # This should never happen because we just searched for it and found it\n                assert False, \"'{}' not found\".format(fullpath)\n        return loaded_cfg\n\n    def list_groups(self, parent_name):\n        ret = list(set(self.get_group_options(parent_name, file_type=\"dir\")))\n        return ret\n\n    def get_group_options(self, group_name, file_type=\"file\"):\n        options = []\n        for search_path in self.config_search_path.config_search_path:\n            search_path = search_path.path\n            is_pkg, path = self._split_path(search_path)\n            files = []\n            if is_pkg:\n                module_name, resource_name = ConfigLoader._split_module_and_resource(\n                    \"{}/{}\".format(path, group_name)",
  "start_line": 200,
  "end_line": 250,
  "total_lines": 453,
  "is_truncated": false
}

{
  "status": "ok",
  "filepath": "hydra/_internal/config_loader.py",
  "content": "                    \"{}/{}\".format(path, group_name)\n                )\n                if self._exists(is_pkg, path) and resource_isdir(\n                    module_name, resource_name\n                ):\n                    all_files = resource_listdir(module_name, resource_name)\n                    files = []\n                    for file in all_files:\n                        if (\n                            file_type == \"dir\"\n                            and resource_isdir(\n                                module_name, os.path.join(group_name, file)\n                            )\n                            and file != \"__pycache__\"\n                        ):\n                            files.append(file)\n                        elif file_type == \"file\" and file.endswith(\".yaml\"):\n                            files.append(file[0 : -len(\".yaml\")])\n            else:\n                group_path = \"{}/{}\".format(path, group_name)\n                if os.path.isdir(group_path):\n                    all_files = os.listdir(group_path)\n                    files = []\n                    for file in all_files:\n                        full_path = os.path.join(group_path, group_name, file)\n                        if (\n                            file_type == \"dir\"\n                            and os.path.isdir(full_path)\n                            and file != \"__pycache__\"\n                        ):\n                            files.append(file)\n                        elif file_type == \"file\" and file.endswith(\".yaml\"):\n                            files.append(file[0 : -len(\".yaml\")])\n\n            options.extend(files)\n        return options\n\n    def _merge_config(self, cfg, family, name, required):\n\n        if family != \"\":\n            new_cfg = \"{}/{}\".format(family, name)\n        else:\n            new_cfg = name\n\n        loaded_cfg = self._load_config_impl(new_cfg)\n        if loaded_cfg is None:\n            if required:\n                if family == \"\":\n                    msg = \"Could not load {}\".format(new_cfg)\n                    raise MissingConfigException(msg, new_cfg)\n                else:",
  "start_line": 250,
  "end_line": 300,
  "total_lines": 453,
  "is_truncated": false
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "Traceback (most recent call last):\n  File \"<stdin>\", line 2, in <module>\nImportError: cannot import name 'initialize' from 'hydra' (/workspace/hydra/__init__.py)\n",
  "details": {
    "stdout": "",
    "stderr": "Traceback (most recent call last):\n  File \"<stdin>\", line 2, in <module>\nImportError: cannot import name 'initialize' from 'hydra' (/workspace/hydra/__init__.py)\n",
    "exit_code": 1
  }
}

{
  "status": "ok",
  "filepath": "hydra/__init__.py",
  "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nfrom . import utils\nfrom .errors import MissingConfigException\nfrom .main import main\n\n# Source of truth for Hydra's version\n__version__ = \"0.10.0\"\n\n__all__ = [\"__version__\", \"MissingConfigException\", \"main\", \"utils\"]",
  "start_line": 1,
  "end_line": 9,
  "total_lines": 9,
  "is_truncated": false
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "",
  "details": {
    "stdout": "",
    "stderr": "",
    "exit_code": 1
  }
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "",
  "details": {
    "stdout": "",
    "stderr": "",
    "exit_code": 1
  }
}

{
  "status": "ok",
  "filepath": "hydra/main.py",
  "content": "# Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved\nimport functools\nimport sys\nfrom ._internal.utils import run_hydra, get_args_parser\n\n\ndef main(config_path=\"\", strict=None):\n    \"\"\"\n    :param config_path: the config path, can be a directory in which it's used as the config root\n    or a file to load\n    :param strict: strict mode, will throw an error if command line overrides are not changing an\n    existing key or\n           if the code is accessing a non existent key\n    \"\"\"\n\n    def main_decorator(task_function):\n        @functools.wraps(task_function)\n        def decorated_main():\n            try:\n                run_hydra(\n                    args_parser=get_args_parser(),\n                    task_function=task_function,\n                    config_path=config_path,\n                    strict=strict,\n                )\n            except KeyboardInterrupt:\n                sys.exit(-1)\n            except SystemExit:\n                pass\n\n        return decorated_main\n\n    return main_decorator",
  "start_line": 1,
  "end_line": 33,
  "total_lines": 33,
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "Caught expected error: Could not load lr/0.1, available options:\nlr:\n\t0.1\n\t0.01\n",
  "stderr": "Traceback (most recent call last):\n  File \"<stdin>\", line 44, in <module>\n  File \"/workspace/hydra/_internal/config_loader.py\", line 402, in _merge_defaults\n    cfg = merge_defaults(cfg, user_list)\n  File \"/workspace/hydra/_internal/config_loader.py\", line 381, in merge_defaults\n    required=not is_optional,\n  File \"/workspace/hydra/_internal/config_loader.py\", line 308, in _merge_config\n    raise MissingConfigException(msg, new_cfg, options)\nhydra.errors.MissingConfigException: Could not load lr/0.1, available options:\nlr:\n\t0.1\n\t0.01\n",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "6\n0\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "hydra/_internal/config_loader.py",
  "content": "                        module_name, resource_name\n                    )\n                )\n            # Uncomment to debug issues with configs not getting loaded.\n            # WARNING: breaks integration tests that relies on specified output from the process.\n            # print(\"_exists? {}, {} == {}\".format(module_name, resource_name, ret))\n            return ret\n        else:\n            return os.path.exists(filename)\n\n    def _merge_defaults(self, cfg, defaults, split_at):\n        def get_filename(config_name):\n            filename, ext = os.path.splitext(config_name)\n            if ext == \"\":\n                ext = \".yaml\"\n            return \"{}{}\".format(filename, ext)\n\n        def merge_defaults(merged_cfg, def_list):\n            cfg_with_list = OmegaConf.create(dict(defaults=def_list))\n            for default1 in cfg_with_list.defaults:\n                if isinstance(default1, DictConfig):\n                    is_optional = False\n                    if default1.optional is not None:\n                        is_optional = default1.optional\n                        del default1[\"optional\"]\n                    family = next(iter(default1.keys()))\n                    name = default1[family]\n                    # Name is none if default value is removed",
  "start_line": 348,
  "end_line": 375,
  "total_lines": 453,
  "is_truncated": false,
  "budget_warning": "Only 10 tool call(s) remaining (20/30 used). Finalize your edits and call submit_patch soon."
}

{
  "status": "ok",
  "filepath": "hydra/_internal/config_loader.py",
  "occurrences": 1,
  "strategy": "unescaped",
  "diff": "--- a/hydra/_internal/config_loader.py\n+++ b/hydra/_internal/config_loader.py\n@@ -357,10 +357,9 @@\n \n     def _merge_defaults(self, cfg, defaults, split_at):\n         def get_filename(config_name):\n-            filename, ext = os.path.splitext(config_name)\n-            if ext == \"\":\n-                ext = \".yaml\"\n-            return \"{}{}\".format(filename, ext)\n+            if not config_name.endswith(\".yaml\"):\n+                return \"{}.yaml\".format(config_name)\n+            return config_name\n \n         def merge_defaults(merged_cfg, def_list):\n             cfg_with_list = OmegaConf.create(dict(defaults=def_list))\n",
  "is_truncated": false,
  "budget_warning": "Only 9 tool call(s) remaining (21/30 used). Finalize your edits and call submit_patch soon."
}

{
  "status": "ok",
  "stdout": "Successfully merged defaults!\n",
  "stderr": "",
  "exit_code": 0,
  "budget_warning": "Only 8 tool call(s) remaining (22/30 used). Finalize your edits and call submit_patch soon."
}