iterative__dvc-8904

{
  "status": "ok",
  "stdout": "./tests/func/test_stage_load.py:130:        \"stages\": {\"build\": {\"foreach\": \"${vars}\", \"do\": {\"cmd\": \"echo ${item}\"}}},\n./tests/func/test_dvcfile.py:379:            {\"build\": {\"foreach\": [\"us\", \"gb\"], \"do\": {\"cmd\": \"echo ${foo}\"}}},\n./tests/func/utils/test_strict_yaml.py:192:    foreach: 3\n./tests/func/utils/test_strict_yaml.py:200:expected dict, in stages -> group -> foreach, line 3, column 5\n./tests/func/utils/test_strict_yaml.py:202:  3 │   foreach: 3\n./tests/func/utils/test_strict_yaml.py:208:    foreach: [1,2,3]\n./tests/func/utils/test_strict_yaml.py:218:  3 │   foreach: [1,2,3]\n./tests/func/utils/test_strict_yaml.py:225:    foreach: [1,2,3]\n./tests/func/utils/test_strict_yaml.py:235:    foreach: [1,2,3]\n./tests/func/utils/test_strict_yaml.py:245:  3 │   foreach: [1,2,3]\n./tests/func/utils/test_strict_yaml.py:250:  3 │   foreach: [1,2,3]\n./tests/func/utils/test_strict_yaml.py:312:    \"foreach_scalar\": (FOREACH_SCALAR_VALUE, FOREACH_SCALAR_VALUE_OUTPUT),\n./tests/func/utils/test_strict_yaml.py:313:    \"foreach_do_do_null\": (FOREACH_DO_NULL, FOREACH_DO_NULL_OUTPUT),\n./tests/func/utils/test_strict_yaml.py:314:    \"foreach_do_missing_cmd\": (\n./tests/func/utils/test_strict_yaml.py:318:    \"foreach_unknown_cmd_missing_do\": (\n./tests/func/parsing/test_errors.py:14:from . import make_entry_definition, make_foreach_def\n./tests/func/parsing/test_errors.py:171:# Tests foreach generated stages and their error messages\n./tests/func/parsing/test_errors.py:174:def test_foreach_data_syntax_error(tmp_dir, dvc):\n./tests/func/parsing/test_errors.py:175:    definition = make_foreach_def(tmp_dir, \"build\", \"${syntax.[error}\", {})\n./tests/func/parsing/test_errors.py:181:        == \"failed to parse 'stages.build.foreach' in 'dvc.yaml':\\n\"\n./tests/func/parsing/test_errors.py:190:def test_foreach_data_key_does_not_exists(tmp_dir, dvc, key):\n./tests/func/parsing/test_errors.py:191:    definition = make_foreach_def(tmp_dir, \"build\", embrace(key), {})\n./tests/func/parsing/test_errors.py:195:        str(exc_info.value) == \"failed to parse 'stages.build.foreach' in 'dvc.yaml': \"\n./tests/func/parsing/test_errors.py:201:    \"foreach_data\", [\"${foo}\", \"${dct.model1}\", \"${lst.0}\", \"foobar\"]\n./tests/func/parsing/test_errors.py:203:def test_foreach_data_expects_list_or_dict(tmp_dir, dvc, foreach_data):\n./tests/func/parsing/test_errors.py:205:    definition = make_foreach_def(tmp_dir, \"build\", foreach_data, {}, context)\n./tests/func/parsing/test_errors.py:210:        == \"failed to resolve 'stages.build.foreach' in 'dvc.yaml': \"\n./tests/func/parsing/test_errors.py:223:def test_foreach_overwriting_item_in_list(tmp_dir, dvc, caplog, global_data, where):\n./tests/func/parsing/test_errors.py:225:    definition = make_foreach_def(\n./tests/func/parsing/test_errors.py:237:def test_foreach_do_syntax_errors(tmp_dir, dvc):\n./tests/func/parsing/test_errors.py:238:    definition = make_foreach_def(\n./tests/func/parsing/test_errors.py:265:def test_foreach_do_definition_item_does_not_exist(tmp_dir, dvc, key, loc):\n./tests/func/parsing/test_errors.py:267:    definition = make_foreach_def(\n./tests/func/parsing/test_errors.py:304:    definition = make_foreach_def(\n./tests/func/parsing/test_errors.py:328:def test_foreach_wdir_key_does_not_exist(tmp_dir, dvc):\n./tests/func/parsing/test_errors.py:329:    definition = make_foreach_def(\n./tests/func/parsing/test_foreach.py:1:\"\"\"Testing happy paths for the foreach.\"\"\"\n./tests/func/parsing/test_foreach.py:11:    \"\"\"Testing a simple non-nested list as a foreach data\"\"\"\n./tests/func/parsing/test_foreach.py:15:    data = {\"foreach\": [\"foo\", \"bar\", \"baz\"], \"do\": {\"cmd\": \"echo ${item}\"}}\n./tests/func/parsing/test_foreach.py:20:    # check that `foreach` item-key replacement didnot leave any leftovers.\n./tests/func/parsing/test_foreach.py:30:    foreach_data = {\"model1\": \"foo\", \"model2\": \"bar\"}\n./tests/func/parsing/test_foreach.py:31:    data = {\"foreach\": foreach_data, \"do\": {\"cmd\": \"echo ${key} ${item}\"}}\n./tests/func/parsing/test_foreach.py:41:    # check that `foreach` item-key replacement didnot leave any leftovers.\n./tests/func/parsing/test_foreach.py:51:    foreach_data = [{\"thresh\": \"foo\"}, {\"thresh\": \"bar\"}]\n./tests/func/parsing/test_foreach.py:52:    data = {\"foreach\": foreach_data, \"do\": {\"cmd\": \"echo ${item.thresh}\"}}\n./tests/func/parsing/test_foreach.py:56:    # check that `foreach` item-key replacement didnot leave any leftovers.\n./tests/func/parsing/test_foreach.py:64:def test_foreach_interpolated_simple_list(tmp_dir, dvc):\n./tests/func/parsing/test_foreach.py:65:    foreach_data = [\"foo\", \"bar\", \"baz\"]\n./tests/func/parsing/test_foreach.py:66:    vars_ = {\"models\": foreach_data}\n./tests/func/parsing/test_foreach.py:68:    data = {\"foreach\": \"${models}\", \"do\": {\"cmd\": \"echo ${item}\"}}\n./tests/func/parsing/test_foreach.py:76:    assert resolver.context == {\"models\": foreach_data}\n./tests/func/parsing/test_foreach.py:80:@pytest.mark.parametrize(\"foreach_def\", [\"${item.thresh}\", \"${item[thresh]}\"])\n./tests/func",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "./tests/func/parsing/__init__.py:49:def make_foreach_def(\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "dvc/repo/stage.py:230:        if accept_group and stages.is_foreach_generated(name):\ndvc/repo/stage.py:249:                foreach are returned.\ndvc/repo/stage.py:308:                - a foreach group name or a stage name in the `dvc.yaml` file.\ndvc/repo/stage.py:309:                - a generated stage name from a foreach group.\ndvc/repo/stage.py:356:        - foreach_group_name or stage_name\ndvc/repo/stage.py:357:        - generated stage name from a foreach group\ndvc/parsing/__init__.py:53:FOREACH_KWD = \"foreach\"\ndvc/parsing/__init__.py:108:def split_foreach_name(name: str) -> Tuple[str, Optional[str]]:\ndvc/parsing/__init__.py:170:        group, key = split_foreach_name(name)\ndvc/parsing/__init__.py:192:        return self._has_group_and_key(*split_foreach_name(key))\ndvc/parsing/__init__.py:258:            # but for foreach-generated ones, once is enough, which it does\ndvc/parsing/__init__.py:330:        self.foreach_data = definition[FOREACH_KWD]\ndvc/parsing/__init__.py:338:        # optimization: check for syntax errors only once for `foreach` stages\ndvc/parsing/__init__.py:344:        return self._resolve_foreach_data()\ndvc/parsing/__init__.py:346:    def _resolve_foreach_data(self) -> \"SeqOrMap\":\ndvc/parsing/__init__.py:348:            iterable = self.context.resolve(self.foreach_data, unwrap=False)\ndvc/parsing/__init__.py:350:            format_and_raise(exc, f\"'{self.where}.{self.name}.foreach'\", self.relpath)\ndvc/parsing/__init__.py:352:        # foreach data can be a resolved dictionary/list.\ndvc/parsing/__init__.py:354:        # foreach stages will have `item` and `key` added to the context\ndvc/parsing/__init__.py:366:                f\"failed to resolve '{self.where}.{self.name}.foreach'\"\ndvc/parsing/__init__.py:421:        err_message = f\"Could not find '{key}' in foreach group '{self.name}'\"\ndvc/parsing/__init__.py:425:        # NOTE: we need to use resolved iterable/foreach-data,\ndvc/parsing/__init__.py:446:                # optimization: skip checking for syntax errors on each foreach\ndvc/parsing/interpolate.py:201:    not_foreach = key is not None and \"foreach\" not in key\ndvc/parsing/interpolate.py:202:    if not_primitive and not_foreach:\ndvc/stage/loader.py:174:    def is_foreach_generated(self, name: str):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "dvc/parsing/__init__.py",
  "content": "            format_and_raise(exc, f\"'{where}.{name}.{key}'\", path)\n\n\ndef is_map_or_seq(data: Any) -> bool:\n    _is_map_or_seq = isa(Mapping, Sequence)\n    return not isinstance(data, str) and _is_map_or_seq(data)\n\n\ndef split_foreach_name(name: str) -> Tuple[str, Optional[str]]:\n    group, *keys = name.rsplit(JOIN, maxsplit=1)\n    return group, first(keys)\n\n\ndef check_interpolations(data: \"DictStrAny\", where: str, path: str):\n    def func(s: \"DictStrAny\") -> None:\n        if is_interpolated_string(s):\n            raise ResolveError(\n                _format_preamble(f\"'{where}'\", path) + \"interpolating is not allowed\"\n            )\n\n    return recurse(func)(data)\n\n\nDefinition = Union[\"ForeachDefinition\", \"EntryDefinition\"]\n\n\ndef make_definition(\n    resolver: \"DataResolver\", name: str, definition: \"DictStrAny\", **kwargs\n) -> Definition:\n    args = resolver, resolver.context, name, definition\n    if FOREACH_KWD in definition:\n        return ForeachDefinition(*args, **kwargs)\n    return EntryDefinition(*args, **kwargs)\n\n\nclass DataResolver:\n    def __init__(self, repo: \"Repo\", wdir: str, d: dict):\n        self.fs = fs = repo.fs\n\n        if os.path.isabs(wdir):\n            wdir = fs.path.relpath(wdir)\n            wdir = \"\" if wdir == os.curdir else wdir\n\n        self.wdir = wdir\n        self.relpath = fs.path.normpath(fs.path.join(self.wdir, \"dvc.yaml\"))\n\n        vars_ = d.get(VARS_KWD, [])\n        check_interpolations(vars_, VARS_KWD, self.relpath)\n        self.context: Context = Context()\n\n        try:\n            args = fs, vars_, wdir  # load from `vars` section\n            self.context.load_from_vars(*args, default=DEFAULT_PARAMS_FILE)\n        except ContextError as exc:\n            format_and_raise(exc, \"'vars'\", self.relpath)\n\n        # we use `tracked_vars` to keep a dictionary of used variables\n        # by the interpolated entries.\n        self.tracked_vars: Dict[str, Mapping] = {}\n\n        stages_data = d.get(STAGES_KWD, {})\n        # we wrap the definitions into ForeachDefinition and EntryDefinition,\n        # that helps us to optimize, cache and selectively load each one of\n        # them as we need, and simplify all of this DSL/parsing logic.\n        self.definitions: Dict[str, Definition] = {\n            name: make_definition(self, name, definition)\n            for name, definition in stages_data.items()\n        }\n\n    def resolve_one(self, name: str):\n        group, key = split_foreach_name(name)\n\n        if not self._has_group_and_key(group, key):\n            raise EntryNotFound(f\"Could not find '{name}'\")\n\n        # all of the checks for `key` not being None for `ForeachDefinition`\n        # and/or `group` not existing in the `interim`, etc. should be\n        # handled by the `self.has_key()` above.\n        definition = self.definitions[group]\n        if isinstance(definition, EntryDefinition):\n            return definition.resolve()\n\n        assert key\n        return definition.resolve_one(key)\n\n    def resolve(self):\n        \"\"\"Used for testing purposes, otherwise use resolve_one().\"\"\"\n        data = join(map(self.resolve_one, self.get_keys()))\n        logger.trace(\"Resolved dvc.yaml:\\n%s\", data)  # type: ignore[attr-defined]\n        return {STAGES_KWD: data}\n\n    def has_key(self, key: str):\n        return self._has_group_and_key(*split_foreach_name(key))\n\n    def _has_group_and_key(self, group: str, key: Optional[str] = None):\n        try:\n            definition = self.definitions[group]\n        except KeyError:\n            return False\n\n        if key:\n            return isinstance(definition, ForeachDefinition) and definition.has_member(\n                key\n            )\n        return not isinstance(definition, ForeachDefinition)\n\n    @collecting\n    def get_keys(self):\n        for name, definition in self.definitions.items():\n            if isinstance(definition, ForeachDefinition):\n                yield from definition.get_generated_names()\n                continue\n            yield name\n\n    def track_vars(self, name: str, vars_) -> None:\n        self.tracked_vars[name] = vars_\n\n\nclass EntryDefinition:\n    def __init__(\n        self,\n        resolver: DataResolver,\n        context: Context,\n        name: str,\n        definition: \"DictStrAny\",\n        where: str = STAGES_KWD,\n    ):\n        self.resolver = resolver\n        self.wdir = self.resolver.wdir\n        self.relpath = self.resolver.relpath\n        self.context = context\n        self.name = name\n        self.definition = definition\n        self.where = where\n\n    def _resolve_wdir(\n        self, context: Context, name: str, wdir: Optional[str] = None\n    ) -> str:\n        if not wdir:\n            return self.wdir\n\n        try:\n            wdir = to_str(context.resolve_str(wdir))\n        except (ContextError, ParseError) as exc:\n            format_and_raise(exc, f\"'{self.where}.{name}.wdir'\", self.relpath)\n        return self.resolver.fs.path.join(self.wdir, wdir)\n\n    def resolve(self, **kwargs):\n        try:\n            return self.resolve_stage(**kwargs)",
  "start_line": 100,
  "end_line": 249,
  "total_lines": 455,
  "is_truncated": true
}

{
  "status": "ok",
  "filepath": "dvc/parsing/__init__.py",
  "content": "import logging\nimport os\nfrom collections.abc import Mapping, Sequence\nfrom copy import deepcopy\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Dict,\n    List,\n    NamedTuple,\n    Optional,\n    Tuple,\n    Type,\n    Union,\n)\n\nfrom funcy import collecting, first, isa, join, reraise\n\nfrom dvc.exceptions import DvcException\nfrom dvc.parsing.interpolate import ParseError\nfrom dvc.utils.objects import cached_property\n\nfrom .context import (\n    Context,\n    ContextError,\n    KeyNotInContext,\n    MergeError,\n    Node,\n    VarsAlreadyLoaded,\n)\nfrom .interpolate import (\n    check_recursive_parse_errors,\n    is_interpolated_string,\n    recurse,\n    to_str,\n)\n\nif TYPE_CHECKING:\n    from typing import NoReturn\n\n    from dvc.repo import Repo\n    from dvc.types import DictStrAny\n\n    from .context import SeqOrMap\n\n\nlogger = logging.getLogger(__name__)\n\nSTAGES_KWD = \"stages\"\nVARS_KWD = \"vars\"\nWDIR_KWD = \"wdir\"\nPARAMS_KWD = \"params\"\nFOREACH_KWD = \"foreach\"\nDO_KWD = \"do\"\n\nDEFAULT_PARAMS_FILE = \"params.yaml\"\n\nJOIN = \"@\"\n\n\nclass ResolveError(DvcException):\n    pass\n\n\nclass EntryNotFound(DvcException):\n    pass\n\n\ndef _format_preamble(msg: str, path: str, spacing: str = \" \") -> str:\n    return f\"failed to parse {msg} in '{path}':{spacing}\"\n\n\ndef format_and_raise(exc: Exception, msg: str, path: str) -> \"NoReturn\":\n    spacing = (\n        \"\\n\" if isinstance(exc, (ParseError, MergeError, VarsAlreadyLoaded)) else \" \"\n    )\n    message = _format_preamble(msg, path, spacing) + str(exc)\n\n    # FIXME: cannot reraise because of how we log \"cause\" of the exception\n    # the error message is verbose, hence need control over the spacing\n    _reraise_err(ResolveError, message, from_exc=exc)\n\n\ndef _reraise_err(\n    exc_cls: Type[Exception], *args, from_exc: Optional[Exception] = None\n) -> \"NoReturn\":\n    err = exc_cls(*args)\n    if from_exc and logger.isEnabledFor(logging.DEBUG):\n        raise err from from_exc\n    raise err\n\n\ndef check_syntax_errors(\n    definition: \"DictStrAny\", name: str, path: str, where: str = \"stages\"\n):\n    for key, d in definition.items():\n        try:\n            check_recursive_parse_errors(d)\n        except ParseError as exc:\n            format_and_raise(exc, f\"'{where}.{name}.{key}'\", path)\n\n\ndef is_map_or_seq(data: Any) -> bool:\n    _is_map_or_seq = isa(Mapping, Sequence)\n    return not isinstance(data, str) and _is_map_or_seq(data)\n\n\ndef split_foreach_name(name: str) -> Tuple[str, Optional[str]]:\n    group, *keys = name.rsplit(JOIN, maxsplit=1)\n    return group, first(keys)\n\n\ndef check_interpolations(data: \"DictStrAny\", where: str, path: str):\n    def func(s: \"DictStrAny\") -> None:\n        if is_interpolated_string(s):\n            raise ResolveError(\n                _format_preamble(f\"'{where}'\", path) + \"interpolating is not allowed\"\n            )\n\n    return recurse(func)(data)\n\n\nDefinition = Union[\"ForeachDefinition\", \"EntryDefinition\"]\n\n\ndef make_definition(\n    resolver: \"DataResolver\", name: str, definition: \"DictStrAny\", **kwargs\n) -> Definition:\n    args = resolver, resolver.context, name, definition\n    if FOREACH_KWD in definition:\n        return ForeachDefinition(*args, **kwargs)\n    return EntryDefinition(*args, **kwargs)\n\n\nclass DataResolver:\n    def __init__(self, repo: \"Repo\", wdir: str, d: dict):\n        self.fs = fs = repo.fs\n\n        if os.path.isabs(wdir):\n            wdir = fs.path.relpath(wdir)\n            wdir = \"\" if wdir == os.curdir else wdir\n\n        self.wdir = wdir\n        self.relpath = fs.path.normpath(fs.path.join(self.wdir, \"dvc.yaml\"))\n\n        vars_ = d.get(VARS_KWD, [])\n        check_interpolations(vars_, VARS_KWD, self.relpath)\n        self.context: Context = Context()\n\n        try:",
  "start_line": 1,
  "end_line": 150,
  "total_lines": 455,
  "is_truncated": true
}

{
  "status": "ok",
  "filepath": "dvc/parsing/__init__.py",
  "content": "        except ContextError as exc:\n            format_and_raise(exc, f\"stage '{self.name}'\", self.relpath)\n\n    def resolve_stage(self, skip_checks: bool = False) -> \"DictStrAny\":\n        context = self.context\n        name = self.name\n        if not skip_checks:\n            # we can check for syntax errors as we go for interpolated entries,\n            # but for foreach-generated ones, once is enough, which it does\n            # that itself. See `ForeachDefinition.do_definition`.\n            check_syntax_errors(self.definition, name, self.relpath)\n\n        # we need to pop vars from generated/evaluated data\n        definition = deepcopy(self.definition)\n\n        wdir = self._resolve_wdir(context, name, definition.get(WDIR_KWD))\n        vars_ = definition.pop(VARS_KWD, [])\n        # FIXME: Should `vars` be templatized?\n        check_interpolations(vars_, f\"{self.where}.{name}.vars\", self.relpath)\n        if vars_:\n            # Optimization: Lookahead if it has any vars, if it does not, we\n            # don't need to clone them.\n            context = Context.clone(context)\n\n        try:\n            fs = self.resolver.fs\n            context.load_from_vars(fs, vars_, wdir, stage_name=name)\n        except VarsAlreadyLoaded as exc:\n            format_and_raise(exc, f\"'{self.where}.{name}.vars'\", self.relpath)\n\n        logger.trace(  # type: ignore[attr-defined]\n            \"Context during resolution of stage %s:\\n%s\", name, context\n        )\n\n        with context.track() as tracked_data:\n            # NOTE: we do not pop \"wdir\", and resolve it again\n            # this does not affect anything and is done to try to\n            # track the source of `wdir` interpolation.\n            # This works because of the side-effect that we do not\n            # allow overwriting and/or str interpolating complex objects.\n            # Fix if/when those assumptions are no longer valid.\n            resolved = {\n                key: self._resolve(context, value, key, skip_checks)\n                for key, value in definition.items()\n            }\n\n        self.resolver.track_vars(name, tracked_data)\n        return {name: resolved}\n\n    def _resolve(\n        self, context: \"Context\", value: Any, key: str, skip_checks: bool\n    ) -> \"DictStrAny\":\n        try:\n            return context.resolve(\n                value, skip_interpolation_checks=skip_checks, key=key\n            )\n        except (ParseError, KeyNotInContext) as exc:\n            format_and_raise(exc, f\"'{self.where}.{self.name}.{key}'\", self.relpath)\n\n\nclass IterationPair(NamedTuple):\n    key: str = \"key\"\n    value: str = \"item\"\n\n\nclass ForeachDefinition:\n    def __init__(\n        self,\n        resolver: DataResolver,\n        context: Context,\n        name: str,\n        definition: \"DictStrAny\",\n        where: str = STAGES_KWD,\n    ):\n        self.resolver = resolver\n        self.relpath = self.resolver.relpath\n        self.context = context\n        self.name = name\n\n        assert DO_KWD in definition\n        self.foreach_data = definition[FOREACH_KWD]\n        self._do_definition = definition[DO_KWD]\n\n        self.pair = IterationPair()\n        self.where = where\n\n    @cached_property\n    def do_definition(self):\n        # optimization: check for syntax errors only once for `foreach` stages\n        check_syntax_errors(self._do_definition, self.name, self.relpath)\n        return self._do_definition\n\n    @cached_property\n    def resolved_iterable(self):\n        return self._resolve_foreach_data()\n\n    def _resolve_foreach_data(self) -> \"SeqOrMap\":\n        try:\n            iterable = self.context.resolve(self.foreach_data, unwrap=False)\n        except (ContextError, ParseError) as exc:\n            format_and_raise(exc, f\"'{self.where}.{self.name}.foreach'\", self.relpath)\n\n        # foreach data can be a resolved dictionary/list.\n        self._check_is_map_or_seq(iterable)\n        # foreach stages will have `item` and `key` added to the context\n        # so, we better warn them if they have them already in the context\n        # from the global vars. We could add them in `set_temporarily`, but\n        # that'd make it display for each iteration.\n        self._warn_if_overwriting(self._inserted_keys(iterable))\n        return iterable\n\n    def _check_is_map_or_seq(self, iterable):\n        if not is_map_or_seq(iterable):\n            node = iterable.value if isinstance(iterable, Node) else iterable\n            typ = type(node).__name__\n            raise ResolveError(\n                f\"failed to resolve '{self.where}.{self.name}.foreach'\"\n                f\" in '{self.relpath}': expected list/dictionary, got \" + typ\n            )\n\n    def _warn_if_overwriting(self, keys: List[str]):\n        warn_for = [k for k in keys if k in self.context]\n        if warn_for:\n            linking_verb = \"is\" if len(warn_for) == 1 else \"are\"\n            logger.warning(\n                (\n                    \"%s %s already specified, \"\n                    \"will be overwritten for stages generated from '%s'\"\n                ),\n                \" and \".join(warn_for),\n                linking_verb,\n                self.name,\n            )\n\n    def _inserted_keys(self, iterable) -> List[str]:\n        keys = [self.pair.value]\n        if isinstance(iterable, Mapping):\n            keys.append(self.pair.key)\n        return keys\n\n    @cached_property\n    def normalized_iterable(self):\n        \"\"\"Convert sequence to Mapping with keys normalized.\"\"\"\n        iterable = self.resolved_iterable\n        if isinstance(iterable, Mapping):\n            return iterable\n\n        assert isinstance(iterable, Sequence)\n        if any(map(is_map_or_seq, iterable)):\n            # if the list contains composite data, index are the keys",
  "start_line": 250,
  "end_line": 399,
  "total_lines": 455,
  "is_truncated": true
}

{
  "status": "ok",
  "filepath": "dvc/parsing/__init__.py",
  "content": "            return {to_str(idx): value for idx, value in enumerate(iterable)}\n\n        # for simple lists, eg: [\"foo\", \"bar\"],  contents are the key itself\n        return {to_str(value): value for value in iterable}\n\n    def has_member(self, key: str) -> bool:\n        return key in self.normalized_iterable\n\n    def get_generated_names(self):\n        return list(map(self._generate_name, self.normalized_iterable))\n\n    def _generate_name(self, key: str) -> str:\n        return f\"{self.name}{JOIN}{key}\"\n\n    def resolve_all(self) -> \"DictStrAny\":\n        return join(map(self.resolve_one, self.normalized_iterable))\n\n    def resolve_one(self, key: str) -> \"DictStrAny\":\n        return self._each_iter(key)\n\n    def _each_iter(self, key: str) -> \"DictStrAny\":\n        err_message = f\"Could not find '{key}' in foreach group '{self.name}'\"\n        with reraise(KeyError, EntryNotFound(err_message)):\n            value = self.normalized_iterable[key]\n\n        # NOTE: we need to use resolved iterable/foreach-data,\n        # not the normalized ones to figure out whether to make item/key\n        # available\n        inserted = self._inserted_keys(self.resolved_iterable)\n        temp_dict = {self.pair.value: value}\n        key_str = self.pair.key\n        if key_str in inserted:\n            temp_dict[key_str] = key\n\n        with self.context.set_temporarily(temp_dict, reserve=True):\n            # optimization: item and key can be removed on __exit__() as they\n            # are top-level values, and are not merged recursively.\n            # This helps us avoid cloning context, which is slower\n            # (increasing the size of the context might increase\n            # the no. of items to be generated which means more cloning,\n            # i.e. quadratic complexity).\n            generated = self._generate_name(key)\n            entry = EntryDefinition(\n                self.resolver, self.context, generated, self.do_definition\n            )\n            try:\n                # optimization: skip checking for syntax errors on each foreach\n                # generated stages. We do it once when accessing do_definition.\n                return entry.resolve_stage(skip_checks=True)\n            except ContextError as exc:\n                format_and_raise(exc, f\"stage '{generated}'\", self.relpath)\n\n            # let mypy know that this state is unreachable as format_and_raise\n            # does not return at all (it's not able to understand it for some\n            # reason)\n            raise AssertionError(\"unreachable\")",
  "start_line": 400,
  "end_line": 455,
  "total_lines": 455,
  "is_truncated": false
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "ERROR: Stage 'build@2021' not found inside 'dvc.yaml' file\n",
  "details": {
    "stdout": "Initialized DVC repository.\n\n+---------------------------------------------------------------------+\n|                                                                     |\n|        DVC has enabled anonymous aggregate usage analytics.         |\n|     Read the analytics documentation (and how to opt-out) here:     |\n|             <https://dvc.org/doc/user-guide/analytics>              |\n|                                                                     |\n+---------------------------------------------------------------------+\n\nWhat's next?\n------------\n- Check out the documentation: <https://dvc.org/doc>\n- Get help and share ideas: <https://dvc.org/chat>\n- Star us on GitHub: <https://github.com/iterative/dvc>\n",
    "stderr": "ERROR: Stage 'build@2021' not found inside 'dvc.yaml' file\n",
    "exit_code": 1
  }
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "",
  "details": {
    "stdout": "",
    "stderr": "",
    "exit_code": 1
  }
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "",
  "details": {
    "stdout": "",
    "stderr": "",
    "exit_code": 1
  }
}

{
  "status": "ok",
  "filepath": "dvc/parsing/__init__.py",
  "content": "import logging\nimport os\nfrom collections.abc import Mapping, Sequence\nfrom copy import deepcopy\nfrom typing import (\n    TYPE_CHECKING,\n    Any,\n    Dict,\n    List,\n    NamedTuple,\n    Optional,\n    Tuple,\n    Type,\n    Union,\n)\n\nfrom funcy import collecting, first, isa, join, reraise\n\nfrom dvc.exceptions import DvcException\nfrom dvc.parsing.interpolate import ParseError\nfrom dvc.utils.objects import cached_property\n\nfrom .context import (\n    Context,\n    ContextError,\n    KeyNotInContext,\n    MergeError,\n    Node,\n    VarsAlreadyLoaded,\n)\nfrom .interpolate import (\n    check_recursive_parse_errors,\n    is_interpolated_string,\n    recurse,\n    to_str,\n)\n\nif TYPE_CHECKING:\n    from typing import NoReturn\n\n    from dvc.repo import Repo\n    from dvc.types import DictStrAny\n\n    from .context import SeqOrMap\n\n\nlogger = logging.getLogger(__name__)\n\nSTAGES_KWD = \"stages\"\nVARS_KWD = \"vars\"\nWDIR_KWD = \"wdir\"\nPARAMS_KWD = \"params\"\nFOREACH_KWD = \"foreach\"\nDO_KWD = \"do\"\n\nDEFAULT_PARAMS_FILE = \"params.yaml\"\n\nJOIN = \"@\"\n\n\nclass ResolveError(DvcException):\n    pass\n\n\nclass EntryNotFound(DvcException):\n    pass\n\n\ndef _format_preamble(msg: str, path: str, spacing: str = \" \") -> str:\n    return f\"failed to parse {msg} in '{path}':{spacing}\"\n\n\ndef format_and_raise(exc: Exception, msg: str, path: str) -> \"NoReturn\":\n    spacing = (\n        \"\\n\" if isinstance(exc, (ParseError, MergeError, VarsAlreadyLoaded)) else \" \"\n    )\n    message = _format_preamble(msg, path, spacing) + str(exc)\n\n    # FIXME: cannot reraise because of how we log \"cause\" of the exception\n    # the error message is verbose, hence need control over the spacing\n    _reraise_err(ResolveError, message, from_exc=exc)\n\n\ndef _reraise_err(\n    exc_cls: Type[Exception], *args, from_exc: Optional[Exception] = None\n) -> \"NoReturn\":\n    err = exc_cls(*args)\n    if from_exc and logger.isEnabledFor(logging.DEBUG):\n        raise err from from_exc\n    raise err\n\n\ndef check_syntax_errors(\n    definition: \"DictStrAny\", name: str, path: str, where: str = \"stages\"\n):\n    for key, d in definition.items():\n        try:\n            check_recursive_parse_errors(d)\n        except ParseError as exc:\n            format_and_raise(exc, f\"'{where}.{name}.{key}'\", path)",
  "start_line": 1,
  "end_line": 100,
  "total_lines": 455,
  "is_truncated": false
}

{
  "status": "ok",
  "filepath": "dvc/parsing/__init__.py",
  "occurrences": 1,
  "strategy": "exact",
  "diff": "--- a/dvc/parsing/__init__.py\n+++ b/dvc/parsing/__init__.py\n@@ -392,7 +392,7 @@\n         \"\"\"Convert sequence to Mapping with keys normalized.\"\"\"\n         iterable = self.resolved_iterable\n         if isinstance(iterable, Mapping):\n-            return iterable\n+            return {to_str(k): v for k, v in iterable.items()}\n \n         assert isinstance(iterable, Sequence)\n         if any(map(is_map_or_seq, iterable)):\n",
  "is_truncated": false
}

{
  "status": "ok",
  "stdout": "build@2021:\n\tchanged outs:\n\t\tdeleted:            model-2021.hdfs\nbuild@2022:\n\tchanged outs:\n\t\tdeleted:            model-2022.hdfs\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "stdout": "230:        if accept_group and stages.is_foreach_generated(name):\n249:                foreach are returned.\n308:                - a foreach group name or a stage name in the `dvc.yaml` file.\n309:                - a generated stage name from a foreach group.\n356:        - foreach_group_name or stage_name\n357:        - generated stage name from a foreach group\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "filepath": "dvc/repo/stage.py",
  "content": "        return self.load_one(path=path, name=name)\n\n    def _get_filepath(\n        self, path: Optional[str] = None, name: Optional[str] = None\n    ) -> str:\n        if path:\n            return self.repo.fs.path.realpath(path)\n\n        path = PROJECT_FILE\n        logger.debug(\"Assuming '%s' to be a stage inside '%s'\", name, path)\n        return path\n\n    @staticmethod\n    def _get_group_keys(stages: \"StageLoader\", group: str) -> Iterable[str]:\n        from dvc.parsing import JOIN\n\n        for key in stages:\n            assert isinstance(key, str)\n            if key.startswith(f\"{group}{JOIN}\"):\n                yield key\n\n    def _get_keys(\n        self,\n        stages: \"StageLoader\",\n        name: Optional[str] = None,\n        accept_group: bool = True,\n        glob: bool = False,\n    ) -> Iterable[str]:\n        if not name:\n            return stages.keys()\n        if accept_group and stages.is_foreach_generated(name):\n            return self._get_group_keys(stages, name)\n        if glob:\n            return fnmatch.filter(stages.keys(), name)\n        return [name]\n\n    def load_all(\n        self,\n        path: Optional[str] = None,\n        name: Optional[str] = None,\n        accept_group: bool = True,\n        glob: bool = False,\n    ) -> StageList:\n        \"\"\"Load a list of stages from a file.\n\n        Args:\n            path: if not provided, default `dvc.yaml` is assumed.\n            name: required for `dvc.yaml` files, ignored for `.dvc` files.\n            accept_group: if true, all of the the stages generated from `name`\n                foreach are returned.\n            glob: if true, `name` is considered as a glob, which is\n                used to filter list of stages from the given `path`.\n        \"\"\"\n        from dvc.dvcfile import load_file\n        from dvc.stage.loader import SingleStageLoader, StageLoader\n\n        path = self._get_filepath(path, name)\n        dvcfile = load_file(self.repo, path)\n        # `dvcfile.stages` is not cached\n        stages = dvcfile.stages  # type: ignore[attr-defined]\n\n        if isinstance(stages, SingleStageLoader):\n            stage = stages[name]\n            return [stage]\n\n        assert isinstance(stages, StageLoader)\n        keys = self._get_keys(stages, name, accept_group, glob)\n        return [stages[key] for key in keys]\n\n    def load_one(\n        self, path: Optional[str] = None, name: Optional[str] = None\n    ) -> \"Stage\":\n        \"\"\"Load a single stage from a file.\n\n        Args:\n            path: if not provided, default `dvc.yaml` is assumed.\n            name: required for `dvc.yaml` files, ignored for `.dvc` files.\n        \"\"\"\n        from dvc.dvcfile import load_file\n\n        path = self._get_filepath(path, name)\n        dvcfile = load_file(self.repo, path)\n        stages = dvcfile.stages  # type: ignore[attr-defined]\n\n        return stages[name]\n\n    def load_file(self, path: Optional[str] = None) -> StageList:\n        \"\"\"Load all of the stages from a file.\"\"\"\n        return self.load_all(path)\n\n    def load_glob(self, path: str, expr: Optional[str] = None):\n        \"\"\"Load stages from `path`, filtered with `expr` provided.\"\"\"\n        return self.load_all(path, expr, glob=True)\n\n    def collect(\n        self,\n        target: Optional[str] = None,\n        with_deps: bool = False,\n        recursive: bool = False,\n        graph: Optional[\"DiGraph\"] = None,\n        glob: bool = False,\n    ) -> StageIter:\n        \"\"\"Collect list of stages from the provided target.\n\n        Args:\n            target: if not provided, all of the stages in the graph are\n                returned.\n                Target can be:\n                - a foreach group name or a stage name in the `dvc.yaml` file.\n                - a generated stage name from a foreach group.\n                - a path to `dvc.yaml` or `.dvc` file.\n                - in case of a stage to a dvc.yaml file in a different\n                  directory than current working directory, it can be a path\n                  to dvc.yaml file, followed by a colon `:`, followed by stage\n                  name (eg: `../dvc.yaml:build`).\n                - in case of `recursive`, it can be a path to a directory.\n                - in case of `glob`, it can be a wildcard pattern to match\n                  stages. Example: `build*` for stages in `dvc.yaml` file, or\n                  `../dvc.yaml:build*` for stages in dvc.yaml in a different\n                  directory.\n                  Note that, glob only applies for the stage name, not to the\n                  file, so `**/dvc.yaml:build*` is not possible.\n            with_deps: if true, the stages including their dependencies are\n                returned.\n            recursive: if true and if `target` is a directory, all of the\n                stages inside that directory is returned.\n            graph: graph to use. Defaults to `repo.graph`.\n            glob: Use `target` as a pattern to match stages in a file.\n        \"\"\"\n        if not target:\n            return list(graph) if graph else self.repo.index.stages\n\n        if recursive and self.fs.isdir(target):\n            from dvc.repo.graph import collect_inside_path\n\n            path = self.fs.path.abspath(target)\n            return collect_inside_path(path, graph or self.repo.index.graph)\n\n        stages = self.from_target(target, glob=glob)\n        if not with_deps:\n            return stages\n\n        return _collect_with_deps(stages, graph or self.repo.index.graph)\n\n    def collect_granular(\n        self,\n        target: Optional[str] = None,\n        with_deps: bool = False,\n        recursive: bool = False,\n        graph: Optional[\"DiGraph\"] = None,",
  "start_line": 200,
  "end_line": 349,
  "total_lines": 404,
  "is_truncated": true
}

{
  "status": "ok",
  "stdout": "Initialized DVC repository.\n\n+---------------------------------------------------------------------+\n|                                                                     |\n|        DVC has enabled anonymous aggregate usage analytics.         |\n|     Read the analytics documentation (and how to opt-out) here:     |\n|             <https://dvc.org/doc/user-guide/analytics>              |\n|                                                                     |\n+---------------------------------------------------------------------+\n\nWhat's next?\n------------\n- Check out the documentation: <https://dvc.org/doc>\n- Get help and share ideas: <https://dvc.org/chat>\n- Star us on GitHub: <https://github.com/iterative/dvc>\nbuild@2021:\n\tchanged outs:\n\t\tdeleted:            model-2021.hdfs\nbuild@2022:\n\tchanged outs:\n\t\tdeleted:            model-2022.hdfs\nbuild@foo:\n\tchanged outs:\n\t\tdeleted:            model-foo.hdfs\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "error",
  "error_type": "CommandError",
  "error_message": "Traceback (most recent call last):\n  File \"<stdin>\", line 36, in <module>\n  File \"/workspace/dvc/parsing/__init__.py\", line 415, in resolve_all\n    return join(map(self.resolve_one, self.normalized_iterable))\n  File \"/opt/miniconda3/envs/testbed/lib/python3.9/site-packages/funcy/colls.py\", line 68, in join\n    dest = next(it)\n  File \"/workspace/dvc/parsing/__init__.py\", line 418, in resolve_one\n    return self._each_iter(key)\n  File \"/workspace/dvc/parsing/__init__.py\", line 434, in _each_iter\n    with self.context.set_temporarily(temp_dict, reserve=True):\n  File \"/opt/miniconda3/envs/testbed/lib/python3.9/contextlib.py\", line 119, in __enter__\n    return next(self.gen)\n  File \"/workspace/dvc/parsing/context.py\", line 487, in set_temporarily\n    self.update(temp)\n  File \"/opt/miniconda3/envs/testbed/lib/python3.9/_collections_abc.py\", line 941, in update\n    self[key] = other[key]\n  File \"/workspace/dvc/parsing/context.py\", line 275, in __setitem__\n    return super().__setitem__(key, value)\n  File \"/workspace/dvc/parsing/context.py\", line 191, in __setitem__\n    self.data[key] = self._convert(key, value)\n  File \"/workspace/dvc/parsing/context.py\", line 167, in _convert\n    meta = Meta.update_path(self.meta, key)\nAttributeError: 'MockContext' object has no attribute 'meta'\n",
  "details": {
    "stdout": "Normalized iterable keys: ['2021', '2022', 'foo']\n",
    "stderr": "Traceback (most recent call last):\n  File \"<stdin>\", line 36, in <module>\n  File \"/workspace/dvc/parsing/__init__.py\", line 415, in resolve_all\n    return join(map(self.resolve_one, self.normalized_iterable))\n  File \"/opt/miniconda3/envs/testbed/lib/python3.9/site-packages/funcy/colls.py\", line 68, in join\n    dest = next(it)\n  File \"/workspace/dvc/parsing/__init__.py\", line 418, in resolve_one\n    return self._each_iter(key)\n  File \"/workspace/dvc/parsing/__init__.py\", line 434, in _each_iter\n    with self.context.set_temporarily(temp_dict, reserve=True):\n  File \"/opt/miniconda3/envs/testbed/lib/python3.9/contextlib.py\", line 119, in __enter__\n    return next(self.gen)\n  File \"/workspace/dvc/parsing/context.py\", line 487, in set_temporarily\n    self.update(temp)\n  File \"/opt/miniconda3/envs/testbed/lib/python3.9/_collections_abc.py\", line 941, in update\n    self[key] = other[key]\n  File \"/workspace/dvc/parsing/context.py\", line 275, in __setitem__\n    return super().__setitem__(key, value)\n  File \"/workspace/dvc/parsing/context.py\", line 191, in __setitem__\n    self.data[key] = self._convert(key, value)\n  File \"/workspace/dvc/parsing/context.py\", line 167, in _convert\n    meta = Meta.update_path(self.meta, key)\nAttributeError: 'MockContext' object has no attribute 'meta'\n",
    "exit_code": 1
  }
}

{
  "status": "ok",
  "stdout": " M dvc/parsing/__init__.py\ndiff --git a/dvc/parsing/__init__.py b/dvc/parsing/__init__.py\nindex 1c1e03b..6d3f47a 100644\n--- a/dvc/parsing/__init__.py\n+++ b/dvc/parsing/__init__.py\n@@ -392,7 +392,7 @@ class ForeachDefinition:\n         \"\"\"Convert sequence to Mapping with keys normalized.\"\"\"\n         iterable = self.resolved_iterable\n         if isinstance(iterable, Mapping):\n-            return iterable\n+            return {to_str(k): v for k, v in iterable.items()}\n \n         assert isinstance(iterable, Sequence)\n         if any(map(is_map_or_seq, iterable)):\n",
  "stderr": "",
  "exit_code": 0
}

{
  "status": "ok",
  "patch_size": 538,
  "files_changed": 1
}