fix(dlt): surface underlying error and hint when attaching to pipeline fails

When `sqlmesh init -t dlt --dlt-path <path>` could not attach to the
pipeline, the underlying dlt error was swallowed and replaced with an
uninformative message. `--dlt-path` points to dlt's pipelines working
directory (by default ~/.dlt/pipelines), not the directory containing
the pipeline scripts, which is a common source of confusion.

Now the error includes the original dlt exception, the directory that
was searched, and a hint to omit `--dlt-path` when the pipeline exists
in the default working directory. Also clarifies the `--dlt-path` help
text and docstring accordingly.

Fixes #5660.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: nk <nkwork1234@gmail.com>

Author: nkwork9999

docs/reference/cli.md

@@ -279,7 +279,8 @@ Options:
                        empty.
   --dlt-pipeline TEXT  DLT pipeline for which to generate a SQLMesh project.
                        Use alongside template: dlt
-  --dlt-path TEXT      The directory where the DLT pipeline resides. Use
+  --dlt-path TEXT      The DLT pipelines working directory, where DLT stores
+                       pipeline state (by default ~/.dlt/pipelines). Use
                        alongside template: dlt
   --help               Show this message and exit.
 ```

sqlmesh/cli/main.py

@@ -169,7 +169,7 @@ def cli(
 @click.option(
     "--dlt-path",
     type=str,
-    help="The directory where the DLT pipeline resides. Use alongside template: dlt",
+    help="The DLT pipelines working directory, where DLT stores pipeline state (by default ~/.dlt/pipelines). Use alongside template: dlt",
 )
 @click.pass_context
 @error_handler
@@ -1155,7 +1155,7 @@ def table_name(
 @click.option(
     "--dlt-path",
     type=str,
-    help="The directory where the DLT pipeline resides.",
+    help="The DLT pipelines working directory, where DLT stores pipeline state (by default ~/.dlt/pipelines).",
 )
 @click.pass_context
 @error_handler

sqlmesh/integrations/dlt.py

@@ -22,7 +22,8 @@ def generate_dlt_models_and_settings(
         pipeline_name: The name of the DLT pipeline to attach to.
         dialect: The SQL dialect to use for generating SQLMesh models.
         tables: A list of table names to include.
-        dlt_path: The path to the directory containing the DLT pipelines.
+        dlt_path: The path to the DLT pipelines working directory, where DLT stores
+            pipeline state (by default ~/.dlt/pipelines).
 
     Returns:
         A tuple containing a set of the SQLMesh model definitions, the connection config and the start date.
@@ -34,8 +35,21 @@ def generate_dlt_models_and_settings(
 
     try:
         pipeline = dlt.attach(pipeline_name=pipeline_name, pipelines_dir=dlt_path or "")
-    except CannotRestorePipelineException:
-        raise click.ClickException(f"Could not attach to pipeline {pipeline_name}")
+    except CannotRestorePipelineException as e:
+        from pathlib import Path
+        from dlt.common.pipeline import get_dlt_pipelines_dir
+
+        searched_dir = dlt_path or get_dlt_pipelines_dir()
+        msg = f"Could not attach to pipeline {pipeline_name}.\nSearched in: {searched_dir}\n{e}"
+        if dlt_path and (Path(get_dlt_pipelines_dir()) / pipeline_name).exists():
+            msg += (
+                f"\nHint: A pipeline named '{pipeline_name}' exists in the default pipelines "
+                f"working directory '{get_dlt_pipelines_dir()}'. Note that --dlt-path must "
+                "point to the directory where DLT stores pipeline working state (by default "
+                "~/.dlt/pipelines), not the directory containing your pipeline scripts. "
+                "Try omitting --dlt-path."
+            )
+        raise click.ClickException(msg)
 
     schema = pipeline.default_schema
     dataset = pipeline.dataset_name

tests/cli/test_cli.py

@@ -995,7 +995,7 @@ def test_dlt_pipeline(runner, tmp_path):
         exec(file.read())
 
     # This should fail since it won't be able to locate the pipeline in this path
-    with pytest.raises(ClickException, match=r".*Could not attach to pipeline*"):
+    with pytest.raises(ClickException, match=r".*Could not attach to pipeline*") as excinfo:
         init_example_project(
             tmp_path,
             "duckdb",
@@ -1004,6 +1004,12 @@ def test_dlt_pipeline(runner, tmp_path):
             dlt_path="./dlt2/pipelines",
         )
 
+    # The error should surface where the pipeline was searched for and, since the
+    # pipeline exists in the default working directory, a hint about --dlt-path
+    error_message = str(excinfo.value)
+    assert "Searched in: ./dlt2/pipelines" in error_message
+    assert "Try omitting --dlt-path" in error_message
+
     # By setting the pipelines path where the pipeline directory is located, it should work
     dlt_path = get_dlt_pipelines_dir()
     init_example_project(