Replace regex-based interpolation with character scanner

acceptance/bundle/resource_deps/bad_syntax/out.deploy.direct.txt

@@ -1,3 +1,6 @@
+Warning: invalid variable reference ${resources.volumes.bar.bad..syntax}: invalid path
+  in databricks.yml:11:21
+
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files...
 Deploying resources...
 Updating deployment state...

acceptance/bundle/resource_deps/bad_syntax/out.plan.direct.txt

@@ -1,3 +1,6 @@
+Warning: invalid variable reference ${resources.volumes.bar.bad..syntax}: invalid path
+  in databricks.yml:11:21
+
 create volumes.bar
 create volumes.foo
 

acceptance/bundle/resource_deps/bad_syntax/out.plan.terraform.txt

@@ -1,3 +1,6 @@
+Warning: invalid variable reference ${resources.volumes.bar.bad..syntax}: invalid path
+  in databricks.yml:11:21
+
 Error: exit status 1
 
 Error: Invalid attribute name

acceptance/bundle/resource_deps/bad_syntax/output.txt

@@ -1,5 +1,8 @@
 
 >>> [CLI] bundle validate -o json
+Warning: invalid variable reference ${resources.volumes.bar.bad..syntax}: invalid path
+  in databricks.yml:11:21
+
 {
   "volumes": {
     "bar": {

acceptance/bundle/variables/malformed_reference/databricks.yml

@@ -0,0 +1,6 @@
+bundle:
+  name: "${foo.bar-}"
+
+variables:
+  a:
+    default: hello

acceptance/bundle/variables/malformed_reference/output.txt

@@ -0,0 +1,12 @@
+
+>>> [CLI] bundle validate
+Warning: invalid variable reference ${foo.bar-}: invalid key "bar-"
+  in databricks.yml:2:9
+
+Name: ${foo.bar-}
+Target: default
+Workspace:
+  User: [USERNAME]
+  Path: /Workspace/Users/[USERNAME]/.bundle/${foo.bar-}/default
+
+Found 1 warning

acceptance/bundle/variables/malformed_reference/script

@@ -0,0 +1 @@
+trace $CLI bundle validate

acceptance/bundle/variables/var_in_var/output.txt

@@ -1,9 +1,12 @@
 
 >>> [CLI] bundle validate -o json -t target_x
+Warning: nested variable references are not supported (at position 0)
+  in databricks.yml:14:14
+
 {
   "final": {
-    "default": "hello from foo x",
-    "value": "hello from foo x"
+    "default": "${var.foo_${var.tail}}",
+    "value": "${var.foo_${var.tail}}"
   },
   "foo_x": {
     "default": "hello from foo x",
@@ -20,10 +23,13 @@
 }
 
 >>> [CLI] bundle validate -o json -t target_y
+Warning: nested variable references are not supported (at position 0)
+  in databricks.yml:14:14
+
 {
   "final": {
-    "default": "hi from foo y",
-    "value": "hi from foo y"
+    "default": "${var.foo_${var.tail}}",
+    "value": "${var.foo_${var.tail}}"
   },
   "foo_x": {
     "default": "hello from foo x",

bundle/config/mutator/warn_malformed_references.go

@@ -0,0 +1,47 @@
+package mutator
+
+import (
+	"context"
+
+	"github.com/databricks/cli/bundle"
+	"github.com/databricks/cli/libs/diag"
+	"github.com/databricks/cli/libs/dyn"
+	"github.com/databricks/cli/libs/dyn/dynvar"
+)
+
+type warnMalformedReferences struct{}
+
+// WarnMalformedReferences returns a mutator that emits warnings for strings
+// containing malformed variable references (e.g. "${foo.bar-}").
+func WarnMalformedReferences() bundle.Mutator {
+	return &warnMalformedReferences{}
+}
+
+func (*warnMalformedReferences) Name() string {
+	return "WarnMalformedReferences"
+}
+
+func (*warnMalformedReferences) Validate(ctx context.Context, b *bundle.Bundle) error {
+	return nil
+}
+
+func (*warnMalformedReferences) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics {
+	var diags diag.Diagnostics
+	err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) {
+		_, err := dyn.Walk(root, func(p dyn.Path, v dyn.Value) (dyn.Value, error) {
+			// Only check values with source locations to avoid false positives
+			// from synthesized/computed values.
+			if len(v.Locations()) == 0 {
+				return v, nil
+			}
+			_, _, refDiags := dynvar.NewRefWithDiagnostics(v)
+			diags = diags.Extend(refDiags)
+			return v, nil
+		})
+		return root, err
+	})
+	if err != nil {
+		diags = diags.Extend(diag.FromErr(err))
+	}
+	return diags
+}

bundle/phases/initialize.go

@@ -94,6 +94,10 @@ func Initialize(ctx context.Context, b *bundle.Bundle) {
 		// searches for strings with variable references in them.
 		mutator.RewriteWorkspacePrefix(),
 
+		// Walks the config tree and emits warnings for malformed variable references
+		// (e.g. "${foo.bar-}") before variable resolution occurs.
+		mutator.WarnMalformedReferences(),
+
 		// Reads (dynamic): variables.* (checks if there's a value assigned to variable already or if it has lookup reference)
 		// Updates (dynamic): variables.*.value (sets values from environment variables, variable files, or defaults)
 		// Resolves and sets values for bundle variables in the following order: from environment variables, from variable files and then defaults

design/interpolation-parser.md

@@ -0,0 +1,72 @@
+# Variable Interpolation: Character Scanner Parser
+
+Author: Shreyas Goenka
+Date: 12 March 2026
+
+## Motivation
+
+DABs variable interpolation (`${...}`) was regex-based. This caused:
+
+1. **Silent failures** — `${foo.bar-}` silently treated as literal text with no warning.
+2. **No suggestions** — `${bundle.nme}` produces "reference does not exist" with no hint.
+3. **No escape mechanism** — no way to produce a literal `${` in output.
+4. **No extensibility** — cannot support structured path features like key-value references `tasks[task_key="x"]` that exist in `libs/structs/structpath`.
+
+## Background: How Other Systems Parse `${...}`
+
+| System | Strategy | Escape | Error Quality |
+|--------|----------|--------|---------------|
+| Go `text/template` | State-function lexer | None | Line + template name |
+| HCL2 (Terraform) | Ragel FSM + recursive descent | `$${` → literal `${` | Source range + suggestions |
+| Python f-strings | Mode-stack tokenizer | `{{` → `{` | Line/column |
+| Rust `format!` | Iterator-based descent | `{{`/`}}` | Spans + suggestions |
+| Bash | Char-by-char + depth tracking | `\$` | Line number |
+
+For a syntax as simple as `${path.to.var[0]}` (no nesting, no functions, no
+operators), a full recursive descent parser is overkill. A **two-mode character
+scanner** — the same core pattern used by Go's `text/template` and HCL — gives
+proper error reporting and escape support without the complexity.
+
+## Design Decisions
+
+### Two-mode character scanner
+
+A two-mode scanner (TEXT / REFERENCE) that produces a flat list of tokens.
+No AST, no recursive descent. Easy to port to the Python implementation.
+
+See `libs/interpolation/parse.go`.
+
+### Nested `${` rejection
+
+Nested `${...}` inside a reference (e.g., `${var.foo_${var.tail}}`) is
+rejected as an error. This construct is ambiguous and was never intentionally
+supported — the old regex happened to match only the innermost pair by
+coincidence.
+
+### `\$` escape sequence
+
+`\$` produces a literal `$`, and `\\` produces a literal `\`. This follows
+the same convention used by Bash for escaping `$` and is the least
+surprising option for users working in shell environments.
+
+A standalone `\` before any character other than `$` or `\` is passed
+through as a literal backslash, so existing configurations that happen to
+contain backslashes are not affected.
+
+### Malformed reference warnings
+
+A standalone `WarnMalformedReferences` mutator walks the config tree once
+before variable resolution. It only checks values with source locations
+(`len(v.Locations()) > 0`) to avoid false positives from synthesized values
+(e.g., normalized/computed paths).
+
+### Token-based resolution
+
+The resolver's string interpolation changed from `strings.Replace` (with
+count=1 to avoid double-replacing duplicate refs) to a token concatenation
+loop. Each `TokenRef` maps 1:1 to a resolved value, eliminating the ambiguity.
+
+## Python sync
+
+The Python regex in `python/databricks/bundles/core/_transform.py` needs a
+corresponding update in a follow-up PR.