From a6e76b41ca14933ea7b80a5b74e11552cea7a64c Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Mon, 17 Aug 2026 07:45:34 -0300 Subject: [PATCH 01/30] fix(connection): remove debug log for SQL Server port and instance handling --- core/dbio/connection/connection.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/dbio/connection/connection.go b/core/dbio/connection/connection.go index d1fbb5929..85a3752af 100644 --- a/core/dbio/connection/connection.go +++ b/core/dbio/connection/connection.go @@ -984,7 +984,6 @@ func (c *Connection) setURL() (err error) { switch { case port_ok && instance_ok: template += ":{port}/{instance}" - g.Debug("SQL Server: port %s and instance %s are both set. The driver uses the port and ignores the instance name.", c.Data["port"], c.Data["instance"]) if buildingURL && cast.ToInt(c.Data["port"]) == 1433 { g.Warn("SQL Server: port 1433 and instance %s are both set. The driver uses port 1433 and ignores the instance name. For a named instance, set `port` to the instance TCP port or omit `port`.", c.Data["instance"]) } From e50842fc2f99ae8fe59329cf04842337246850c1 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Mon, 17 Aug 2026 16:58:34 -0300 Subject: [PATCH 02/30] fix(run): honor chunking, retries and threads in ad-hoc runs Ad-hoc runs (without -r) silently ignored chunk_size, SLING_RETRIES and SLING_THREADS because only the replication path calls ProcessChunks. Add RequiresPro, WithRetries and WithThreads helpers to detect these settings, and route ad-hoc tasks through the replication path when a pro feature is used. Stdin/stdout runs stay on the task path and now warn that chunking, threads and retries are unsupported there. Add CLI tests covering both the chunked ad-hoc run and the stdin/stdout guard case. --- cmd/sling/sling_run.go | 13 +++++++++++-- core/sling/config.go | 14 ++++++++++++++ tests/suite.cli.yaml | 28 ++++++++++++++++++++++++++++ 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/cmd/sling/sling_run.go b/cmd/sling/sling_run.go index bc48eb757..beb81cc72 100755 --- a/cmd/sling/sling_run.go +++ b/cmd/sling/sling_run.go @@ -311,8 +311,17 @@ runReplication: // run task, add replication config for md5 rc := cfg.AsReplication() - // run as replication is stream is wildcard - if cfg.HasWildcard() { + // stdin only counts as the source if no source connection is given, + // since Options.StdIn is also true for any non-interactive shell + isStdInOut := (cfg.Options.StdIn && cfg.Source.Conn == "") || cfg.Options.StdOut + + if isStdInOut && (cfg.WithRetries() || cfg.WithChunking() || cfg.WithThreads()) { + g.Warn("chunking, threads and retries are not supported with stdin/stdout") + } + + // run as replication if stream is wildcard, or if a pro feature + // needs the replication path + if cfg.HasWildcard() || (cfg.RequiresPro() && !isStdInOut) { replicationCfgPath = path.Join(env.GetTempFolder(), g.NewTsID("replication.temp")+".json") err = os.WriteFile(replicationCfgPath, []byte(g.Marshal(rc)), 0775) if err != nil { diff --git a/core/sling/config.go b/core/sling/config.go index 910bb6396..91ac3bd76 100644 --- a/core/sling/config.go +++ b/core/sling/config.go @@ -495,6 +495,20 @@ func (cfg *Config) HasWildcard() bool { return false } +// RequiresPro returns true if the config uses a pro feature which +// the replication path applies. An ad-hoc task ignores these. +func (cfg *Config) RequiresPro() bool { + return cfg.WithChunking() || cfg.WithRetries() || cfg.WithThreads() +} + +func (cfg *Config) WithRetries() bool { + return cast.ToInt(cfg.Env["SLING_RETRIES"]) > 0 || cast.ToInt(os.Getenv("SLING_RETRIES")) > 0 +} + +func (cfg *Config) WithThreads() bool { + return cast.ToInt(cfg.Env["SLING_THREADS"]) > 1 || cast.ToInt(os.Getenv("SLING_THREADS")) > 1 +} + func (cfg *Config) AsReplication() (rc ReplicationConfig) { rc = ReplicationConfig{ Source: cfg.Source.Conn, diff --git a/tests/suite.cli.yaml b/tests/suite.cli.yaml index 6cf8468be..7d456c2e4 100644 --- a/tests/suite.cli.yaml +++ b/tests/suite.cli.yaml @@ -2692,3 +2692,31 @@ sling run -d -r tests/replications/r.103.duckdb_low_mem_buffer.yaml output_contains: - 'SUCCESS: 5MB row loaded into duckdb with 1GB memory_limit' + +# Chunking with ad-hoc CLI flags (no -r) was silently ignored, since only the +# replication path calls ProcessChunks. The task path now routes to the +# replication path when a pro feature needs it. +- id: 322 + name: 'ad-hoc CLI flags trigger chunking' + run: | + sling conns exec POSTGRES --limit 0 "drop table if exists public.cli_chunk_test" + sling conns exec POSTGRES --limit 0 "create table public.cli_chunk_test as select g as id, 'v'||g as val from generate_series(1,1000) g" + sling run -d --src-conn POSTGRES --src-stream 'public.cli_chunk_test' --tgt-conn DUCKDB --tgt-object 'main.cli_chunk_test' --mode full-refresh --src-options '{"chunk_size": "250"}' --update-key id --primary-key id + sling conns exec POSTGRES --limit 0 "drop table if exists public.cli_chunk_test" + output_contains: + - 'determined 4 chunks' + - '(part-004)' + - 'execution succeeded' + +# Guard: an ad-hoc run with no pro feature must stay on the task path, +# and stdout must keep streaming when chunking is requested. +- id: 323 + name: 'ad-hoc CLI without pro features keeps task path' + run: | + sling conns exec POSTGRES --limit 0 "drop table if exists public.cli_nochunk_test" + sling conns exec POSTGRES --limit 0 "create table public.cli_nochunk_test as select g as id from generate_series(1,10) g" + sling run -d --src-conn POSTGRES --src-stream 'public.cli_nochunk_test' --stdout --src-options '{"chunk_size": "5"}' + sling conns exec POSTGRES --limit 0 "drop table if exists public.cli_nochunk_test" + output_contains: + - 'chunking, threads and retries are not supported with stdin/stdout' + - 'execution succeeded' From acbd38335731cdc1f727999ed1e5ed7c8ab9beb8 Mon Sep 17 00:00:00 2001 From: Niccolo Cantu Date: Thu, 20 Aug 2026 11:36:14 +0200 Subject: [PATCH 03/30] fix(filesys): force request checksum calculation --- core/dbio/filesys/fs_s3.go | 4 +- go.mod | 89 +++---------------- go.sum | 169 ++++--------------------------------- 3 files changed, 32 insertions(+), 230 deletions(-) diff --git a/core/dbio/filesys/fs_s3.go b/core/dbio/filesys/fs_s3.go index 134fda3f3..8f2e316f5 100644 --- a/core/dbio/filesys/fs_s3.go +++ b/core/dbio/filesys/fs_s3.go @@ -512,7 +512,9 @@ func (fs *S3FileSysClient) Write(uri string, reader io.Reader) (bw int64, err er } svc := s3.NewFromConfig(fs.getConfig()) - uploader := manager.NewUploader(svc) + uploader := manager.NewUploader(svc, func(d *manager.Uploader) { + d.RequestChecksumCalculation = fs.awsConfig.RequestChecksumCalculation + }) uploader.Concurrency = fs.Context().Wg.Limit // Create pipe to get bytes written. diff --git a/go.mod b/go.mod index b2d17a1b1..8ca22aacd 100644 --- a/go.mod +++ b/go.mod @@ -19,12 +19,12 @@ require ( github.com/apache/arrow-go/v18 v18.5.0 github.com/apache/iceberg-go v0.3.0 github.com/aws/aws-sdk-go-v2 v1.41.5 - github.com/aws/aws-sdk-go-v2/config v1.29.15 - github.com/aws/aws-sdk-go-v2/credentials v1.17.68 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.69 + github.com/aws/aws-sdk-go-v2/config v1.31.2 + github.com/aws/aws-sdk-go-v2/credentials v1.18.6 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0 github.com/aws/aws-sdk-go-v2/service/athena v1.51.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 - github.com/aws/aws-sdk-go-v2/service/sts v1.33.20 + github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 github.com/aws/smithy-go v1.24.2 github.com/bits-and-blooms/bloom/v3 v3.7.0 github.com/c-bata/go-prompt v0.2.6 @@ -44,7 +44,6 @@ require ( github.com/google/uuid v1.6.0 github.com/integrii/flaggy v1.5.2 github.com/itchyny/gojq v0.12.18 - github.com/itchyny/timefmt-go v0.1.7 github.com/jackc/pgx/v5 v5.9.2 github.com/jaswdr/faker v1.19.1 github.com/jedib0t/go-pretty v4.3.0+incompatible @@ -54,14 +53,12 @@ require ( github.com/jmoiron/sqlx v1.3.3 github.com/json-iterator/go v1.1.12 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 - github.com/kardianos/service v1.2.4 github.com/klauspost/compress v1.18.5 github.com/kshedden/datareader v0.0.0-20210325133423-816b6ffdd011 github.com/labstack/echo/v4 v4.10.2 github.com/lib/pq v1.10.9 github.com/linkedin/goavro/v2 v2.12.0 github.com/maja42/goval v1.4.0 - github.com/mark3labs/mcp-go v0.57.0 github.com/mattn/go-isatty v0.0.20 github.com/mattn/go-sqlite3 v1.14.28 github.com/microsoft/go-mssqldb v1.9.3 @@ -79,8 +76,6 @@ require ( github.com/shirou/gopsutil/v3 v3.24.4 github.com/shopspring/decimal v1.4.0 github.com/sijms/go-ora/v2 v2.8.24 - github.com/slingdata-io/godbc v0.0.9 - github.com/slingdata-io/sling v0.0.0-20260715135102-01cd9a07a3c8 github.com/snowflakedb/gosnowflake v1.17.1 github.com/spf13/cast v1.7.1 github.com/stretchr/testify v1.11.1 @@ -91,11 +86,6 @@ require ( github.com/xuri/excelize/v2 v2.9.1 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a go.mongodb.org/mongo-driver v1.14.0 - go.opentelemetry.io/otel v1.43.0 - go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 - go.opentelemetry.io/otel/log v0.14.0 - go.opentelemetry.io/otel/sdk v1.43.0 - go.opentelemetry.io/otel/sdk/log v0.14.0 golang.org/x/crypto v0.52.0 golang.org/x/net v0.55.0 golang.org/x/oauth2 v0.34.0 @@ -126,7 +116,6 @@ require ( filippo.io/edwards25519 v1.1.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect - github.com/AlecAivazis/survey/v2 v2.3.7 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/to v0.4.1 // indirect @@ -136,7 +125,6 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect - github.com/Microsoft/go-winio v0.6.2 // indirect github.com/andybalholm/brotli v1.2.0 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect @@ -146,7 +134,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go v1.55.6 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect @@ -156,32 +144,23 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.22.0 // indirect - github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect github.com/cockroachdb/apd/v3 v3.2.1 // indirect - github.com/coder/websocket v1.8.14 // indirect github.com/containerd/console v1.0.5 // indirect - github.com/containerd/errdefs v1.0.0 // indirect - github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/coreos/go-oidc/v3 v3.5.0 // indirect github.com/creasty/defaults v1.8.0 // indirect - github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/disintegration/imaging v1.6.2 // indirect - github.com/distribution/reference v0.6.0 // indirect github.com/dnephin/pflag v1.0.7 // indirect github.com/docker/docker v28.5.2+incompatible // indirect github.com/docker/go-connections v0.7.0 // indirect - github.com/docker/go-units v0.5.0 // indirect - github.com/domodwyer/mailyak/v3 v3.6.2 // indirect github.com/dvsekhvalnov/jose2go v1.7.0 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect @@ -192,34 +171,25 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/gabriel-vasile/mimetype v1.4.7 // indirect - github.com/ganigeorgiev/fexpr v0.4.1 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect - github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect - github.com/go-git/go-billy/v5 v5.5.0 // indirect - github.com/go-git/go-git/v5 v5.11.0 // indirect github.com/go-jose/go-jose/v3 v3.0.5 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-mysql-org/go-mysql v1.13.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/errors v0.21.0 // indirect github.com/go-openapi/strfmt v0.22.0 // indirect - github.com/go-ozzo/ozzo-validation/v4 v4.3.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/goccy/go-yaml v1.17.1 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect - github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/flatbuffers v25.9.23+incompatible // indirect - github.com/google/jsonschema-go v0.4.2 // indirect - github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/wire v0.6.0 // indirect @@ -227,7 +197,6 @@ require ( github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/gookit/color v1.5.4 // indirect github.com/gorilla/websocket v1.5.3 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hamba/avro/v2 v2.30.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -236,12 +205,10 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/imdario/mergo v0.3.16 // indirect - github.com/jackc/pgio v1.0.0 // indirect - github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 // indirect + github.com/itchyny/timefmt-go v0.1.7 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect @@ -251,16 +218,13 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jpillora/backoff v1.0.0 // indirect - github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect - github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 // indirect github.com/labstack/gommon v0.4.0 // indirect github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect - github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect github.com/mattn/go-tty v0.0.3 // indirect @@ -268,10 +232,7 @@ require ( github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect - github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect - github.com/moby/moby/api v1.55.0 // indirect - github.com/moby/moby/client v0.5.1 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -279,45 +240,29 @@ require ( github.com/mtibben/percent v0.2.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect - github.com/nalgeon/redka v0.5.2 // indirect - github.com/nats-io/nats.go v1.36.0 // indirect - github.com/nats-io/nkeys v0.4.7 // indirect - github.com/nats-io/nuid v1.0.1 // indirect - github.com/ncruces/go-strftime v0.1.9 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/onsi/gomega v1.37.0 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect - github.com/opencontainers/image-spec v1.1.1 // indirect github.com/paulmach/orb v0.11.1 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec // indirect - github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a // indirect - github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/term v1.2.0-beta.2 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/pocketbase/dbx v1.11.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/pterm/pterm v0.12.82 // indirect github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect - github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.4 // indirect - github.com/robfig/cron/v3 v3.0.1 // indirect - github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/segmentio/encoding v0.4.0 // indirect github.com/shirou/gopsutil/v4 v4.25.1 // indirect github.com/shoenig/go-m1cpu v0.2.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect - github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect - github.com/slingdata-io/pocketbase v0.22.136 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/substrait-io/substrait v0.75.0 // indirect @@ -346,7 +291,6 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/xuri/efp v0.0.1 // indirect github.com/xuri/nfp v0.0.1 // indirect - github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect @@ -354,13 +298,11 @@ require ( go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect - go.opentelemetry.io/proto/otlp v1.9.0 // indirect - go.uber.org/atomic v1.11.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.0 // indirect gocloud.dev v0.41.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/image v0.38.0 // indirect @@ -383,19 +325,12 @@ require ( gopkg.in/mattn/go-colorable.v0 v0.1.0 // indirect gopkg.in/mattn/go-isatty.v0 v0.0.4 // indirect gopkg.in/mattn/go-runewidth.v0 v0.0.4 // indirect - gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect - gopkg.in/warnings.v0 v0.1.2 // indirect gotest.tools/gotestsum v1.8.2 // indirect - modernc.org/libc v1.66.10 // indirect - modernc.org/mathutil v1.7.1 // indirect - modernc.org/memory v1.11.0 // indirect - modernc.org/sqlite v1.42.2 // indirect + gotest.tools/v3 v3.5.2 // indirect ) // replace github.com/slingdata-io/golyglot => ../golyglot -replace github.com/slingdata-io/sling => ../sling - replace github.com/apache/iceberg-go => github.com/flarco/iceberg-go v0.0.0-20260105175128-f16b74585ee2 // replace github.com/apache/iceberg-go => ../iceberg-go @@ -409,5 +344,3 @@ replace github.com/apache/arrow-adbc/go/adbc => github.com/slingdata-io/arrow-ad // replace github.com/apache/arrow-adbc/go/adbc => ../arrow-adbc/go/adbc replace github.com/gocql/gocql => github.com/scylladb/gocql v1.18.0 - -replace github.com/flarco/g => ../g diff --git a/go.sum b/go.sum index 516af17d6..11d23664d 100644 --- a/go.sum +++ b/go.sum @@ -120,8 +120,6 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= -github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= -github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= @@ -151,7 +149,6 @@ github.com/apache/thrift v0.23.0 h1:wKR6YnefQSEnxpEfmgTPuJibNG4bF0p2TK34tHLWi3s= github.com/apache/thrift v0.23.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= -github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= @@ -161,14 +158,14 @@ github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.29.15 h1:I5XjesVMpDZXZEZonVfjI12VNMrYa38LtLnw4NtY5Ss= -github.com/aws/aws-sdk-go-v2/config v1.29.15/go.mod h1:tNIp4JIPonlsgaO5hxO372a6gjhN63aSWl2GVl5QoBQ= -github.com/aws/aws-sdk-go-v2/credentials v1.17.68 h1:cFb9yjI02/sWHBSYXAtkamjzCuRymvmeFmt0TC0MbYY= -github.com/aws/aws-sdk-go-v2/credentials v1.17.68/go.mod h1:H6E+jBzyqUu8u0vGaU6POkK3P0NylYEeRZ6ynBpMqIk= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.69 h1:6VFPH/Zi9xYFMJKPQOX5URYkQoXRWeJ7V/7Y6ZDYoms= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.69/go.mod h1:GJj8mmO6YT6EqgduWocwhMoxTLFitkhIrK+owzrYL2I= +github.com/aws/aws-sdk-go-v2/config v1.31.2 h1:NOaSZpVGEH2Np/c1toSeW0jooNl+9ALmsUTZ8YvkJR0= +github.com/aws/aws-sdk-go-v2/config v1.31.2/go.mod h1:17ft42Yb2lF6OigqSYiDAiUcX4RIkEMY6XxEMJsrAes= +github.com/aws/aws-sdk-go-v2/credentials v1.18.6 h1:AmmvNEYrru7sYNJnp3pf57lGbiarX4T9qU/6AZ9SucU= +github.com/aws/aws-sdk-go-v2/credentials v1.18.6/go.mod h1:/jdQkh1iVPa01xndfECInp1v1Wnp70v3K4MvtlLGVEc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 h1:lpdMwTzmuDLkgW7086jE94HweHCqG+uOJwHf3LZs7T0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4/go.mod h1:9xzb8/SV62W6gHQGC/8rrvgNXU6ZoYM3sAIJCIrXJxY= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0 h1:2FFgK3oFA8PTNBjprLFfcmkgg7U9YuSimBvR64RUmiA= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0/go.mod h1:xdxj6nC1aU/jAO80RIlIj3fU40MOSqutEA9N2XFct04= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= @@ -191,17 +188,16 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWUR github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= -github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.20 h1:oIaQ1e17CSKaWmUTu62MtraRWVIosn/iONMuZt0gbqc= -github.com/aws/aws-sdk-go-v2/service/sts v1.33.20/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= +github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 h1:ve9dYBB8CfJGTFqcQ3ZLAAb/KXWgYlgu/2R2TZL2Ko0= +github.com/aws/aws-sdk-go-v2/service/sso v1.28.2/go.mod h1:n9bTZFZcBa9hGGqVz3i/a6+NG0zmZgtkB9qVVFDqPA8= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 h1:pd9G9HQaM6UZAZh19pYOkpKSQkyQQ9ftnl/LttQOcGI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2/go.mod h1:eknndR9rU8UpE/OmFpqU78V1EcXPKFTTm5l/buZYgvM= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 h1:iV1Ko4Em/lkJIsoKyGfc0nQySi+v0Udxr6Igq+y9JZc= +github.com/aws/aws-sdk-go-v2/service/sts v1.38.0/go.mod h1:bEPcjW7IbolPfK67G1nilqWyoxYMSPrDiIQ3RdIdKgo= github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/awsdocs/aws-doc-sdk-examples/gov2/testtools v0.0.0-20250407191926-092f3e54b837 h1:8eMceEa0ib+nqJuGsyowuZaVBVAr685oK6WrNIit+0g= github.com/awsdocs/aws-doc-sdk-examples/gov2/testtools v0.0.0-20250407191926-092f3e54b837/go.mod h1:9Oj/8PZn3D5Ftp/Z1QWrIEFE0daERMqfJawL9duHRfc= -github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -235,8 +231,6 @@ github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/T github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg= github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= -github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= -github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/compose-spec/compose-go/v2 v2.6.0 h1:/+oBD2ixSENOeN/TlJqWZmUak0xM8A7J08w/z661Wd4= github.com/compose-spec/compose-go/v2 v2.6.0/go.mod h1:vPlkN0i+0LjLf9rv52lodNMUTJF5YHVfHVGLLIP67NA= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= @@ -266,13 +260,8 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= -github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= -github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= -github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= -github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= -github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -283,12 +272,8 @@ github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMS github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= -github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= -github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= -github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= -github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= github.com/docker/buildx v0.22.0 h1:pGTcGZa+kxpYUlM/6ACsp1hXhkEDulz++RNXPdE8Afk= @@ -313,8 +298,6 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8= -github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -353,12 +336,12 @@ github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fergusstrange/embedded-postgres v1.31.0 h1:JmRxw2BcPRcU141nOEuGXbIU6jsh437cBB40rmftZSk= -github.com/fergusstrange/embedded-postgres v1.31.0/go.mod h1:w0YvnCgf19o6tskInrOOACtnqfVlOvluz3hlNLY7tRk= github.com/flarco/bigquery v0.0.9 h1:WfxO6XuuHZTJV+55Bq24FhdHYpmAOzgVk9xOcJpEecY= github.com/flarco/bigquery v0.0.9/go.mod h1:IpRSw4quaXxHjFyDSXUo7B6v+XcNF2pSmnNfeqXa/gM= github.com/flarco/databricks-sql-go v0.0.0-20250613120556-51f7c1f3b4ad h1:z5mgsXmNXsgskClg/s6zelILFihJTyK6x7+zX1jUgyU= github.com/flarco/databricks-sql-go v0.0.0-20250613120556-51f7c1f3b4ad/go.mod h1:mnsep6/uctUEgRONJy1pQdb17s/lRtqgcZZZV/WPdkc= +github.com/flarco/g v0.1.178 h1:7CTHpePoj+4z4/Jz2kGTNYuYfRZ4w4CNMnXqZvSk2PQ= +github.com/flarco/g v0.1.178/go.mod h1:Ho26Asm6DWBEcBW3Hs8q8ntdWDc/oe9wdF24m8jYSuQ= github.com/flarco/iceberg-go v0.0.0-20260105175128-f16b74585ee2 h1:k01grALOonWx3nBt0LfbUqY39kjaIXaiNP+2LHNuJYY= github.com/flarco/iceberg-go v0.0.0-20260105175128-f16b74585ee2/go.mod h1:S7rRApCtYThojNe57sOBhV/ZmH7EMfgEyou4TBEdQHU= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -378,8 +361,6 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= -github.com/ganigeorgiev/fexpr v0.4.1 h1:hpUgbUEEWIZhSDBtf4M9aUNfQQ0BZkGRaMePy7Gcx5k= -github.com/ganigeorgiev/fexpr v0.4.1/go.mod h1:RyGiGqmeXhEQ6+mlGdnUleLHgtzzu/VGO2WtJkF5drE= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -391,12 +372,6 @@ github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= -github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= -github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= -github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= -github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4= -github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-jose/go-jose/v3 v3.0.5 h1:BLLJWbC4nMZOfuPVxoZIxeYsn6Nl2r1fITaJ78UQlVQ= github.com/go-jose/go-jose/v3 v3.0.5/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= @@ -407,8 +382,6 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-mysql-org/go-mysql v1.13.0 h1:Hlsa5x1bX/wBFtMbdIOmb6YzyaVNBWnwrb8gSIEPMDc= -github.com/go-mysql-org/go-mysql v1.13.0/go.mod h1:FQxw17uRbFvMZFK+dPtIPufbU46nBdrGaxOw0ac9MFs= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/errors v0.21.0 h1:FhChC/duCnfoLj1gZ0BgaBmzhJC2SL/sJr8a2vAobSY= @@ -421,9 +394,6 @@ github.com/go-openapi/strfmt v0.22.0 h1:Ew9PnEYc246TwrEspvBdDHS4BVKXy/AOVsfqGDgA github.com/go-openapi/strfmt v0.22.0/go.mod h1:HzJ9kokGIju3/K6ap8jL+OlGAbjpSv27135Yr9OivU4= github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-ozzo/ozzo-validation/v4 v4.3.0 h1:byhDUpfEwjsVQb1vBunvIjh2BHQ9ead57VkAEY4V+Es= -github.com/go-ozzo/ozzo-validation/v4 v4.3.0/go.mod h1:2NKgrcHl3z6cJs+3Oo940FPRiTzuqKbvfrL2RxCj6Ew= -github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= @@ -443,8 +413,6 @@ github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeH github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= -github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= @@ -505,15 +473,11 @@ github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= -github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= -github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= @@ -544,6 +508,7 @@ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/z github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway v1.5.0 h1:WcmKMm43DR7RdtlkEXQJyo5ws8iTp98CyhCCbOHMvNI= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= @@ -569,8 +534,6 @@ github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKe github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= -github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= -github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= @@ -592,8 +555,6 @@ github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= -github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 h1:+PJCokZ2BhyDKlncScmiNzBwqOx+yH1i8xRlWN/wn6A= -github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510/go.mod h1:UzTJ5Jjuf4O9hYWW+HYVwVldYz9J7CaePW0iuNJkrPQ= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= @@ -610,8 +571,6 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaswdr/faker v1.19.1 h1:xBoz8/O6r0QAR8eEvKJZMdofxiRH+F0M/7MU9eNKhsM= github.com/jaswdr/faker v1.19.1/go.mod h1:x7ZlyB1AZqwqKZgyQlnqEG8FDptmHlncA5u2zY/yi6w= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= -github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -653,8 +612,6 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= -github.com/kardianos/service v1.2.4 h1:XNlGtZOYNx2u91urOdg/Kfmc+gfmuIo1Dd3rEi2OgBk= -github.com/kardianos/service v1.2.4/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= @@ -687,8 +644,6 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.10.2 h1:n1jAhnq/elIFTHr1EYpiYtyKgx4RW9ccVgkqByZaN2M= github.com/labstack/echo/v4 v4.10.2/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k= -github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 h1:FwuzbVh87iLiUQj1+uQUsuw9x5t9m5n5g7rG7o4svW4= -github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61/go.mod h1:paQfF1YtHe+GrGg5fOgjsjoCX/UKDr9bc1DoWpZfns8= github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -708,11 +663,6 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maja42/goval v1.4.0 h1:tlX0X+GvjKzWW2Q6qzWwL4Av2KV1bLtzxwzgxiiwEPc= github.com/maja42/goval v1.4.0/go.mod h1:LDMwF8ocOwIsMZdwoyHC/3UpV8ABDwEzalxkVV2z/rI= -github.com/mark3labs/mcp-go v0.57.0 h1:jzWKyCzdWnwnZt05cvcQQ+ngiUl2RnixXJa7Kj4qP1E= -github.com/mark3labs/mcp-go v0.57.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas= -github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= -github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -743,7 +693,6 @@ github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxU github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= @@ -767,10 +716,6 @@ github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8 github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= -github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= -github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= -github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw= -github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= @@ -812,14 +757,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= -github.com/nalgeon/redka v0.5.2 h1:CX71v88kYj55EwJ10zq7U2eJdH0xcLAIjvKFvUMoM0o= -github.com/nalgeon/redka v0.5.2/go.mod h1:vLxjY3XS9IwBID2YEFWeeMiN4Ar/DtKd4JW62JTAxuU= -github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU= -github.com/nats-io/nats.go v1.36.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= -github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= -github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= -github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= @@ -831,8 +768,6 @@ github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= -github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -855,13 +790,8 @@ github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9F github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec h1:3EiGmeJWoNixU+EwllIn26x6s4njiWRXewdx2zlYa84= github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= -github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a h1:WIhmJBlNGmnCWH6TLMdZfNEDaiU8cFpZe3iaqDbQ0M8= -github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a/go.mod h1:ORfBOFp1eteu2odzsyaxI+b8TzJwgjwyQcGhI+9SfEA= -github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d h1:3Ej6eTuLZp25p3aH/EXdReRHY12hjZYs3RrGp7iLdag= -github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d/go.mod h1:+8feuexTKcXHZF/dkDfvCwEyBAmgb4paFc3/WeYV2eE= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -876,8 +806,6 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pocketbase/dbx v1.11.0 h1:LpZezioMfT3K4tLrqA55wWFw1EtH1pM4tzSVa7kgszU= -github.com/pocketbase/dbx v1.11.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -920,8 +848,6 @@ github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTK github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= -github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= -github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= @@ -930,8 +856,6 @@ github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= -github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/scylladb/gocql v1.18.0 h1:bmaMHNOUJyu0GHnTsVv0RRUAquQoTtYjCL/9jTxrg9Q= github.com/scylladb/gocql v1.18.0/go.mod h1:PZU+XJQ3fDymccIlTacmTdO+aTGGDSgXF1hw+yULUMk= github.com/secure-systems-lab/go-securesystemslib v0.4.0 h1:b23VGrQhTA8cN2CbBw7/FulN9fTtqYUdS5+Oxzt+DUE= @@ -943,9 +867,8 @@ github.com/segmentio/encoding v0.4.0/go.mod h1:/d03Cd8PoaDeceuhUUUQWjU0KhWjrmYrW github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQEo87pStk/a99dzIO1mM9KxIyLPGTU= github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc= github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= @@ -992,10 +915,6 @@ github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EE github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/slingdata-io/arrow-adbc/go/adbc v0.0.0-20260806214312-6da3e7189c98 h1:HT0E/6EiVUe8wew4uIgYEpUYPHcswmGkxpbbqFges3g= github.com/slingdata-io/arrow-adbc/go/adbc v0.0.0-20260806214312-6da3e7189c98/go.mod h1:ikEb6zQgczMp7Alx/RYGaBEn/Mwwz1kfXbirnkbgWUo= -github.com/slingdata-io/godbc v0.0.9 h1:wh5MSI6l+eyTQ8pSbQcO4j195HwBXrVmNYRnHaGJKK8= -github.com/slingdata-io/godbc v0.0.9/go.mod h1:oBLg0zDZSK1BhLpcNhZjsvCLdfvB4OiTReAreXCrb3M= -github.com/slingdata-io/pocketbase v0.22.136 h1:RtAvPvYdK0qm9EB1r8GzNeEfSiqDK+tV8jyxwbpKKBA= -github.com/slingdata-io/pocketbase v0.22.136/go.mod h1:RYAdoMZtW+3OIgKqg+YhgWGIiwjtcBHGxRcVF2+1klA= github.com/snowflakedb/gosnowflake v1.17.1 h1:sBYExPDRv6hHF7fCqeXMT745L326Byw/cROxvCiEJzo= github.com/snowflakedb/gosnowflake v1.17.1/go.mod h1:TaHvQGh9MA2lopZZMm1AvvENDfwcnKtuskIr1e6Fpic= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= @@ -1117,8 +1036,6 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= -github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xo/dburl v0.3.0 h1:KGkeJB/oQhY/DeeJoYl/1+pNE/JnF6ouAuA8nzpQEQ8= github.com/xo/dburl v0.3.0/go.mod h1:TM8VMBT+LWqC3MBOulZjb8FAthcvZq0t/qvDLwS6skU= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= @@ -1132,8 +1049,6 @@ github.com/xuri/nfp v0.0.1 h1:MDamSGatIvp8uOmDP8FnmjuQpu90NzdJxo7242ANR9Q= github.com/xuri/nfp v0.0.1/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= -github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= -github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= @@ -1167,8 +1082,6 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0 h1:FZ6ei8GFW7kyPYdxJaV2rgI6M+4tvZzhYsQ2wgyVC08= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0/go.mod h1:MdEu/mC6j3D+tTEfvI15b5Ci2Fn7NneJ71YMoiS3tpI= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0 h1:ZsXq73BERAiNuuFXYqP4MR5hBrjXfMGSO+Cx7qoOZiM= @@ -1181,16 +1094,10 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 h1:Ckwye go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0/go.mod h1:teIFJh5pW2y+AN7riv6IBPX2DuesS3HgP39mwOspKwU= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= -go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= -go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= -go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= -go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= -go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= -go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= @@ -1198,22 +1105,10 @@ go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLh go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= -go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= -go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= -go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= -go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= @@ -1240,7 +1135,6 @@ golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGb golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= -golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE= golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1388,7 +1282,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= @@ -1410,7 +1303,6 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1440,7 +1332,6 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -1508,10 +1399,6 @@ gopkg.in/mattn/go-isatty.v0 v0.0.4 h1:NtS1rQGQr4IaFWBGz4Cz4BhB///gyys4gDVtKA7hIs gopkg.in/mattn/go-isatty.v0 v0.0.4/go.mod h1:wt691ab7g0X4ilKZNmMII3egK0bTxl37fEn/Fwbd8gc= gopkg.in/mattn/go-runewidth.v0 v0.0.4 h1:r0P71TnzQDlNIcizCqvPSSANoFa3WVGtcNJf3TWurcY= gopkg.in/mattn/go-runewidth.v0 v0.0.4/go.mod h1:BmXejnxvhwdaATwiJbB1vZ2dtXkQKZGu9yLFCZb4msQ= -gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= -gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= -gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= -gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1551,34 +1438,14 @@ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7F k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= -modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= -modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A= -modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q= -modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= -modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= -modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= -modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= -modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= -modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A= modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= -modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= -modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= -modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= -modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= modernc.org/sqlite v1.42.2 h1:7hkZUNJvJFN2PgfUdjni9Kbvd4ef4mNLOu0B9FGxM74= modernc.org/sqlite v1.42.2/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8= -modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= -modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= -modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= -modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= -pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= -pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= From dc26c7de021f2f4c9988b1635ed01853b29ead2b Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Thu, 20 Aug 2026 08:42:35 -0300 Subject: [PATCH 04/30] revert go.mod and go.sum changes --- go.mod | 89 ++++++++++++++++++++++++++---- go.sum | 169 +++++++++++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 229 insertions(+), 29 deletions(-) diff --git a/go.mod b/go.mod index 8ca22aacd..b2d17a1b1 100644 --- a/go.mod +++ b/go.mod @@ -19,12 +19,12 @@ require ( github.com/apache/arrow-go/v18 v18.5.0 github.com/apache/iceberg-go v0.3.0 github.com/aws/aws-sdk-go-v2 v1.41.5 - github.com/aws/aws-sdk-go-v2/config v1.31.2 - github.com/aws/aws-sdk-go-v2/credentials v1.18.6 - github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0 + github.com/aws/aws-sdk-go-v2/config v1.29.15 + github.com/aws/aws-sdk-go-v2/credentials v1.17.68 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.69 github.com/aws/aws-sdk-go-v2/service/athena v1.51.0 github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 - github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 + github.com/aws/aws-sdk-go-v2/service/sts v1.33.20 github.com/aws/smithy-go v1.24.2 github.com/bits-and-blooms/bloom/v3 v3.7.0 github.com/c-bata/go-prompt v0.2.6 @@ -44,6 +44,7 @@ require ( github.com/google/uuid v1.6.0 github.com/integrii/flaggy v1.5.2 github.com/itchyny/gojq v0.12.18 + github.com/itchyny/timefmt-go v0.1.7 github.com/jackc/pgx/v5 v5.9.2 github.com/jaswdr/faker v1.19.1 github.com/jedib0t/go-pretty v4.3.0+incompatible @@ -53,12 +54,14 @@ require ( github.com/jmoiron/sqlx v1.3.3 github.com/json-iterator/go v1.1.12 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 + github.com/kardianos/service v1.2.4 github.com/klauspost/compress v1.18.5 github.com/kshedden/datareader v0.0.0-20210325133423-816b6ffdd011 github.com/labstack/echo/v4 v4.10.2 github.com/lib/pq v1.10.9 github.com/linkedin/goavro/v2 v2.12.0 github.com/maja42/goval v1.4.0 + github.com/mark3labs/mcp-go v0.57.0 github.com/mattn/go-isatty v0.0.20 github.com/mattn/go-sqlite3 v1.14.28 github.com/microsoft/go-mssqldb v1.9.3 @@ -76,6 +79,8 @@ require ( github.com/shirou/gopsutil/v3 v3.24.4 github.com/shopspring/decimal v1.4.0 github.com/sijms/go-ora/v2 v2.8.24 + github.com/slingdata-io/godbc v0.0.9 + github.com/slingdata-io/sling v0.0.0-20260715135102-01cd9a07a3c8 github.com/snowflakedb/gosnowflake v1.17.1 github.com/spf13/cast v1.7.1 github.com/stretchr/testify v1.11.1 @@ -86,6 +91,11 @@ require ( github.com/xuri/excelize/v2 v2.9.1 github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a go.mongodb.org/mongo-driver v1.14.0 + go.opentelemetry.io/otel v1.43.0 + go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 + go.opentelemetry.io/otel/log v0.14.0 + go.opentelemetry.io/otel/sdk v1.43.0 + go.opentelemetry.io/otel/sdk/log v0.14.0 golang.org/x/crypto v0.52.0 golang.org/x/net v0.55.0 golang.org/x/oauth2 v0.34.0 @@ -116,6 +126,7 @@ require ( filippo.io/edwards25519 v1.1.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.2 // indirect + github.com/AlecAivazis/survey/v2 v2.3.7 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect github.com/Azure/go-autorest/autorest/to v0.4.1 // indirect @@ -125,6 +136,7 @@ require ( github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.30.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect github.com/andybalholm/brotli v1.2.0 // indirect github.com/andybalholm/cascadia v1.1.0 // indirect github.com/antlr4-go/antlr/v4 v4.13.1 // indirect @@ -134,7 +146,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/aws/aws-sdk-go v1.55.6 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 // indirect github.com/aws/aws-sdk-go-v2/internal/ini v1.8.3 // indirect @@ -144,23 +156,32 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.13 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.21 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 // indirect - github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.22.0 // indirect + github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect github.com/cockroachdb/apd/v3 v3.2.1 // indirect + github.com/coder/websocket v1.8.14 // indirect github.com/containerd/console v1.0.5 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect github.com/coreos/go-oidc/v3 v3.5.0 // indirect github.com/creasty/defaults v1.8.0 // indirect + github.com/cyphar/filepath-securejoin v0.2.4 // indirect github.com/danieljoos/wincred v1.2.2 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/disintegration/imaging v1.6.2 // indirect + github.com/distribution/reference v0.6.0 // indirect github.com/dnephin/pflag v1.0.7 // indirect github.com/docker/docker v28.5.2+incompatible // indirect github.com/docker/go-connections v0.7.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/domodwyer/mailyak/v3 v3.6.2 // indirect github.com/dvsekhvalnov/jose2go v1.7.0 // indirect github.com/ebitengine/purego v0.10.0 // indirect github.com/elastic/elastic-transport-go/v8 v8.6.0 // indirect @@ -171,25 +192,34 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.8.0 // indirect github.com/gabriel-vasile/mimetype v1.4.7 // indirect + github.com/ganigeorgiev/fexpr v0.4.1 // indirect github.com/go-faster/city v1.0.1 // indirect github.com/go-faster/errors v0.7.1 // indirect + github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect + github.com/go-git/go-billy/v5 v5.5.0 // indirect + github.com/go-git/go-git/v5 v5.11.0 // indirect github.com/go-jose/go-jose/v3 v3.0.5 // indirect github.com/go-jose/go-jose/v4 v4.1.4 // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-mysql-org/go-mysql v1.13.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/errors v0.21.0 // indirect github.com/go-openapi/strfmt v0.22.0 // indirect + github.com/go-ozzo/ozzo-validation/v4 v4.3.0 // indirect github.com/go-viper/mapstructure/v2 v2.4.0 // indirect github.com/goccy/go-json v0.10.5 // indirect github.com/goccy/go-yaml v1.17.1 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect + github.com/golang-jwt/jwt/v4 v4.5.2 // indirect github.com/golang-jwt/jwt/v5 v5.3.0 // indirect github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect github.com/golang-sql/sqlexp v0.1.0 // indirect github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect github.com/golang/snappy v1.0.0 // indirect github.com/google/flatbuffers v25.9.23+incompatible // indirect + github.com/google/jsonschema-go v0.4.2 // indirect + github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/google/s2a-go v0.1.9 // indirect github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect github.com/google/wire v0.6.0 // indirect @@ -197,6 +227,7 @@ require ( github.com/googleapis/gax-go/v2 v2.15.0 // indirect github.com/gookit/color v1.5.4 // indirect github.com/gorilla/websocket v1.5.3 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/hamba/avro/v2 v2.30.0 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -205,10 +236,12 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect github.com/imdario/mergo v0.3.16 // indirect - github.com/itchyny/timefmt-go v0.1.7 // indirect + github.com/jackc/pgio v1.0.0 // indirect + github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect + github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/gofork v1.7.6 // indirect @@ -218,13 +251,16 @@ require ( github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/jpillora/backoff v1.0.0 // indirect + github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect github.com/klauspost/cpuid/v2 v2.3.0 // indirect github.com/kr/fs v0.1.0 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 // indirect github.com/labstack/gommon v0.4.0 // indirect github.com/lithammer/fuzzysearch v1.1.8 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/matoous/go-nanoid/v2 v2.1.0 // indirect github.com/mattn/go-colorable v0.1.14 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect github.com/mattn/go-tty v0.0.3 // indirect @@ -232,7 +268,10 @@ require ( github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/moby/api v1.55.0 // indirect + github.com/moby/moby/client v0.5.1 // indirect github.com/moby/sys/user v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -240,29 +279,45 @@ require ( github.com/mtibben/percent v0.2.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect + github.com/nalgeon/redka v0.5.2 // indirect + github.com/nats-io/nats.go v1.36.0 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/ncruces/go-strftime v0.1.9 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/onsi/gomega v1.37.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect github.com/paulmach/orb v0.11.1 // indirect github.com/pierrec/lz4 v2.6.1+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec // indirect + github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a // indirect + github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pkg/term v1.2.0-beta.2 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/pocketbase/dbx v1.11.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/procfs v0.15.1 // indirect github.com/pterm/pterm v0.12.82 // indirect github.com/puzpuzpuz/xsync/v3 v3.5.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/richardlehane/mscfb v1.0.4 // indirect github.com/richardlehane/msoleps v1.0.4 // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 // indirect github.com/segmentio/asm v1.2.0 // indirect github.com/segmentio/encoding v0.4.0 // indirect github.com/shirou/gopsutil/v4 v4.25.1 // indirect github.com/shoenig/go-m1cpu v0.2.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect + github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect + github.com/slingdata-io/pocketbase v0.22.136 // indirect github.com/spiffe/go-spiffe/v2 v2.6.0 // indirect github.com/stretchr/objx v0.5.2 // indirect github.com/substrait-io/substrait v0.75.0 // indirect @@ -291,6 +346,7 @@ require ( github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/xuri/efp v0.0.1 // indirect github.com/xuri/nfp v0.0.1 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect @@ -298,11 +354,13 @@ require ( go.opentelemetry.io/contrib/detectors/gcp v1.39.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 // indirect - go.opentelemetry.io/otel v1.43.0 // indirect go.opentelemetry.io/otel/metric v1.43.0 // indirect - go.opentelemetry.io/otel/sdk v1.43.0 // indirect go.opentelemetry.io/otel/sdk/metric v1.43.0 // indirect go.opentelemetry.io/otel/trace v1.43.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.0 // indirect gocloud.dev v0.41.0 // indirect golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/image v0.38.0 // indirect @@ -325,12 +383,19 @@ require ( gopkg.in/mattn/go-colorable.v0 v0.1.0 // indirect gopkg.in/mattn/go-isatty.v0 v0.0.4 // indirect gopkg.in/mattn/go-runewidth.v0 v0.0.4 // indirect + gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect + gopkg.in/warnings.v0 v0.1.2 // indirect gotest.tools/gotestsum v1.8.2 // indirect - gotest.tools/v3 v3.5.2 // indirect + modernc.org/libc v1.66.10 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.42.2 // indirect ) // replace github.com/slingdata-io/golyglot => ../golyglot +replace github.com/slingdata-io/sling => ../sling + replace github.com/apache/iceberg-go => github.com/flarco/iceberg-go v0.0.0-20260105175128-f16b74585ee2 // replace github.com/apache/iceberg-go => ../iceberg-go @@ -344,3 +409,5 @@ replace github.com/apache/arrow-adbc/go/adbc => github.com/slingdata-io/arrow-ad // replace github.com/apache/arrow-adbc/go/adbc => ../arrow-adbc/go/adbc replace github.com/gocql/gocql => github.com/scylladb/gocql v1.18.0 + +replace github.com/flarco/g => ../g diff --git a/go.sum b/go.sum index 11d23664d..516af17d6 100644 --- a/go.sum +++ b/go.sum @@ -120,6 +120,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= +github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/PuerkitoBio/goquery v1.6.0 h1:j7taAbelrdcsOlGeMenZxc2AWXD5fieT1/znArdnx94= @@ -149,6 +151,7 @@ github.com/apache/thrift v0.23.0 h1:wKR6YnefQSEnxpEfmgTPuJibNG4bF0p2TK34tHLWi3s= github.com/apache/thrift v0.23.0/go.mod h1:zPt6WxgvTOM6hF92y8C+MkEM5LMxZuk4JcQOiU4Esvs= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/atomicgo/cursor v0.0.1/go.mod h1:cBON2QmmrysudxNBFthvMtN32r3jxVRIvzkUiF/RuIk= @@ -158,14 +161,14 @@ github.com/aws/aws-sdk-go-v2 v1.41.5 h1:dj5kopbwUsVUVFgO4Fi5BIT3t4WyqIDjGKCangnV github.com/aws/aws-sdk-go-v2 v1.41.5/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8 h1:eBMB84YGghSocM7PsjmmPffTa+1FBUeNvGvFou6V/4o= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.8/go.mod h1:lyw7GFp3qENLh7kwzf7iMzAxDn+NzjXEAGjKS2UOKqI= -github.com/aws/aws-sdk-go-v2/config v1.31.2 h1:NOaSZpVGEH2Np/c1toSeW0jooNl+9ALmsUTZ8YvkJR0= -github.com/aws/aws-sdk-go-v2/config v1.31.2/go.mod h1:17ft42Yb2lF6OigqSYiDAiUcX4RIkEMY6XxEMJsrAes= -github.com/aws/aws-sdk-go-v2/credentials v1.18.6 h1:AmmvNEYrru7sYNJnp3pf57lGbiarX4T9qU/6AZ9SucU= -github.com/aws/aws-sdk-go-v2/credentials v1.18.6/go.mod h1:/jdQkh1iVPa01xndfECInp1v1Wnp70v3K4MvtlLGVEc= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4 h1:lpdMwTzmuDLkgW7086jE94HweHCqG+uOJwHf3LZs7T0= -github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.4/go.mod h1:9xzb8/SV62W6gHQGC/8rrvgNXU6ZoYM3sAIJCIrXJxY= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0 h1:2FFgK3oFA8PTNBjprLFfcmkgg7U9YuSimBvR64RUmiA= -github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.19.0/go.mod h1:xdxj6nC1aU/jAO80RIlIj3fU40MOSqutEA9N2XFct04= +github.com/aws/aws-sdk-go-v2/config v1.29.15 h1:I5XjesVMpDZXZEZonVfjI12VNMrYa38LtLnw4NtY5Ss= +github.com/aws/aws-sdk-go-v2/config v1.29.15/go.mod h1:tNIp4JIPonlsgaO5hxO372a6gjhN63aSWl2GVl5QoBQ= +github.com/aws/aws-sdk-go-v2/credentials v1.17.68 h1:cFb9yjI02/sWHBSYXAtkamjzCuRymvmeFmt0TC0MbYY= +github.com/aws/aws-sdk-go-v2/credentials v1.17.68/go.mod h1:H6E+jBzyqUu8u0vGaU6POkK3P0NylYEeRZ6ynBpMqIk= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30 h1:x793wxmUWVDhshP8WW2mlnXuFrO4cOd3HLBroh1paFw= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.30/go.mod h1:Jpne2tDnYiFascUEs2AWHJL9Yp7A5ZVy3TNyxaAjD6M= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.69 h1:6VFPH/Zi9xYFMJKPQOX5URYkQoXRWeJ7V/7Y6ZDYoms= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.69/go.mod h1:GJj8mmO6YT6EqgduWocwhMoxTLFitkhIrK+owzrYL2I= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21 h1:Rgg6wvjjtX8bNHcvi9OnXWwcE0a2vGpbwmtICOsvcf4= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.21/go.mod h1:A/kJFst/nm//cyqonihbdpQZwiUhhzpqTsdbhDdRF9c= github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.21 h1:PEgGVtPoB6NTpPrBgqSE5hE/o47Ij9qk/SEZFbUOe9A= @@ -188,16 +191,17 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21 h1:ZlvrNcHSFFWUR github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.21/go.mod h1:cv3TNhVrssKR0O/xxLJVRfd2oazSnZnkUeTf6ctUwfQ= github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3 h1:HwxWTbTrIHm5qY+CAEur0s/figc3qwvLWsNkF4RPToo= github.com/aws/aws-sdk-go-v2/service/s3 v1.97.3/go.mod h1:uoA43SdFwacedBfSgfFSjjCvYe8aYBS7EnU5GZ/YKMM= -github.com/aws/aws-sdk-go-v2/service/sso v1.28.2 h1:ve9dYBB8CfJGTFqcQ3ZLAAb/KXWgYlgu/2R2TZL2Ko0= -github.com/aws/aws-sdk-go-v2/service/sso v1.28.2/go.mod h1:n9bTZFZcBa9hGGqVz3i/a6+NG0zmZgtkB9qVVFDqPA8= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2 h1:pd9G9HQaM6UZAZh19pYOkpKSQkyQQ9ftnl/LttQOcGI= -github.com/aws/aws-sdk-go-v2/service/ssooidc v1.33.2/go.mod h1:eknndR9rU8UpE/OmFpqU78V1EcXPKFTTm5l/buZYgvM= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.0 h1:iV1Ko4Em/lkJIsoKyGfc0nQySi+v0Udxr6Igq+y9JZc= -github.com/aws/aws-sdk-go-v2/service/sts v1.38.0/go.mod h1:bEPcjW7IbolPfK67G1nilqWyoxYMSPrDiIQ3RdIdKgo= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3 h1:1Gw+9ajCV1jogloEv1RRnvfRFia2cL6c9cuKV2Ps+G8= +github.com/aws/aws-sdk-go-v2/service/sso v1.25.3/go.mod h1:qs4a9T5EMLl/Cajiw2TcbNt2UNo/Hqlyp+GiuG4CFDI= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1 h1:hXmVKytPfTy5axZ+fYbR5d0cFmC3JvwLm5kM83luako= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.30.1/go.mod h1:MlYRNmYu/fGPoxBQVvBYr9nyr948aY/WLUvwBMBJubs= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.20 h1:oIaQ1e17CSKaWmUTu62MtraRWVIosn/iONMuZt0gbqc= +github.com/aws/aws-sdk-go-v2/service/sts v1.33.20/go.mod h1:cQnB8CUnxbMU82JvlqjKR2HBOm3fe9pWorWBza6MBJ4= github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/awsdocs/aws-doc-sdk-examples/gov2/testtools v0.0.0-20250407191926-092f3e54b837 h1:8eMceEa0ib+nqJuGsyowuZaVBVAr685oK6WrNIit+0g= github.com/awsdocs/aws-doc-sdk-examples/gov2/testtools v0.0.0-20250407191926-092f3e54b837/go.mod h1:9Oj/8PZn3D5Ftp/Z1QWrIEFE0daERMqfJawL9duHRfc= +github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -231,6 +235,8 @@ github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 h1:6xNmx7iTtyBRev0+D/T github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5/go.mod h1:KdCmV+x/BuvyMxRnYBlmVaq4OLiKW6iRQfvC62cvdkI= github.com/cockroachdb/apd/v3 v3.2.1 h1:U+8j7t0axsIgvQUqthuNm82HIrYXodOV2iWLWtEaIwg= github.com/cockroachdb/apd/v3 v3.2.1/go.mod h1:klXJcjp+FffLTHlhIG69tezTDvdP065naDsHzKhYSqc= +github.com/coder/websocket v1.8.14 h1:9L0p0iKiNOibykf283eHkKUHHrpG7f65OE3BhhO7v9g= +github.com/coder/websocket v1.8.14/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= github.com/compose-spec/compose-go/v2 v2.6.0 h1:/+oBD2ixSENOeN/TlJqWZmUak0xM8A7J08w/z661Wd4= github.com/compose-spec/compose-go/v2 v2.6.0/go.mod h1:vPlkN0i+0LjLf9rv52lodNMUTJF5YHVfHVGLLIP67NA= github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= @@ -260,8 +266,13 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= github.com/creasty/defaults v1.8.0 h1:z27FJxCAa0JKt3utc0sCImAEb+spPucmKoOdLHvHYKk= github.com/creasty/defaults v1.8.0/go.mod h1:iGzKe6pbEHnpMPtfDXZEr0NVxWnPTjb1bbDy08fPzYM= +github.com/cyphar/filepath-securejoin v0.2.4 h1:Ugdm7cg7i6ZK6x3xDF1oEu1nfkyfH53EtKeQYTC3kyg= +github.com/cyphar/filepath-securejoin v0.2.4/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4= github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -272,8 +283,12 @@ github.com/denisbrodbeck/machineid v1.0.1 h1:geKr9qtkB876mXguW2X6TU4ZynleN6ezuMS github.com/denisbrodbeck/machineid v1.0.1/go.mod h1:dJUwb7PTidGDeYyUBmXZ2GphQBbjJCrnectwCyxcUSI= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c= +github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4= github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo= +github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= github.com/dnephin/pflag v1.0.7 h1:oxONGlWxhmUct0YzKTgrpQv9AUA1wtPBn7zuSjJqptk= github.com/dnephin/pflag v1.0.7/go.mod h1:uxE91IoWURlOiTUIA8Mq5ZZkAv3dPUfZNaT80Zm7OQE= github.com/docker/buildx v0.22.0 h1:pGTcGZa+kxpYUlM/6ACsp1hXhkEDulz++RNXPdE8Afk= @@ -298,6 +313,8 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/domodwyer/mailyak/v3 v3.6.2 h1:x3tGMsyFhTCaxp6ycgR0FE/bu5QiNp+hetUuCOBXMn8= +github.com/domodwyer/mailyak/v3 v3.6.2/go.mod h1:lOm/u9CyCVWHeaAmHIdF4RiKVxKUT/H5XX10lIKAL6c= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= @@ -336,12 +353,12 @@ github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fergusstrange/embedded-postgres v1.31.0 h1:JmRxw2BcPRcU141nOEuGXbIU6jsh437cBB40rmftZSk= +github.com/fergusstrange/embedded-postgres v1.31.0/go.mod h1:w0YvnCgf19o6tskInrOOACtnqfVlOvluz3hlNLY7tRk= github.com/flarco/bigquery v0.0.9 h1:WfxO6XuuHZTJV+55Bq24FhdHYpmAOzgVk9xOcJpEecY= github.com/flarco/bigquery v0.0.9/go.mod h1:IpRSw4quaXxHjFyDSXUo7B6v+XcNF2pSmnNfeqXa/gM= github.com/flarco/databricks-sql-go v0.0.0-20250613120556-51f7c1f3b4ad h1:z5mgsXmNXsgskClg/s6zelILFihJTyK6x7+zX1jUgyU= github.com/flarco/databricks-sql-go v0.0.0-20250613120556-51f7c1f3b4ad/go.mod h1:mnsep6/uctUEgRONJy1pQdb17s/lRtqgcZZZV/WPdkc= -github.com/flarco/g v0.1.178 h1:7CTHpePoj+4z4/Jz2kGTNYuYfRZ4w4CNMnXqZvSk2PQ= -github.com/flarco/g v0.1.178/go.mod h1:Ho26Asm6DWBEcBW3Hs8q8ntdWDc/oe9wdF24m8jYSuQ= github.com/flarco/iceberg-go v0.0.0-20260105175128-f16b74585ee2 h1:k01grALOonWx3nBt0LfbUqY39kjaIXaiNP+2LHNuJYY= github.com/flarco/iceberg-go v0.0.0-20260105175128-f16b74585ee2/go.mod h1:S7rRApCtYThojNe57sOBhV/ZmH7EMfgEyou4TBEdQHU= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -361,6 +378,8 @@ github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= +github.com/ganigeorgiev/fexpr v0.4.1 h1:hpUgbUEEWIZhSDBtf4M9aUNfQQ0BZkGRaMePy7Gcx5k= +github.com/ganigeorgiev/fexpr v0.4.1/go.mod h1:RyGiGqmeXhEQ6+mlGdnUleLHgtzzu/VGO2WtJkF5drE= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -372,6 +391,12 @@ github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= +github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= +github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= +github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= +github.com/go-git/go-git/v5 v5.11.0 h1:XIZc1p+8YzypNr34itUfSvYJcv+eYdTnTvOZ2vD3cA4= +github.com/go-git/go-git/v5 v5.11.0/go.mod h1:6GFcX2P3NM7FPBfpePbpLd21XxsgdAt+lKqXmCUiUCY= github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= github.com/go-jose/go-jose/v3 v3.0.5 h1:BLLJWbC4nMZOfuPVxoZIxeYsn6Nl2r1fITaJ78UQlVQ= github.com/go-jose/go-jose/v3 v3.0.5/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ= @@ -382,6 +407,8 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-mysql-org/go-mysql v1.13.0 h1:Hlsa5x1bX/wBFtMbdIOmb6YzyaVNBWnwrb8gSIEPMDc= +github.com/go-mysql-org/go-mysql v1.13.0/go.mod h1:FQxw17uRbFvMZFK+dPtIPufbU46nBdrGaxOw0ac9MFs= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/errors v0.21.0 h1:FhChC/duCnfoLj1gZ0BgaBmzhJC2SL/sJr8a2vAobSY= @@ -394,6 +421,9 @@ github.com/go-openapi/strfmt v0.22.0 h1:Ew9PnEYc246TwrEspvBdDHS4BVKXy/AOVsfqGDgA github.com/go-openapi/strfmt v0.22.0/go.mod h1:HzJ9kokGIju3/K6ap8jL+OlGAbjpSv27135Yr9OivU4= github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-ozzo/ozzo-validation/v4 v4.3.0 h1:byhDUpfEwjsVQb1vBunvIjh2BHQ9ead57VkAEY4V+Es= +github.com/go-ozzo/ozzo-validation/v4 v4.3.0/go.mod h1:2NKgrcHl3z6cJs+3Oo940FPRiTzuqKbvfrL2RxCj6Ew= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo= github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= @@ -413,6 +443,8 @@ github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeH github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 h1:au07oEsX2xN0ktxqI+Sida1w446QrXBRJ0nee3SNZlA= @@ -473,11 +505,15 @@ github.com/google/go-replayers/httpreplay v1.2.0/go.mod h1:WahEFFZZ7a1P4VM1qEeHy github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/jsonschema-go v0.4.2 h1:tmrUohrwoLZZS/P3x7ex0WAVknEkBZM46iALbcqoRA8= +github.com/google/jsonschema-go v0.4.2/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= @@ -508,7 +544,6 @@ github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/z github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/grpc-gateway v1.5.0 h1:WcmKMm43DR7RdtlkEXQJyo5ws8iTp98CyhCCbOHMvNI= github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= @@ -534,6 +569,8 @@ github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKe github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= +github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= @@ -555,6 +592,8 @@ github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= +github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510 h1:+PJCokZ2BhyDKlncScmiNzBwqOx+yH1i8xRlWN/wn6A= +github.com/jackc/pglogrepl v0.0.0-20260401131349-e37c41485510/go.mod h1:UzTJ5Jjuf4O9hYWW+HYVwVldYz9J7CaePW0iuNJkrPQ= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= @@ -571,6 +610,8 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/jaswdr/faker v1.19.1 h1:xBoz8/O6r0QAR8eEvKJZMdofxiRH+F0M/7MU9eNKhsM= github.com/jaswdr/faker v1.19.1/go.mod h1:x7ZlyB1AZqwqKZgyQlnqEG8FDptmHlncA5u2zY/yi6w= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= @@ -612,6 +653,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= +github.com/kardianos/service v1.2.4 h1:XNlGtZOYNx2u91urOdg/Kfmc+gfmuIo1Dd3rEi2OgBk= +github.com/kardianos/service v1.2.4/go.mod h1:E4V9ufUuY82F7Ztlu1eN9VXWIQxg8NoLQlmFe0MtrXc= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= @@ -644,6 +687,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.10.2 h1:n1jAhnq/elIFTHr1EYpiYtyKgx4RW9ccVgkqByZaN2M= github.com/labstack/echo/v4 v4.10.2/go.mod h1:OEyqf2//K1DFdE57vw2DRgWY0M7s65IVQO2FzvI4J5k= +github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61 h1:FwuzbVh87iLiUQj1+uQUsuw9x5t9m5n5g7rG7o4svW4= +github.com/labstack/echo/v5 v5.0.0-20230722203903-ec5b858dab61/go.mod h1:paQfF1YtHe+GrGg5fOgjsjoCX/UKDr9bc1DoWpZfns8= github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -663,6 +708,11 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0 github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maja42/goval v1.4.0 h1:tlX0X+GvjKzWW2Q6qzWwL4Av2KV1bLtzxwzgxiiwEPc= github.com/maja42/goval v1.4.0/go.mod h1:LDMwF8ocOwIsMZdwoyHC/3UpV8ABDwEzalxkVV2z/rI= +github.com/mark3labs/mcp-go v0.57.0 h1:jzWKyCzdWnwnZt05cvcQQ+ngiUl2RnixXJa7Kj4qP1E= +github.com/mark3labs/mcp-go v0.57.0/go.mod h1:+8WclSK1ZUweCP3hvktSji8n8ABG/95QaEkeVE/Uwas= +github.com/matoous/go-nanoid/v2 v2.1.0 h1:P64+dmq21hhWdtvZfEAofnvJULaRR1Yib0+PnU669bE= +github.com/matoous/go-nanoid/v2 v2.1.0/go.mod h1:KlbGNQ+FhrUNIHUxZdL63t7tl4LaPkZNpUULS8H4uVM= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -693,6 +743,7 @@ github.com/mattn/go-sqlite3 v1.14.28/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxU github.com/mattn/go-tty v0.0.3 h1:5OfyWorkyO7xP52Mq7tB36ajHDG5OHrmBGIS/DtakQI= github.com/mattn/go-tty v0.0.3/go.mod h1:ihxohKRERHTVzN+aSVRwACLCeqIoZAWpoICkkvrWyR0= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= @@ -716,6 +767,10 @@ github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8 github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= +github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= +github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw= +github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM= github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk= github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= github.com/moby/spdystream v0.4.0 h1:Vy79D6mHeJJjiPdFEL2yku1kl0chZpJfZcPpb16BRl8= @@ -757,6 +812,14 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nalgeon/redka v0.5.2 h1:CX71v88kYj55EwJ10zq7U2eJdH0xcLAIjvKFvUMoM0o= +github.com/nalgeon/redka v0.5.2/go.mod h1:vLxjY3XS9IwBID2YEFWeeMiN4Ar/DtKd4JW62JTAxuU= +github.com/nats-io/nats.go v1.36.0 h1:suEUPuWzTSse/XhESwqLxXGuj8vGRuPRoG7MoRN/qyU= +github.com/nats-io/nats.go v1.36.0/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= @@ -768,6 +831,8 @@ github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/onsi/gomega v1.37.0 h1:CdEG8g0S133B4OswTDC/5XPSzE1OeP29QOioj2PID2Y= +github.com/onsi/gomega v1.37.0/go.mod h1:8D9+Txp43QWKhM24yyOBEdpkzN8FvJyAwecBgsU4KU0= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= @@ -790,8 +855,13 @@ github.com/pierrec/lz4 v2.6.1+incompatible h1:9UY3+iC23yxF0UfGaYrGplQ+79Rg+h/q9F github.com/pierrec/lz4 v2.6.1+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec h1:3EiGmeJWoNixU+EwllIn26x6s4njiWRXewdx2zlYa84= github.com/pingcap/errors v0.11.5-0.20250318082626-8f80e5cb09ec/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= +github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a h1:WIhmJBlNGmnCWH6TLMdZfNEDaiU8cFpZe3iaqDbQ0M8= +github.com/pingcap/log v1.1.1-0.20241212030209-7e3ff8601a2a/go.mod h1:ORfBOFp1eteu2odzsyaxI+b8TzJwgjwyQcGhI+9SfEA= +github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d h1:3Ej6eTuLZp25p3aH/EXdReRHY12hjZYs3RrGp7iLdag= +github.com/pingcap/tidb/pkg/parser v0.0.0-20250421232622-526b2c79173d/go.mod h1:+8feuexTKcXHZF/dkDfvCwEyBAmgb4paFc3/WeYV2eE= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -806,6 +876,8 @@ github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pocketbase/dbx v1.11.0 h1:LpZezioMfT3K4tLrqA55wWFw1EtH1pM4tzSVa7kgszU= +github.com/pocketbase/dbx v1.11.0/go.mod h1:xXRCIAKTHMgUCyCKZm55pUOdvFziJjQfXaWKhu2vhMs= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -848,6 +920,8 @@ github.com/richardlehane/msoleps v1.0.1/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTK github.com/richardlehane/msoleps v1.0.4 h1:WuESlvhX3gH2IHcd8UqyCuFY5yiq/GR/yqaSM/9/g00= github.com/richardlehane/msoleps v1.0.4/go.mod h1:BWev5JBpU9Ko2WAgmZEuiz4/u3ZYTKbjLycmwiWUfWg= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= @@ -856,6 +930,8 @@ github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/samber/lo v1.39.0 h1:4gTz1wUhNYLhFSKl6O+8peW0v2F4BCY034GRpU9WnuA= github.com/samber/lo v1.39.0/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 h1:KRzFb2m7YtdldCEkzs6KqmJw4nqEVZGK7IN2kJkjTuQ= +github.com/santhosh-tekuri/jsonschema/v6 v6.0.2/go.mod h1:JXeL+ps8p7/KNMjDQk3TCwPpBy0wYklyWTfbkIzdIFU= github.com/scylladb/gocql v1.18.0 h1:bmaMHNOUJyu0GHnTsVv0RRUAquQoTtYjCL/9jTxrg9Q= github.com/scylladb/gocql v1.18.0/go.mod h1:PZU+XJQ3fDymccIlTacmTdO+aTGGDSgXF1hw+yULUMk= github.com/secure-systems-lab/go-securesystemslib v0.4.0 h1:b23VGrQhTA8cN2CbBw7/FulN9fTtqYUdS5+Oxzt+DUE= @@ -867,8 +943,9 @@ github.com/segmentio/encoding v0.4.0/go.mod h1:/d03Cd8PoaDeceuhUUUQWjU0KhWjrmYrW github.com/segmentio/ksuid v1.0.4 h1:sBo2BdShXjmcugAMwjugoGUdUV0pcxY5mW4xKRn3v4c= github.com/segmentio/ksuid v1.0.4/go.mod h1:/XUiZBD3kVx5SmUOl55voK5yeAbBNNIed+2O73XgrPE= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= +github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b h1:h+3JX2VoWTFuyQEo87pStk/a99dzIO1mM9KxIyLPGTU= github.com/serialx/hashring v0.0.0-20200727003509-22c0c7ab6b1b/go.mod h1:/yeG0My1xr/u+HZrFQ1tOQQQQrOawfyMUH13ai5brBc= github.com/shibumi/go-pathspec v1.3.0 h1:QUyMZhFo0Md5B8zV8x2tesohbb5kfbpTi9rBnKh5dkI= @@ -915,6 +992,10 @@ github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 h1:JIAuq3EE github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/slingdata-io/arrow-adbc/go/adbc v0.0.0-20260806214312-6da3e7189c98 h1:HT0E/6EiVUe8wew4uIgYEpUYPHcswmGkxpbbqFges3g= github.com/slingdata-io/arrow-adbc/go/adbc v0.0.0-20260806214312-6da3e7189c98/go.mod h1:ikEb6zQgczMp7Alx/RYGaBEn/Mwwz1kfXbirnkbgWUo= +github.com/slingdata-io/godbc v0.0.9 h1:wh5MSI6l+eyTQ8pSbQcO4j195HwBXrVmNYRnHaGJKK8= +github.com/slingdata-io/godbc v0.0.9/go.mod h1:oBLg0zDZSK1BhLpcNhZjsvCLdfvB4OiTReAreXCrb3M= +github.com/slingdata-io/pocketbase v0.22.136 h1:RtAvPvYdK0qm9EB1r8GzNeEfSiqDK+tV8jyxwbpKKBA= +github.com/slingdata-io/pocketbase v0.22.136/go.mod h1:RYAdoMZtW+3OIgKqg+YhgWGIiwjtcBHGxRcVF2+1klA= github.com/snowflakedb/gosnowflake v1.17.1 h1:sBYExPDRv6hHF7fCqeXMT745L326Byw/cROxvCiEJzo= github.com/snowflakedb/gosnowflake v1.17.1/go.mod h1:TaHvQGh9MA2lopZZMm1AvvENDfwcnKtuskIr1e6Fpic= github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE= @@ -1036,6 +1117,8 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/xhit/go-str2duration/v2 v2.1.0 h1:lxklc02Drh6ynqX+DdPyp5pCKLUQpRT8bp8Ydu2Bstc= github.com/xhit/go-str2duration/v2 v2.1.0/go.mod h1:ohY8p+0f07DiV6Em5LKB0s2YpLtXVyJfNt1+BlmyAsU= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= +github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xo/dburl v0.3.0 h1:KGkeJB/oQhY/DeeJoYl/1+pNE/JnF6ouAuA8nzpQEQ8= github.com/xo/dburl v0.3.0/go.mod h1:TM8VMBT+LWqC3MBOulZjb8FAthcvZq0t/qvDLwS6skU= github.com/xo/terminfo v0.0.0-20210125001918-ca9a967f8778/go.mod h1:2MuV+tbUrU1zIOPMxZ5EncGwgmMJsa+9ucAQZXxsObs= @@ -1049,6 +1132,8 @@ github.com/xuri/nfp v0.0.1 h1:MDamSGatIvp8uOmDP8FnmjuQpu90NzdJxo7242ANR9Q= github.com/xuri/nfp v0.0.1/go.mod h1:WwHg+CVyzlv/TX9xqBFXEZAuxOPxn2k1GNHwG41IIUQ= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk= github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4= @@ -1082,6 +1167,8 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0 h1:RbKq8BG go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0/go.mod h1:h06DGIukJOevXaj/xrNjhi/2098RZzcLTbc0jDAUbsg= go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc= +go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0 h1:FZ6ei8GFW7kyPYdxJaV2rgI6M+4tvZzhYsQ2wgyVC08= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.31.0/go.mod h1:MdEu/mC6j3D+tTEfvI15b5Ci2Fn7NneJ71YMoiS3tpI= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.31.0 h1:ZsXq73BERAiNuuFXYqP4MR5hBrjXfMGSO+Cx7qoOZiM= @@ -1094,10 +1181,16 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 h1:Ckwye go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0/go.mod h1:teIFJh5pW2y+AN7riv6IBPX2DuesS3HgP39mwOspKwU= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= +go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= +go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= go.opentelemetry.io/otel/sdk v1.43.0 h1:pi5mE86i5rTeLXqoF/hhiBtUNcrAGHLKQdhg4h4V9Dg= go.opentelemetry.io/otel/sdk v1.43.0/go.mod h1:P+IkVU3iWukmiit/Yf9AWvpyRDlUeBaRg6Y+C58QHzg= +go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= +go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= +go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= go.opentelemetry.io/otel/sdk/metric v1.43.0 h1:S88dyqXjJkuBNLeMcVPRFXpRw2fuwdvfCGLEo89fDkw= go.opentelemetry.io/otel/sdk/metric v1.43.0/go.mod h1:C/RJtwSEJ5hzTiUz5pXF1kILHStzb9zFlIEe85bhj6A= go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= @@ -1105,10 +1198,22 @@ go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLh go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/mock v0.5.0 h1:KAMbZvZPyBPWgD14IrIQ38QCyjwpvVVV6K/bHl1IwQU= go.uber.org/mock v0.5.0/go.mod h1:ge71pBPLYDk7QIi1LupWxdAykm7KIEFchiOqd6z7qMM= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= @@ -1135,6 +1240,7 @@ golang.org/x/crypto v0.52.0/go.mod h1:1QgfPxDqh0T2M/elOJtp9RvuR95kVjir0e6/BvEmGb golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= +golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/image v0.38.0 h1:5l+q+Y9JDC7mBOMjo4/aPhMDcxEptsX+Tt3GgRQRPuE= golang.org/x/image v0.38.0/go.mod h1:/3f6vaXC+6CEanU4KJxbcUZyEePbyKbaLoDOe4ehFYY= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1282,6 +1388,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= @@ -1303,6 +1410,7 @@ golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= @@ -1332,6 +1440,7 @@ google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9Ywl google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -1399,6 +1508,10 @@ gopkg.in/mattn/go-isatty.v0 v0.0.4 h1:NtS1rQGQr4IaFWBGz4Cz4BhB///gyys4gDVtKA7hIs gopkg.in/mattn/go-isatty.v0 v0.0.4/go.mod h1:wt691ab7g0X4ilKZNmMII3egK0bTxl37fEn/Fwbd8gc= gopkg.in/mattn/go-runewidth.v0 v0.0.4 h1:r0P71TnzQDlNIcizCqvPSSANoFa3WVGtcNJf3TWurcY= gopkg.in/mattn/go-runewidth.v0 v0.0.4/go.mod h1:BmXejnxvhwdaATwiJbB1vZ2dtXkQKZGu9yLFCZb4msQ= +gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= +gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= +gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1438,14 +1551,34 @@ k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7F k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +modernc.org/cc/v4 v4.26.5 h1:xM3bX7Mve6G8K8b+T11ReenJOT+BmVqQj0FY5T4+5Y4= +modernc.org/cc/v4 v4.26.5/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= +modernc.org/ccgo/v4 v4.28.1 h1:wPKYn5EC/mYTqBO373jKjvX2n+3+aK7+sICCv4Fjy1A= +modernc.org/ccgo/v4 v4.28.1/go.mod h1:uD+4RnfrVgE6ec9NGguUNdhqzNIeeomeXf6CL0GTE5Q= +modernc.org/fileutil v1.3.40 h1:ZGMswMNc9JOCrcrakF1HrvmergNLAmxOPjizirpfqBA= +modernc.org/fileutil v1.3.40/go.mod h1:HxmghZSZVAz/LXcMNwZPA/DRrQZEVP9VX0V4LQGQFOc= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= modernc.org/libc v1.66.10 h1:yZkb3YeLx4oynyR+iUsXsybsX4Ubx7MQlSYEw4yj59A= modernc.org/libc v1.66.10/go.mod h1:8vGSEwvoUoltr4dlywvHqjtAqHBaw0j1jI7iFBTAr2I= modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.1.4 h1:2kNGMRiUjrp4LcaPuLY2PzUfqM/w9N23quVwhKt5Qm8= +modernc.org/opt v0.1.4/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= modernc.org/sqlite v1.42.2 h1:7hkZUNJvJFN2PgfUdjni9Kbvd4ef4mNLOu0B9FGxM74= modernc.org/sqlite v1.42.2/go.mod h1:+VkC6v3pLOAE0A0uVucQEcbVW0I5nHCeDaBf+DpsQT8= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= From 09f52afdbe56ffbe0457d005e2ec937ba1f8bb03 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Thu, 20 Aug 2026 22:48:05 -0300 Subject: [PATCH 05/30] fix(database): ensure correct catalog is used for BulkImportStream --- core/dbio/database/database_adbc.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/core/dbio/database/database_adbc.go b/core/dbio/database/database_adbc.go index 3e0254b36..8dbdecf20 100644 --- a/core/dbio/database/database_adbc.go +++ b/core/dbio/database/database_adbc.go @@ -1366,6 +1366,11 @@ func (conn *ArrowDBConn) BulkImportStream(tableFName string, ds *iop.Datastream) DBSchema: table.Schema, } + // For 2-part targets (schema.table), ParseTableName leaves table.Database empty + if opts.Catalog == "" { + opts.Catalog = conn.GetProp("database") + } + g.Trace("arrow schema => %s", iop.ColumnsToArrowSchema(ds.Columns)) for batch := range ds.BatchChan { From c66a0e2b1792f42852c52a0b1ed3723eeabe1ca2 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Thu, 27 Aug 2026 10:50:11 -0300 Subject: [PATCH 06/30] feat: add sling build command Introduce a new `sling build` CLI command enabling dbt-style transformation workflows natively in Sling: - Add cmd/sling/sling_build.go with flags for target connection, model selection (--select/--exclude supporting glob, tag:, and +upstream syntax), full-refresh, dev/prod schema modes, vars, compile-only, list, fail-fast, seed skipping, backfill ranges, and parallel threads - Add core build engine that compiles models into a DAG and executes selected models with dependency-aware parallelism - Support incremental models via SLING_STATE watermarks (tier A) with fallback to target MAX() high-watermark detection (tier B) - Support lookback windows with inclusive bounds and paged/range backfill with explicit or auto-detected origin from source MIN - Add integration tests covering full refresh, incremental runs, lookback, and paged execution against Postgres --- cmd/sling/sling_build.go | 285 +++ core/sling/build/build.go | 749 ++++++++ core/sling/build/build_test.go | 484 +++++ core/sling/build/ddl.go | 284 +++ core/sling/build/executor.go | 1644 +++++++++++++++++ core/sling/build/executor_test.go | 533 ++++++ core/sling/build/hook_runner.go | 140 ++ core/sling/build/project.go | 1329 +++++++++++++ core/sling/build/project_test.go | 1516 +++++++++++++++ core/sling/build/selector.go | 666 +++++++ core/sling/build/selector_test.go | 861 +++++++++ core/sling/build/template.go | 896 +++++++++ core/sling/build/template_test.go | 1598 ++++++++++++++++ core/sling/build/tests.go | 177 ++ .../marts/core/fct_orders.sql | 10 + .../build/clickhouse_project/sling_build.yml | 4 + .../clickhouse_project/staging/stg_orders.sql | 3 + tests/build/cycle_project/sling_build.yml | 1 + tests/build/cycle_project/staging/model_a.sql | 1 + tests/build/cycle_project/staging/model_b.sql | 1 + .../models/staging/stg_orders.sql | 1 + .../seeds/staging/country_codes.csv | 3 + .../build/dbt_compat_project/sling_build.yml | 2 + .../marts/fct_orders.sql | 12 + .../defaults_expanded_project/sling_build.yml | 9 + .../staging/sling_build.yml | 6 + .../staging/stg_disabled.sql | 4 + .../staging/stg_orders.sql | 5 + .../seeds/customers.csv | 3 + .../duckdb_parallel_project/seeds/orders.csv | 3 + .../duckdb_parallel_project/sling_build.yml | 4 + .../staging/stg_ok.sql | 4 + .../archive/stg_orders.sql | 1 + .../duplicate_names_project/sling_build.yml | 1 + .../staging/stg_orders.sql | 1 + .../build/hooks_project/marts/fct_orders.sql | 17 + tests/build/hooks_project/sling_build.yml | 4 + .../hooks_project/staging/stg_orders.sql | 14 + .../macro_project/marts/product_margins.sql | 9 + tests/build/macro_project/sling_build.yml | 4 + .../staging/staging_helpers.macros.sql | 7 + .../macro_project/staging/stg_products.sql | 11 + tests/build/macro_project/utils.macros.sql | 7 + .../multi_statement_project/sling_build.yml | 4 + .../staging/stg_multi.sql | 8 + .../warehouse_a/sling_build.yml | 1 + .../warehouse_a/staging/stg_orders.sql | 6 + .../warehouse_b/sling_build.yml | 1 + .../warehouse_b/staging/stg_events.sql | 1 + .../marts/core/dim_customers.sql | 1 + .../build/nested_yml_project/sling_build.yml | 4 + .../staging/sling_build.yml | 2 + .../nested_yml_project/staging/stg_orders.sql | 1 + .../marts/fct_orders.sql | 5 + .../pipeline_step_project/sling_build.yml | 7 + .../staging/stg_orders.sql | 6 + .../models/bad_model.sql | 11 + .../range_bad_dbt_with_range/sling_build.yml | 5 + .../range_bad_mixed_style/sling_build.yml | 4 + .../staging/bad_model.sql | 10 + .../sling_build.yml | 4 + .../staging/bad_model.sql | 9 + tests/build/range_test_project/.gitignore | 1 + .../range_test/fact_orders.sql | 9 + .../range_test/fact_orders_lookback.sql | 11 + .../range_test/fact_orders_paged.sql | 11 + .../range_test/fact_orders_paged_start.sql | 12 + .../range_test/stg_orders.csv | 31 + .../build/range_test_project/sling_build.yml | 4 + .../marts/core/dim_customers.sql | 7 + .../sample_project/marts/core/fct_orders.sql | 10 + .../sample_project/marts/finance/revenue.sql | 8 + tests/build/sample_project/raw.sql | 1 + .../sample_project/seeds/status_map.json | 4 + tests/build/sample_project/sling_build.yml | 5 + .../sample_project/staging/country_codes.csv | 4 + .../staging/staging_helpers.macros.sql | 3 + .../sample_project/staging/stg_customers.sql | 3 + .../sample_project/staging/stg_orders.sql | 3 + tests/build/sample_project/utils.macros.sql | 7 + tests/pipelines/p.45.build_step.yaml | 81 + .../p.46.build_step_replication_hook.yaml | 56 + tests/suite.cli.build.yaml | 528 ++++++ 83 files changed, 12203 insertions(+) create mode 100644 cmd/sling/sling_build.go create mode 100644 core/sling/build/build.go create mode 100644 core/sling/build/build_test.go create mode 100644 core/sling/build/ddl.go create mode 100644 core/sling/build/executor.go create mode 100644 core/sling/build/executor_test.go create mode 100644 core/sling/build/hook_runner.go create mode 100644 core/sling/build/project.go create mode 100644 core/sling/build/project_test.go create mode 100644 core/sling/build/selector.go create mode 100644 core/sling/build/selector_test.go create mode 100644 core/sling/build/template.go create mode 100644 core/sling/build/template_test.go create mode 100644 core/sling/build/tests.go create mode 100644 tests/build/clickhouse_project/marts/core/fct_orders.sql create mode 100644 tests/build/clickhouse_project/sling_build.yml create mode 100644 tests/build/clickhouse_project/staging/stg_orders.sql create mode 100644 tests/build/cycle_project/sling_build.yml create mode 100644 tests/build/cycle_project/staging/model_a.sql create mode 100644 tests/build/cycle_project/staging/model_b.sql create mode 100644 tests/build/dbt_compat_project/models/staging/stg_orders.sql create mode 100644 tests/build/dbt_compat_project/seeds/staging/country_codes.csv create mode 100644 tests/build/dbt_compat_project/sling_build.yml create mode 100644 tests/build/defaults_expanded_project/marts/fct_orders.sql create mode 100644 tests/build/defaults_expanded_project/sling_build.yml create mode 100644 tests/build/defaults_expanded_project/staging/sling_build.yml create mode 100644 tests/build/defaults_expanded_project/staging/stg_disabled.sql create mode 100644 tests/build/defaults_expanded_project/staging/stg_orders.sql create mode 100644 tests/build/duckdb_parallel_project/seeds/customers.csv create mode 100644 tests/build/duckdb_parallel_project/seeds/orders.csv create mode 100644 tests/build/duckdb_parallel_project/sling_build.yml create mode 100644 tests/build/duckdb_parallel_project/staging/stg_ok.sql create mode 100644 tests/build/duplicate_names_project/archive/stg_orders.sql create mode 100644 tests/build/duplicate_names_project/sling_build.yml create mode 100644 tests/build/duplicate_names_project/staging/stg_orders.sql create mode 100644 tests/build/hooks_project/marts/fct_orders.sql create mode 100644 tests/build/hooks_project/sling_build.yml create mode 100644 tests/build/hooks_project/staging/stg_orders.sql create mode 100644 tests/build/macro_project/marts/product_margins.sql create mode 100644 tests/build/macro_project/sling_build.yml create mode 100644 tests/build/macro_project/staging/staging_helpers.macros.sql create mode 100644 tests/build/macro_project/staging/stg_products.sql create mode 100644 tests/build/macro_project/utils.macros.sql create mode 100644 tests/build/multi_statement_project/sling_build.yml create mode 100644 tests/build/multi_statement_project/staging/stg_multi.sql create mode 100644 tests/build/multi_target_project/warehouse_a/sling_build.yml create mode 100644 tests/build/multi_target_project/warehouse_a/staging/stg_orders.sql create mode 100644 tests/build/multi_target_project/warehouse_b/sling_build.yml create mode 100644 tests/build/multi_target_project/warehouse_b/staging/stg_events.sql create mode 100644 tests/build/nested_yml_project/marts/core/dim_customers.sql create mode 100644 tests/build/nested_yml_project/sling_build.yml create mode 100644 tests/build/nested_yml_project/staging/sling_build.yml create mode 100644 tests/build/nested_yml_project/staging/stg_orders.sql create mode 100644 tests/build/pipeline_step_project/marts/fct_orders.sql create mode 100644 tests/build/pipeline_step_project/sling_build.yml create mode 100644 tests/build/pipeline_step_project/staging/stg_orders.sql create mode 100644 tests/build/range_bad_dbt_with_range/models/bad_model.sql create mode 100644 tests/build/range_bad_dbt_with_range/sling_build.yml create mode 100644 tests/build/range_bad_mixed_style/sling_build.yml create mode 100644 tests/build/range_bad_mixed_style/staging/bad_model.sql create mode 100644 tests/build/range_bad_start_no_advance/sling_build.yml create mode 100644 tests/build/range_bad_start_no_advance/staging/bad_model.sql create mode 100644 tests/build/range_test_project/.gitignore create mode 100644 tests/build/range_test_project/range_test/fact_orders.sql create mode 100644 tests/build/range_test_project/range_test/fact_orders_lookback.sql create mode 100644 tests/build/range_test_project/range_test/fact_orders_paged.sql create mode 100644 tests/build/range_test_project/range_test/fact_orders_paged_start.sql create mode 100644 tests/build/range_test_project/range_test/stg_orders.csv create mode 100644 tests/build/range_test_project/sling_build.yml create mode 100644 tests/build/sample_project/marts/core/dim_customers.sql create mode 100644 tests/build/sample_project/marts/core/fct_orders.sql create mode 100644 tests/build/sample_project/marts/finance/revenue.sql create mode 100644 tests/build/sample_project/raw.sql create mode 100644 tests/build/sample_project/seeds/status_map.json create mode 100644 tests/build/sample_project/sling_build.yml create mode 100644 tests/build/sample_project/staging/country_codes.csv create mode 100644 tests/build/sample_project/staging/staging_helpers.macros.sql create mode 100644 tests/build/sample_project/staging/stg_customers.sql create mode 100644 tests/build/sample_project/staging/stg_orders.sql create mode 100644 tests/build/sample_project/utils.macros.sql create mode 100644 tests/pipelines/p.45.build_step.yaml create mode 100644 tests/pipelines/p.46.build_step_replication_hook.yaml create mode 100644 tests/suite.cli.build.yaml diff --git a/cmd/sling/sling_build.go b/cmd/sling/sling_build.go new file mode 100644 index 000000000..2e7c79f94 --- /dev/null +++ b/cmd/sling/sling_build.go @@ -0,0 +1,285 @@ +package main + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/flarco/g" + "github.com/integrii/flaggy" + "github.com/slingdata-io/sling-cli/core/env" + "github.com/slingdata-io/sling-cli/core/sling/build" + "github.com/spf13/cast" + "gopkg.in/yaml.v3" +) + +var cliBuildFlags = []g.Flag{ + { + Name: "target", + ShortName: "t", + Type: "string", + Description: "Target connection (required if no sling_build.yml).", + }, + { + Name: "select", + ShortName: "s", + Type: "string", + Description: "Model selector (glob, tag:xxx, +model for upstream).", + }, + { + Name: "exclude", + Type: "string", + Description: "Exclude models matching pattern.", + }, + { + Name: "full-refresh", + ShortName: "f", + Type: "bool", + Description: "Force full-refresh for all models.", + }, + { + Name: "schema", + Type: "string", + Description: "Override dev schema (forces dev mode, cannot combine with --prod).", + }, + { + Name: "prod", + Type: "bool", + Description: "Force prod mode (overrides yml mode: dev).", + }, + { + Name: "vars", + Type: "string", + Description: "Variables as YAML/JSON string.", + }, + { + Name: "compile", + ShortName: "c", + Type: "bool", + Description: "Compile only — show SQL + DAG, don't execute.", + }, + { + Name: "list", + ShortName: "l", + Type: "bool", + Description: "List selected models and exit.", + }, + { + Name: "fail-fast", + ShortName: "x", + Type: "bool", + Description: "Stop on first failure (in-flight models finish).", + }, + { + Name: "no-seeds", + Type: "bool", + Description: "Skip seed loading.", + }, + { + Name: "range", + Type: "string", + Description: "Backfill range for incremental models: 'start,end[,step]'. E.g. '2024-01-01,2024-12-31,1mo'. Does not advance SLING_STATE.", + }, + { + Name: "threads", + Type: "string", + Description: "Parallel model executions (default: 4).", + }, + { + Name: "recursive", + ShortName: "R", + Type: "bool", + Description: "Recursively discover sling_build.yml in immediate subdirectories.", + }, + { + Name: "test", + Type: "bool", + Description: "Run declarative data tests only (skip materialization).", + }, + { + Name: "json", + Type: "bool", + Description: "Emit machine-readable JSON for --compile / --list.", + }, + { + Name: "debug", + ShortName: "d", + Type: "bool", + Description: "Set logging level to DEBUG.", + }, + { + Name: "trace", + Type: "bool", + Description: "Set logging level to TRACE.", + }, +} + +var cliBuild = &g.CliSC{ + Name: "build", + Description: "Build and execute SQL models", + AdditionalHelpPrepend: "\nA lightweight SQL model builder with dependency resolution, Jinja templating, and incremental materializations.", + ExecuteWithoutFlags: true, + Flags: cliBuildFlags, + PosFlags: []g.Flag{ + { + Name: "path", + Type: "string", + Description: "The project directory path (default: current directory).\n", + Required: false, + }, + }, + ExecProcess: processBuild, +} + +func init() { + cliBuild.Make().Add() +} + +func processBuild(c *g.CliSC) (ok bool, err error) { + ok = true + + opts := build.BuildOptions{ + Threads: build.DefaultThreads, + } + + projectPath := "." + compileMode := false + + for k, v := range c.Vals { + switch k { + case "path": + if p := cast.ToString(v); p != "" { + projectPath = p + } + case "target": + opts.Target = cast.ToString(v) + case "select": + if s := cast.ToString(v); s != "" { + opts.Select = strings.Split(s, ",") + } + case "exclude": + if s := cast.ToString(v); s != "" { + opts.Exclude = strings.Split(s, ",") + } + case "full-refresh": + opts.FullRefresh = cast.ToBool(v) + case "schema": + opts.Schema = cast.ToString(v) + case "prod": + opts.Prod = cast.ToBool(v) + case "vars": + if varsStr := cast.ToString(v); varsStr != "" { + varsMap := make(map[string]any) + if err := yaml.Unmarshal([]byte(varsStr), &varsMap); err != nil { + return ok, g.Error(err, "could not parse --vars") + } + opts.Vars = varsMap + } + case "compile": + compileMode = cast.ToBool(v) + case "list": + opts.List = cast.ToBool(v) + case "fail-fast": + opts.FailFast = cast.ToBool(v) + case "no-seeds": + opts.NoSeeds = cast.ToBool(v) + case "range": + if s := cast.ToString(v); s != "" { + opts.Range = g.String(s) + } + case "threads": + if t := cast.ToInt(v); t > 0 { + opts.Threads = t + } + case "recursive": + opts.Recursive = cast.ToBool(v) + case "test": + opts.Test = cast.ToBool(v) + case "json": + opts.JSON = cast.ToBool(v) + case "debug": + if cast.ToBool(v) { + os.Setenv("DEBUG", "LOW") + env.InitLogger() + } + case "trace": + if cast.ToBool(v) { + os.Setenv("DEBUG", "TRACE") + env.InitLogger() + } + } + } + + opts.Compile = compileMode + + // Validate flag combinations + if opts.Prod && opts.Schema != "" { + return ok, g.Error("cannot combine --prod and --schema") + } + + // If there's no sling_build.yml at the path and the user gave us nothing to + // work with (no --target, no -r), show help instead of walking the tree. + // This avoids slurping every .sql file under cwd as "models". + if opts.Target == "" && !opts.Recursive { + if _, err := os.Stat(filepath.Join(projectPath, build.ConfigFileName)); os.IsNotExist(err) { + flaggy.ShowHelp("") + return ok, nil + } + } + + // Build and compile + b, err := build.NewBuild(projectPath, opts) + if err != nil { + return ok, g.Error(err, "could not load build project") + } + + if err := b.Compile(); err != nil { + return ok, g.Error(err, "could not compile build project") + } + + if opts.List { + if opts.JSON { + b.PrintListJSON() + } else { + b.PrintListOutput() + } + return ok, nil + } + + if compileMode { + if opts.JSON { + b.PrintCompileJSON() + } else { + b.PrintCompileOutput() + } + return ok, nil + } + + // Execute the build + if err := b.Execute(); err != nil { + return ok, g.Error(err, "build execution failed") + } + return ok, nil +} + +// askPrompt writes label and reads one answer. It returns an error on EOF so +// callers that loop on empty input cannot spin when stdin closes. +func askPrompt(reader *bufio.Reader, label string) (string, error) { + fmt.Print(label) + input, err := reader.ReadString('\n') + if err != nil && input == "" { + return "", g.Error(err, "could not read input") + } + return strings.TrimSpace(input), nil +} + +// isInteractive reports whether stdin is a TTY. +func isInteractive() bool { + fi, err := os.Stdin.Stat() + if err != nil { + return false + } + return (fi.Mode() & os.ModeCharDevice) != 0 +} diff --git a/core/sling/build/build.go b/core/sling/build/build.go new file mode 100644 index 000000000..14869b15a --- /dev/null +++ b/core/sling/build/build.go @@ -0,0 +1,749 @@ +package build + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/flarco/g" + "github.com/slingdata-io/golyglot" + "github.com/slingdata-io/sling-cli/core/dbio" + "github.com/slingdata-io/sling-cli/core/dbio/connection" + "github.com/slingdata-io/sling-cli/core/env" +) + +// Build is the main orchestrator for sling build. +// It loads a project, compiles templates, builds the DAG, +// applies selectors, and provides compile output. +type Build struct { + Project *BuildProject + DAG *DAG + Engine *TemplateEngine + Options BuildOptions + Selected []string // selected node names after selector filtering + SubBuilds []*Build // compiled sub-projects (for multi-target compile mode) + connEntries connection.ConnEntries // pre-resolved connection entries for parallel execution +} + +// NewBuild creates a new Build from the given project directory and options. +func NewBuild(dir string, opts BuildOptions) (*Build, error) { + project, err := LoadProject(dir, opts) + if err != nil { + return nil, g.Error(err, "could not load project from %s", dir) + } + + b := &Build{ + Project: project, + Options: opts, + } + + return b, nil +} + +// Compile loads the project, compiles all model templates, builds the DAG, +// and applies selectors. After Compile(), the Build is ready for execution +// or compile output. +func (b *Build) Compile() error { + // For sub-projects (independent builds), compile each one + if len(b.Project.SubProjects) > 0 { + if !b.Options.Compile { + return nil // sub-projects are compiled individually during Execute + } + for _, subProject := range b.Project.SubProjects { + subBuild := &Build{Project: subProject, Options: b.Options} + if subBuild.Options.Target == "" && subProject.Config != nil { + subBuild.Options.Target = subProject.Config.Target + } + if err := subBuild.Compile(); err != nil { + return g.Error(err, "could not compile sub-project %s", subProject.Dir) + } + b.SubBuilds = append(b.SubBuilds, subBuild) + } + return nil + } + + // Resolve target + target := "" + if b.Project.Config != nil { + target = b.Project.Config.Target + } + if b.Options.Target != "" { + target = b.Options.Target + } + if target == "" { + return g.Error("No target specified. Use '--target ' or set target in sling_build.yml.") + } + + // Get vars + vars := make(map[string]any) + if b.Project.Config != nil { + for k, v := range b.Project.Config.Vars { + vars[k] = v + } + } + for k, v := range b.Options.Vars { + vars[k] = v + } + + // Compile templates + b.Engine = NewTemplateEngine(b.Project, vars) + if err := b.Engine.CompileAll(DefaultIncrementalContext()); err != nil { + return g.Error(err, "could not compile models") + } + + // Rewrite table references: match prod-mode names in SQL and replace with current-mode names. + // This also populates DependsOn for matched references. + // Models with rewrite: false skip bare-name rewriting (ref()/src() still resolve). + for _, model := range b.Project.Models { + if model.Config.Rewrite != nil && !*model.Config.Rewrite { + continue + } + rewritten, deps := RewriteTableReferences(model.CompiledSQL, b.Project, model.Name) + model.CompiledSQL = rewritten + for _, dep := range deps { + if !containsStr(model.DependsOn, dep) { + model.DependsOn = append(model.DependsOn, dep) + } + } + } + + // Split multi-statement models into pre-statements, model query, and post-statements + if err := b.splitMultiStatementModels(); err != nil { + return g.Error(err, "could not parse multi-statement models") + } + + // Auto-detect SQL references as a safety net for DependsOn (catches edge cases + // the rewrite step might miss, e.g., three-part names or unusual patterns) + for _, model := range b.Project.Models { + autoRefs := ExtractTableReferences(model.CompiledSQL) + for _, ref := range autoRefs { + // Check if this matches a project model by full table name + for name, m := range b.Project.Models { + if m.FullTableName == ref && name != model.Name { + if !containsStr(model.DependsOn, name) { + model.DependsOn = append(model.DependsOn, name) + } + } + } + // Check if this matches a project seed by full table name + for name, s := range b.Project.Seeds { + if s.FullTableName == ref { + if !containsStr(model.DependsOn, name) { + model.DependsOn = append(model.DependsOn, name) + } + } + } + } + } + + // Build DAG + dag, err := BuildDAG(b.Project) + if err != nil { + return g.Error(err, "could not build dependency graph") + } + b.DAG = dag + + // Apply selectors + selector := NewSelector(b.Options.Select, b.Options.Exclude) + selected, err := selector.Apply(b.DAG) + if err != nil { + return g.Error(err, "could not apply selectors") + } + b.Selected = selected + + return nil +} + +// GetTarget returns the resolved target connection name. +func (b *Build) GetTarget() string { + if b.Options.Target != "" { + return b.Options.Target + } + if b.Project.Config != nil { + return b.Project.Config.Target + } + return "" +} + +// GetModelMode returns the effective mode for a model, considering: +// 1. CLI --full-refresh flag overrides everything +// 2. Model config() block mode +// 3. Project defaults.mode +// 4. Default: full-refresh +func (b *Build) GetModelMode(model *Model) string { + if b.Options.FullRefresh { + return "full-refresh" + } + mode := model.Config.Mode + if mode == "" && b.Project.Config != nil { + mode = b.Project.Config.Defaults.Mode + } + if mode == "" { + return "full-refresh" + } + canonical, _ := normalizeMode(mode) + if canonical == "" { + return "full-refresh" + } + return canonical +} + +// splitMultiStatementModels parses each model's CompiledSQL and splits +// multi-statement files into pre-statements, the model query, and post-statements. +// This runs after CompileAll but before ExtractTableReferences so that +// auto-ref detection only scans the model query, not pre/post DDL. +func (b *Build) splitMultiStatementModels() error { + dbType := b.resolveDbType() + for _, model := range b.Project.Models { + result, err := MakeModelSQL(model.CompiledSQL, dbType) + if err != nil { + return g.Error(err, "model '%s'", model.Name) + } + model.PreStatements = result.PreStatements + model.CompiledSQL = result.ModelQuery + model.PostStatements = result.PostStatements + } + return nil +} + +// resolveDbType determines the database type from the target connection +// without opening a connection (uses local connection entries). +func (b *Build) resolveDbType() dbio.Type { + entries := connection.GetLocalConns() + entry := entries.Get(b.GetTarget()) + if entry.Name != "" { + return entry.Connection.Type + } + g.Warn("target connection '%s' not found locally; multi-statement splitting will use generic dialect", b.GetTarget()) + return dbio.TypeUnknown +} + +// Execute runs the compiled build against the target database. +// Must be called after Compile(). +func (b *Build) Execute() error { + // Handle sub-projects (independent builds) — run with thread limit + if len(b.Project.SubProjects) > 0 { + // Pre-resolve connections once to avoid concurrent access to GetLocalConns + entries := connection.GetLocalConns() + + threads := b.Options.Threads + if threads < 1 { + threads = 1 + } + if b.resolveDbType().IsSingleWriterDB() && threads > 1 { + g.Debug("capping sub-project threads to 1 for DuckDB-family target (single writer)") + threads = 1 + } + + var wg sync.WaitGroup + sem := make(chan struct{}, threads) + errCh := make(chan error, len(b.Project.SubProjects)) + + for _, subProject := range b.Project.SubProjects { + sem <- struct{}{} // acquire semaphore + wg.Add(1) + go func(sp *BuildProject) { + defer wg.Done() + defer func() { <-sem }() // release semaphore + subBuild := &Build{ + Project: sp, + Options: b.Options, + connEntries: entries, + } + if err := subBuild.Compile(); err != nil { + errCh <- g.Error(err, "could not compile sub-project %s", sp.Dir) + return + } + if err := subBuild.Execute(); err != nil { + errCh <- g.Error(err, "could not execute sub-project %s", sp.Dir) + } + }(subProject) + } + + wg.Wait() + close(errCh) + + var errs []string + for err := range errCh { + errs = append(errs, err.Error()) + } + if len(errs) > 0 { + return g.Error(strings.Join(errs, "; ")) + } + return nil + } + + executor, err := NewExecutor(b) + if err != nil { + return err + } + + return executor.Execute() +} + +// PrintListOutput prints the selected models/seeds and exits. +func (b *Build) PrintListOutput() { + if len(b.SubBuilds) > 0 { + for _, subBuild := range b.SubBuilds { + fmt.Printf("=== Sub-project: %s (target: %s) ===\n", filepath.Base(subBuild.Project.Dir), subBuild.GetTarget()) + subBuild.PrintListOutput() + fmt.Println() + } + return + } + + if b.DAG == nil || len(b.Selected) == 0 { + fmt.Println("No models selected.") + return + } + + for _, name := range b.Selected { + node := b.DAG.Nodes[name] + if b.Options.NoSeeds && node.Seed != nil { + continue + } + nodeType := "" + if node.Seed != nil { + nodeType = "seed" + } else if node.Model != nil { + nodeType = b.GetModelMode(node.Model) + } + fmt.Printf("%s (%s)\n", name, nodeType) + } +} + +// PrintListJSON prints selected nodes as JSON. +func (b *Build) PrintListJSON() { + type item struct { + Name string `json:"name"` + Type string `json:"type"` + } + var items []item + for _, name := range b.Selected { + node := b.DAG.Nodes[name] + if node == nil { + continue + } + if b.Options.NoSeeds && node.Seed != nil { + continue + } + it := item{Name: name} + if node.Seed != nil { + it.Type = "seed" + } else if node.Model != nil { + it.Type = b.GetModelMode(node.Model) + } + items = append(items, it) + } + fmt.Println(g.Marshal(items)) +} + +// PrintCompileOutput prints the compile output in YAML format for each selected node. +func (b *Build) PrintCompileOutput() { + if len(b.SubBuilds) > 0 { + for i, subBuild := range b.SubBuilds { + if i > 0 { + fmt.Println() + } + fmt.Printf("# === Sub-project: %s (target: %s) ===\n", filepath.Base(subBuild.Project.Dir), subBuild.GetTarget()) + subBuild.PrintCompileOutput() + } + return + } + + if b.DAG == nil || len(b.Selected) == 0 { + fmt.Println("# No models selected.") + return + } + + // Print DAG execution order + fmt.Println("DAG Execution Order:") + for _, name := range b.Selected { + node := b.DAG.Nodes[name] + nodeType := "" + if node.Seed != nil { + nodeType = "seed" + } else if node.Model != nil { + nodeType = b.GetModelMode(node.Model) + } + fmt.Printf(" %s (%s)\n", name, nodeType) + } + fmt.Println() + + for i, name := range b.Selected { + node := b.DAG.Nodes[name] + if i > 0 { + fmt.Println() + } + if node.Seed != nil { + b.printSeedYAML(name, node) + } else if node.Model != nil { + b.printModelYAML(name, node) + } + } +} + +// PrintCompileJSON prints compile output as JSON (models, deps, SQL). +func (b *Build) PrintCompileJSON() { + if len(b.SubBuilds) > 0 { + var all []map[string]any + for _, sub := range b.SubBuilds { + all = append(all, sub.compileJSONPayload()) + } + fmt.Println(g.Marshal(all)) + return + } + fmt.Println(g.Marshal(b.compileJSONPayload())) +} + +// CompileJSONPayload returns the --compile --json object. +// Safe when Compile did not finish (e.g. a cycle): order/nodes stay empty. +func (b *Build) CompileJSONPayload() map[string]any { + if b == nil { + return map[string]any{"order": []string{}, "nodes": []map[string]any{}, "target": ""} + } + if b.DAG == nil { + return map[string]any{"order": []string{}, "nodes": []map[string]any{}, "target": b.GetTarget()} + } + return b.compileJSONPayload() +} + +func (b *Build) compileJSONPayload() map[string]any { + nodes := make([]map[string]any, 0, len(b.Selected)) + for _, name := range b.Selected { + node := b.DAG.Nodes[name] + if node == nil { + continue + } + m := map[string]any{"name": name} + if node.Seed != nil { + m["type"] = "seed" + m["table"] = node.Seed.FullTableName + m["file"] = filepath.ToSlash(node.Seed.RelPath) + } else if node.Model != nil { + m["type"] = "model" + m["table"] = node.Model.FullTableName + m["file"] = filepath.ToSlash(node.Model.RelPath) + m["mode"] = b.GetModelMode(node.Model) + m["dependencies"] = node.Dependencies + m["sql"] = node.Model.CompiledSQL + if len(node.Model.Config.Tests) > 0 { + m["tests"] = node.Model.Config.Tests + } + } + nodes = append(nodes, m) + } + return map[string]any{ + "order": append([]string{}, b.Selected...), + "nodes": nodes, + "target": b.GetTarget(), + } +} + +// printSeedYAML prints a seed node in YAML format. +func (b *Build) printSeedYAML(name string, node *DAGNode) { + seed := node.Seed + fmt.Printf("%s:\n", name) + fmt.Printf(" type: seed\n") + fmt.Printf(" table: %s\n", seed.FullTableName) + fmt.Printf(" file: %s\n", filepath.ToSlash(seed.RelPath)) + fmt.Printf(" format: %s\n", seed.Format) +} + +// printModelYAML prints a model node in YAML format. +func (b *Build) printModelYAML(name string, node *DAGNode) { + model := node.Model + mode := b.GetModelMode(model) + + fmt.Printf("%s:\n", name) + fmt.Printf(" type: model\n") + fmt.Printf(" table: %s\n", model.FullTableName) + fmt.Printf(" file: %s\n", filepath.ToSlash(model.RelPath)) + fmt.Printf(" mode: %s\n", mode) + + // Incremental details + if model.Config.UniqueKey != nil { + fmt.Printf(" unique_key: %s\n", formatYAMLValue(model.Config.UniqueKey)) + } + if model.Config.MergeStrategy != "" { + fmt.Printf(" merge_strategy: %s\n", model.Config.MergeStrategy) + } + if model.Config.UpdateKey != "" { + fmt.Printf(" update_key: %s\n", model.Config.UpdateKey) + } + + // Tags + if len(model.Config.Tags) > 0 { + fmt.Printf(" tags: %s\n", formatYAMLValue(model.Config.Tags)) + } + + // Dependencies + fmt.Printf(" dependencies: %s\n", formatYAMLValue(node.Dependencies)) + + // Hooks + if !model.Config.Hooks.IsEmpty() { + if len(model.Config.Hooks.Start) > 0 { + fmt.Printf(" start_hooks:\n") + for _, h := range model.Config.Hooks.Start { + fmt.Printf(" - %s\n", g.Marshal(h)) + } + } + if len(model.Config.Hooks.End) > 0 { + fmt.Printf(" end_hooks:\n") + for _, h := range model.Config.Hooks.End { + fmt.Printf(" - %s\n", g.Marshal(h)) + } + } + } + + // Pre-statements + if len(model.PreStatements) > 0 { + fmt.Printf(" pre_statements:\n") + for _, stmt := range model.PreStatements { + fmt.Printf(" - |\n") + for _, line := range strings.Split(strings.TrimRight(stmt, "\n"), "\n") { + fmt.Printf(" %s\n", line) + } + } + } + + // Compiled SQL + fmt.Printf(" sql: |\n") + for _, line := range strings.Split(strings.TrimRight(model.CompiledSQL, "\n"), "\n") { + fmt.Printf(" %s\n", line) + } + + // Post-statements + if len(model.PostStatements) > 0 { + fmt.Printf(" post_statements:\n") + for _, stmt := range model.PostStatements { + fmt.Printf(" - |\n") + for _, line := range strings.Split(strings.TrimRight(stmt, "\n"), "\n") { + fmt.Printf(" %s\n", line) + } + } + } +} + +// formatYAMLValue formats a value as inline YAML. +func formatYAMLValue(v any) string { + switch val := v.(type) { + case []string: + if len(val) == 0 { + return "[]" + } + items := make([]string, len(val)) + for i, s := range val { + items[i] = s + } + return "[" + strings.Join(items, ", ") + "]" + case string: + return val + default: + return g.Marshal(v) + } +} + +/////////////////////////////////// golyglot + +func init() { + os.Setenv("GOLYGLOT_LIBRARY_FOLDER", filepath.Join(env.HomeDir, "lib", "golyglot")) +} + +// ModelSQL holds the split result of a multi-statement SQL model file. +type ModelSQL struct { + PreStatements []string + ModelQuery string + PostStatements []string +} + +// MakeModelSQL splits a SQL model file into pre-statements, the model query, +// and post-statements. The model query is the single SELECT/WITH/UNION statement. +// All other statements are classified as pre (before) or post (after) the query. +func MakeModelSQL(sql string, dbType dbio.Type) (*ModelSQL, error) { + dialect := mapDialect(dbType) + pre, model, post, err := SplitModelSQL(sql, dialect) + if err != nil { + return nil, err + } + return &ModelSQL{ + PreStatements: pre, + ModelQuery: model, + PostStatements: post, + }, nil +} + +// mapDialect converts a dbio.Type to a polyglot dialect string. +func mapDialect(dbType dbio.Type) string { + switch dbType { + case dbio.TypeDbPostgres: + return "postgresql" + case dbio.TypeDbRedshift: + return "redshift" + case dbio.TypeDbMySQL, dbio.TypeDbMariaDB, dbio.TypeDbStarRocks: + return "mysql" + case dbio.TypeDbSQLServer, dbio.TypeDbAzure, dbio.TypeDbAzureDWH, dbio.TypeDbFabric: + return "tsql" + case dbio.TypeDbClickhouse, dbio.TypeDbProton: + return "clickhouse" + case dbio.TypeDbBigQuery: + return "bigquery" + case dbio.TypeDbSnowflake: + return "snowflake" + case dbio.TypeDbDuckDb, dbio.TypeDbMotherDuck, dbio.TypeDbDuckLake: + return "duckdb" + case dbio.TypeDbDatabricks: + return "databricks" + case dbio.TypeDbSQLite, dbio.TypeDbD1: + return "sqlite" + case dbio.TypeDbTrino, dbio.TypeDbAthena, dbio.TypeDbIceberg: + return "trino" + default: + return "generic" + } +} + +// SplitModelSQL splits multi-statement SQL into pre-statements, the model query, +// and post-statements. The model query is the single SELECT/WITH/UNION statement. +// +// Returns an error if zero or more than one query statement is found. +func SplitModelSQL(sql, dialect string) (preStatements []string, modelQuery string, postStatements []string, err error) { + // Fast path: if no semicolons outside quotes/comments, it's a single statement + if !containsSemicolon(sql) { + return nil, strings.TrimSpace(sql), nil, nil + } + + stmts, classifyErr := golyglot.ClassifyStatements(sql, dialect) + if classifyErr != nil { + return nil, "", nil, classifyErr + } + + if len(stmts) == 0 { + return nil, "", nil, fmt.Errorf("no SQL statements found") + } + + // Single statement: must be the model + if len(stmts) == 1 { + if stmts[0].Type != golyglot.StmtQuery { + return nil, "", nil, fmt.Errorf("model file contains a single %s statement (%s), expected a SELECT query", stmts[0].Type, stmts[0].TypeKey) + } + return nil, stmts[0].SQL, nil, nil + } + + // Multiple statements: find the query + queryIdx := -1 + queryCount := 0 + for i, stmt := range stmts { + if stmt.Type == golyglot.StmtQuery { + queryIdx = i + queryCount++ + } + } + + if queryCount == 0 { + return nil, "", nil, fmt.Errorf("no SELECT query found in model file; found %d statements but none are queries", len(stmts)) + } + if queryCount > 1 { + return nil, "", nil, fmt.Errorf("found %d SELECT queries in model file; expected exactly 1", queryCount) + } + + // Split around the query + for _, stmt := range stmts[:queryIdx] { + preStatements = append(preStatements, stmt.SQL) + } + modelQuery = stmts[queryIdx].SQL + for _, stmt := range stmts[queryIdx+1:] { + postStatements = append(postStatements, stmt.SQL) + } + + return preStatements, modelQuery, postStatements, nil +} + +// containsSemicolon checks if SQL contains a semicolon outside of quotes, comments, +// and Postgres-style dollar-quoting ($tag$ ... $tag$). +// This is a quick heuristic to skip WASM/FFI for simple single-statement files. +func containsSemicolon(sql string) bool { + inSingleQuote := false + inDoubleQuote := false + inLineComment := false + inBlockComment := false + // dollarTag is non-empty when inside $tag$...$tag$ (Postgres dollar quotes) + dollarTag := "" + + for i := 0; i < len(sql); i++ { + c := sql[i] + + if dollarTag != "" { + // Look for closing tag + if c == '$' && i+len(dollarTag) <= len(sql) && sql[i:i+len(dollarTag)] == dollarTag { + i += len(dollarTag) - 1 + dollarTag = "" + } + continue + } + + if inLineComment { + if c == '\n' { + inLineComment = false + } + continue + } + if inBlockComment { + if c == '*' && i+1 < len(sql) && sql[i+1] == '/' { + inBlockComment = false + i++ + } + continue + } + if inSingleQuote { + if c == '\'' { + if i+1 < len(sql) && sql[i+1] == '\'' { + i++ // escaped quote + } else { + inSingleQuote = false + } + } + continue + } + if inDoubleQuote { + if c == '"' { + inDoubleQuote = false + } + continue + } + + switch c { + case '\'': + inSingleQuote = true + case '"': + inDoubleQuote = true + case '$': + // Start of dollar quote: $tag$ or $$ + j := i + 1 + for j < len(sql) && ((sql[j] >= 'a' && sql[j] <= 'z') || + (sql[j] >= 'A' && sql[j] <= 'Z') || + (sql[j] >= '0' && sql[j] <= '9') || sql[j] == '_') { + j++ + } + if j < len(sql) && sql[j] == '$' { + dollarTag = sql[i : j+1] + i = j + } + case '-': + if i+1 < len(sql) && sql[i+1] == '-' { + inLineComment = true + i++ + } + case '/': + if i+1 < len(sql) && sql[i+1] == '*' { + inBlockComment = true + i++ + } + case ';': + return true + } + } + return false +} diff --git a/core/sling/build/build_test.go b/core/sling/build/build_test.go new file mode 100644 index 000000000..8667d4a10 --- /dev/null +++ b/core/sling/build/build_test.go @@ -0,0 +1,484 @@ +package build + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewBuildSampleProject(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{Target: "POSTGRES"}) + require.NoError(t, err) + assert.NotNil(t, b.Project) + assert.Equal(t, "POSTGRES", b.GetTarget()) +} + +func TestNewBuildNoTarget(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{}) + require.NoError(t, err) + + // Compile should fail because no target override and yml has target + // Actually sample_project has target: POSTGRES in sling_build.yml + err = b.Compile() + assert.NoError(t, err) // should succeed since yml has target +} + +func TestCompileSampleProject(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{Target: "POSTGRES"}) + require.NoError(t, err) + + err = b.Compile() + require.NoError(t, err) + + // DAG should be built + assert.NotNil(t, b.DAG) + assert.NotEmpty(t, b.DAG.Order) + + // All models + seeds should be selected (no selector) + assert.Len(t, b.Selected, 8) // 6 models + 2 seeds + + // Verify DAG contains all nodes + assert.Contains(t, b.DAG.Nodes, "stg_orders") + assert.Contains(t, b.DAG.Nodes, "stg_customers") + assert.Contains(t, b.DAG.Nodes, "dim_customers") + assert.Contains(t, b.DAG.Nodes, "fct_orders") + assert.Contains(t, b.DAG.Nodes, "revenue") + assert.Contains(t, b.DAG.Nodes, "raw") + assert.Contains(t, b.DAG.Nodes, "country_codes") + assert.Contains(t, b.DAG.Nodes, "status_map") + + // Verify dependencies resolved correctly + fctOrders := b.DAG.Nodes["fct_orders"] + assert.Contains(t, fctOrders.Dependencies, "stg_orders") + + dimCustomers := b.DAG.Nodes["dim_customers"] + assert.Contains(t, dimCustomers.Dependencies, "stg_customers") + + revenue := b.DAG.Nodes["revenue"] + assert.Contains(t, revenue.Dependencies, "fct_orders") +} + +func TestCompileWithSelector(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{ + Target: "POSTGRES", + Select: []string{"stg_*"}, + }) + require.NoError(t, err) + + err = b.Compile() + require.NoError(t, err) + + // Should only select stg_orders and stg_customers + assert.Len(t, b.Selected, 2) + assert.Contains(t, b.Selected, "stg_orders") + assert.Contains(t, b.Selected, "stg_customers") +} + +func TestCompileWithExclude(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{ + Target: "POSTGRES", + Exclude: []string{"raw"}, + }) + require.NoError(t, err) + + err = b.Compile() + require.NoError(t, err) + + // Should have all except 'raw' + assert.Len(t, b.Selected, 7) + assert.NotContains(t, b.Selected, "raw") +} + +func TestCompileWithUpstreamSelector(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{ + Target: "POSTGRES", + Select: []string{"+revenue"}, + }) + require.NoError(t, err) + + err = b.Compile() + require.NoError(t, err) + + // +revenue should include revenue + all upstream: fct_orders, stg_orders + assert.Contains(t, b.Selected, "revenue") + assert.Contains(t, b.Selected, "fct_orders") + assert.Contains(t, b.Selected, "stg_orders") +} + +func TestCompileNoTargetError(t *testing.T) { + dir := t.TempDir() + + b, err := NewBuild(dir, BuildOptions{}) + require.NoError(t, err) + + err = b.Compile() + assert.Error(t, err) + assert.Contains(t, err.Error(), "No target specified") +} + +func TestGetModelMode(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{Target: "POSTGRES"}) + require.NoError(t, err) + require.NoError(t, b.Compile()) + + // dim_customers has config(mode='view') + dimCustomers := b.Project.Models["dim_customers"] + assert.Equal(t, "view", b.GetModelMode(dimCustomers)) + + // fct_orders has config(mode='incremental') + fctOrders := b.Project.Models["fct_orders"] + assert.Equal(t, "incremental", b.GetModelMode(fctOrders)) + + // stg_orders has no config mode, should use project defaults + stgOrders := b.Project.Models["stg_orders"] + assert.Equal(t, "full-refresh", b.GetModelMode(stgOrders)) + + // raw has no config mode, should use project defaults + raw := b.Project.Models["raw"] + assert.Equal(t, "full-refresh", b.GetModelMode(raw)) +} + +func TestGetModelModeFullRefreshOverride(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{Target: "POSTGRES", FullRefresh: true}) + require.NoError(t, err) + require.NoError(t, b.Compile()) + + // Even view models should be full-refresh when --full-refresh is set + dimCustomers := b.Project.Models["dim_customers"] + assert.Equal(t, "full-refresh", b.GetModelMode(dimCustomers)) + + fctOrders := b.Project.Models["fct_orders"] + assert.Equal(t, "full-refresh", b.GetModelMode(fctOrders)) +} + +func TestCompileDevMode(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{ + Target: "POSTGRES", + Schema: "dev_test", + }) + require.NoError(t, err) + require.NoError(t, b.Compile()) + + // All models should be in dev_test schema + for _, model := range b.Project.Models { + assert.Equal(t, "dev_test", model.Schema, "model %s should use dev schema", model.Name) + } +} + +func TestCompileMultiTarget(t *testing.T) { + dir := getTestFixturePath("multi_target_project") + + b, err := NewBuild(dir, BuildOptions{Recursive: true}) + require.NoError(t, err) + + // Multi-target projects have sub-projects + assert.Len(t, b.Project.SubProjects, 2) + + // Compile should succeed (sub-projects are handled separately) + err = b.Compile() + assert.NoError(t, err) +} + +func TestCompileNestedConfig(t *testing.T) { + dir := getTestFixturePath("nested_yml_project") + + b, err := NewBuild(dir, BuildOptions{Target: "POSTGRES", Recursive: true}) + require.NoError(t, err) + require.NoError(t, b.Compile()) + + // stg_orders should have mode=truncate (from child config) + stgOrders := b.Project.Models["stg_orders"] + assert.Equal(t, "truncate", b.GetModelMode(stgOrders)) + + // dim_customers should have mode=full-refresh (from root config) + dimCustomers := b.Project.Models["dim_customers"] + assert.Equal(t, "full-refresh", b.GetModelMode(dimCustomers)) +} + +func TestPrintCompileOutput(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{Target: "POSTGRES"}) + require.NoError(t, err) + require.NoError(t, b.Compile()) + + // Just verify it doesn't panic — output goes to stdout + b.PrintCompileOutput() +} + +func TestCompileEmptyProject(t *testing.T) { + dir := t.TempDir() + + b, err := NewBuild(dir, BuildOptions{Target: "POSTGRES"}) + require.NoError(t, err) + require.NoError(t, b.Compile()) + + assert.NotNil(t, b.DAG) + assert.Len(t, b.Selected, 0) +} + +func TestGetTarget(t *testing.T) { + dir := getTestFixturePath("sample_project") + + // Target from yml + b, err := NewBuild(dir, BuildOptions{}) + require.NoError(t, err) + assert.Equal(t, "POSTGRES", b.GetTarget()) + + // CLI target overrides yml + b, err = NewBuild(dir, BuildOptions{Target: "CLICKHOUSE"}) + require.NoError(t, err) + assert.Equal(t, "CLICKHOUSE", b.GetTarget()) + + // Empty project, target from options + b, err = NewBuild(t.TempDir(), BuildOptions{Target: "MY_DB"}) + require.NoError(t, err) + assert.Equal(t, "MY_DB", b.GetTarget()) +} + +func TestCompileDbtCompatProject(t *testing.T) { + dir := getTestFixturePath("dbt_compat_project") + + b, err := NewBuild(dir, BuildOptions{Target: "POSTGRES"}) + require.NoError(t, err) + require.NoError(t, b.Compile()) + + assert.Len(t, b.Selected, 2) // 1 model + 1 seed + assert.Contains(t, b.Selected, "stg_orders") + assert.Contains(t, b.Selected, "country_codes") +} + +func TestSplitModelSQL_SingleSelect(t *testing.T) { + sql := `SELECT id, name FROM customers WHERE active = true` + + pre, model, post, err := SplitModelSQL(sql, "postgres") + if err != nil { + t.Fatalf("SplitModelSQL failed: %v", err) + } + if len(pre) != 0 { + t.Errorf("expected 0 pre-statements, got %d", len(pre)) + } + if model == "" { + t.Error("model query is empty") + } + if len(post) != 0 { + t.Errorf("expected 0 post-statements, got %d", len(post)) + } + t.Logf("model: %s", model) +} + +func TestSplitModelSQL_PreAndPost(t *testing.T) { + sql := `CREATE TEMP TABLE raw_orders AS SELECT * FROM source_orders; +CREATE INDEX idx_raw ON raw_orders(id); + +SELECT id, customer_name, total FROM raw_orders; + +DROP TABLE IF EXISTS raw_orders; +ANALYZE staging.dim_orders;` + + pre, model, post, err := SplitModelSQL(sql, "postgres") + if err != nil { + t.Fatalf("SplitModelSQL failed: %v", err) + } + + if len(pre) != 2 { + t.Errorf("expected 2 pre-statements, got %d", len(pre)) + } + if model == "" { + t.Error("model query is empty") + } + if len(post) != 2 { + t.Errorf("expected 2 post-statements, got %d", len(post)) + } + + t.Logf("pre[0]: %s", pre[0]) + t.Logf("pre[1]: %s", pre[1]) + t.Logf("model: %s", model) + t.Logf("post[0]: %s", post[0]) + t.Logf("post[1]: %s", post[1]) +} + +func TestSplitModelSQL_PreOnly(t *testing.T) { + sql := `CREATE TEMP TABLE tmp AS SELECT 1 AS x; +SELECT * FROM tmp;` + + pre, model, post, err := SplitModelSQL(sql, "postgres") + if err != nil { + t.Fatalf("SplitModelSQL failed: %v", err) + } + + if len(pre) != 1 { + t.Errorf("expected 1 pre-statement, got %d", len(pre)) + } + if model == "" { + t.Error("model query is empty") + } + if len(post) != 0 { + t.Errorf("expected 0 post-statements, got %d", len(post)) + } +} + +func TestSplitModelSQL_PostOnly(t *testing.T) { + sql := `SELECT * FROM orders; +DROP TABLE IF EXISTS tmp;` + + pre, model, post, err := SplitModelSQL(sql, "postgres") + if err != nil { + t.Fatalf("SplitModelSQL failed: %v", err) + } + + if len(pre) != 0 { + t.Errorf("expected 0 pre-statements, got %d", len(pre)) + } + if model == "" { + t.Error("model query is empty") + } + if len(post) != 1 { + t.Errorf("expected 1 post-statement, got %d", len(post)) + } +} + +func TestSplitModelSQL_ZeroQueries(t *testing.T) { + sql := `CREATE TABLE t (id INT); +DROP TABLE t;` + + _, _, _, err := SplitModelSQL(sql, "postgres") + if err == nil { + t.Fatal("expected error for zero queries") + } + t.Logf("expected error: %v", err) +} + +func TestSplitModelSQL_MultipleQueries(t *testing.T) { + sql := `SELECT 1; +SELECT 2;` + + _, _, _, err := SplitModelSQL(sql, "postgres") + if err == nil { + t.Fatal("expected error for multiple queries") + } + t.Logf("expected error: %v", err) +} + +func TestSplitModelSQL_CTE(t *testing.T) { + sql := `WITH cte AS (SELECT id FROM raw_data) +SELECT * FROM cte WHERE id > 0` + + pre, model, post, err := SplitModelSQL(sql, "postgres") + if err != nil { + t.Fatalf("SplitModelSQL failed: %v", err) + } + if len(pre) != 0 || len(post) != 0 { + t.Errorf("CTE should have no pre/post: pre=%d, post=%d", len(pre), len(post)) + } + if model == "" { + t.Error("model is empty") + } + t.Logf("model: %s", model) +} + +func TestSplitModelSQL_UnionAll(t *testing.T) { + sql := `SELECT 1 AS id UNION ALL SELECT 2 AS id` + + pre, model, post, err := SplitModelSQL(sql, "postgres") + if err != nil { + t.Fatalf("SplitModelSQL failed: %v", err) + } + if len(pre) != 0 || len(post) != 0 { + t.Errorf("UNION should have no pre/post: pre=%d, post=%d", len(pre), len(post)) + } + if model == "" { + t.Error("model is empty") + } + t.Logf("model: %s", model) +} + +func TestSplitModelSQL_CreateTableAsSelect(t *testing.T) { + // CREATE TABLE AS SELECT should be classified as DDL, not query + sql := `CREATE TEMP TABLE staging AS SELECT * FROM raw; +SELECT id, name FROM staging;` + + pre, model, post, err := SplitModelSQL(sql, "postgres") + if err != nil { + t.Fatalf("SplitModelSQL failed: %v", err) + } + if len(pre) != 1 { + t.Errorf("CTAS should be a pre-statement, got %d pre", len(pre)) + } + if model == "" { + t.Error("model is empty") + } + if len(post) != 0 { + t.Errorf("expected 0 post, got %d", len(post)) + } + t.Logf("pre[0]: %s", pre[0]) + t.Logf("model: %s", model) +} + +func TestSplitModelSQL_SemicolonInStringLiteral(t *testing.T) { + // The semicolon inside the string literal should NOT cause a split + sql := `SELECT 'hello; world' AS greeting` + + pre, model, post, err := SplitModelSQL(sql, "postgres") + if err != nil { + t.Fatalf("SplitModelSQL failed: %v", err) + } + if len(pre) != 0 || len(post) != 0 { + t.Errorf("semicolon in string should not split: pre=%d, post=%d", len(pre), len(post)) + } + if model == "" { + t.Error("model is empty") + } +} + +func TestSplitModelSQL_SemicolonInComment(t *testing.T) { + sql := `-- this is a comment; with semicolon +SELECT 1` + + pre, model, post, err := SplitModelSQL(sql, "postgres") + if err != nil { + t.Fatalf("SplitModelSQL failed: %v", err) + } + if len(pre) != 0 || len(post) != 0 { + t.Errorf("semicolon in comment should not split: pre=%d, post=%d", len(pre), len(post)) + } + if model == "" { + t.Error("model is empty") + } +} + +func TestStyleCheck(t *testing.T) { + sql := `{%- config(mode='incremental', unique_key='id', merge_strategy='delete+insert', update_key='created_at') -%} + +SELECT + id, + name, + created_at +FROM {{ ref('stg_orders') }} +{% if is_incremental() %} +WHERE created_at > (SELECT MAX(created_at) FROM {{ this }}) +{% endif %}` + style, err := detectModelStyle(sql) + fmt.Println("style:", style, "err:", err) + fmt.Println("StyleDbt:", StyleDbt, "StyleSling:", StyleSling) +} diff --git a/core/sling/build/ddl.go b/core/sling/build/ddl.go new file mode 100644 index 000000000..f2c0527de --- /dev/null +++ b/core/sling/build/ddl.go @@ -0,0 +1,284 @@ +package build + +import ( + "strings" + + "github.com/flarco/g" + "github.com/slingdata-io/sling-cli/core/dbio" + "github.com/slingdata-io/sling-cli/core/dbio/database" +) + +// quoteFullTableName parses schema.table and returns a dialect-quoted FDQN. +func (e *Executor) quoteFullTableName(fullName string) (string, error) { + table, err := database.ParseTableName(fullName, e.DbConn.GetType()) + if err != nil { + return "", g.Error(err, "could not parse table name '%s'", fullName) + } + return table.FDQN(), nil +} + +// quotedName returns just the quoted bare table name (no schema). +func (e *Executor) quotedName(name string) string { + return e.DbConn.Quote(name) +} + +// supportsDropCascade reports whether the dialect accepts CASCADE on DROP. +func supportsDropCascade(t dbio.Type) bool { + return g.In(t, + dbio.TypeDbPostgres, dbio.TypeDbRedshift, + dbio.TypeDbDuckDb, dbio.TypeDbMotherDuck, dbio.TypeDbDuckLake, + dbio.TypeDbSnowflake, // accepted (no-op-ish) but harmless + ) +} + +// supportsCreateOrReplaceTable reports dialects with atomic CREATE OR REPLACE TABLE. +func supportsCreateOrReplaceTable(t dbio.Type) bool { + return g.In(t, + dbio.TypeDbSnowflake, + dbio.TypeDbBigQuery, + dbio.TypeDbDuckDb, dbio.TypeDbMotherDuck, dbio.TypeDbDuckLake, + dbio.TypeDbDatabricks, + ) +} + +// supportsRenameTable reports dialects with a rename_table template. +func (e *Executor) supportsRenameTable() bool { + return strings.TrimSpace(e.DbConn.GetTemplateValue("core.rename_table")) != "" +} + +// isSQLServerFamily reports SQL Server / Azure SQL / Fabric dialects. +func isSQLServerFamily(t dbio.Type) bool { + return g.In(t, dbio.TypeDbSQLServer, dbio.TypeDbAzure, dbio.TypeDbAzureDWH, dbio.TypeDbFabric) +} + +// wantCascade returns whether CASCADE should be used for this model. +func (e *Executor) wantCascade(model *Model) bool { + if model != nil && model.Config.DropCascade != nil { + return *model.Config.DropCascade + } + if e.Build != nil && e.Build.Project != nil && e.Build.Project.Config != nil { + if e.Build.Project.Config.Defaults.DropCascade != nil { + return *e.Build.Project.Config.Defaults.DropCascade + } + } + return false // safe default: no CASCADE +} + +// dropTable drops a table using the dialect template. When cascade is requested +// and the dialect supports it, CASCADE is appended. On failure due to dependents, +// the error message hints at drop_cascade: true. +func (e *Executor) dropTable(fullName string, cascade bool) error { + quoted, err := e.quoteFullTableName(fullName) + if err != nil { + return err + } + + sql := g.R(e.DbConn.GetTemplateValue("core.drop_table"), "table", quoted) + if cascade && supportsDropCascade(e.DbConn.GetType()) { + // Avoid double CASCADE if template already has it + if !strings.Contains(strings.ToUpper(sql), "CASCADE") { + sql = strings.TrimRight(sql, "; \t\n") + " CASCADE" + } + } + + if _, err := e.DbConn.Exec(sql); err != nil { + errLower := strings.ToLower(err.Error()) + // Ignore "does not exist" style errors + ignore := e.DbConn.Template().Variable["error_ignore_drop_table"] + if ignore != "" && strings.Contains(errLower, strings.ToLower(ignore)) { + return nil + } + if strings.Contains(errLower, "does not exist") || + strings.Contains(errLower, "unknown table") || + strings.Contains(errLower, "cannot find") || + strings.Contains(errLower, "not found") { + return nil + } + // Dependent objects often surface as "depends on" / "dependent" / FK errors + if !cascade && (strings.Contains(errLower, "depend") || + strings.Contains(errLower, "referenced by") || + strings.Contains(errLower, "foreign key")) { + return g.Error(err, "could not drop table %s (dependent objects may exist; set drop_cascade: true to force)", fullName) + } + return g.Error(err, "could not drop table %s", fullName) + } + return nil +} + +// dropView drops a view using the dialect template. +func (e *Executor) dropView(fullName string, cascade bool) error { + quoted, err := e.quoteFullTableName(fullName) + if err != nil { + return err + } + + sql := g.R(e.DbConn.GetTemplateValue("core.drop_view"), "view", quoted) + if cascade && supportsDropCascade(e.DbConn.GetType()) { + if !strings.Contains(strings.ToUpper(sql), "CASCADE") { + sql = strings.TrimRight(sql, "; \t\n") + " CASCADE" + } + } + + if _, err := e.DbConn.Exec(sql); err != nil { + errLower := strings.ToLower(err.Error()) + ignore := e.DbConn.Template().Variable["error_ignore_drop_view"] + if ignore != "" && strings.Contains(errLower, strings.ToLower(ignore)) { + return nil + } + if strings.Contains(errLower, "does not exist") || + strings.Contains(errLower, "unknown") || + strings.Contains(errLower, "cannot find") || + strings.Contains(errLower, "not found") { + return nil + } + if !cascade && (strings.Contains(errLower, "depend") || + strings.Contains(errLower, "referenced by")) { + return g.Error(err, "could not drop view %s (dependent objects may exist; set drop_cascade: true to force)", fullName) + } + return g.Error(err, "could not drop view %s", fullName) + } + return nil +} + +// truncateTable truncates via the dialect template. +func (e *Executor) truncateTable(fullName string) error { + quoted, err := e.quoteFullTableName(fullName) + if err != nil { + return err + } + sql := g.R(e.DbConn.GetTemplateValue("core.truncate_table"), "table", quoted) + if strings.TrimSpace(sql) == "" { + sql = g.F("TRUNCATE TABLE %s", quoted) + } + _, err = e.DbConn.Exec(sql) + return err +} + +// insertSelect inserts the result of a SELECT into an existing table. +func (e *Executor) insertSelect(fullName, selectSQL string) error { + quoted, err := e.quoteFullTableName(fullName) + if err != nil { + return err + } + _, err = e.DbConn.Exec(g.F("INSERT INTO %s (%s)", quoted, selectSQL)) + return err +} + +// createTableAs creates a table from a SELECT, dialect-aware. +// model may be nil for temp tables that don't need ClickHouse engine config. +func (e *Executor) createTableAs(fullName, selectSQL string, model *Model) error { + quoted, err := e.quoteFullTableName(fullName) + if err != nil { + return err + } + dbType := e.DbConn.GetType() + + if e.isClickHouse() { + engineClause := "ENGINE = Memory" + orderByClause := "ORDER BY tuple()" + settings := "" + if model != nil { + engineClause = e.getEngineClause(model) + orderByClause = e.getOrderByClause(model) + // MergeTree sorting keys cannot be nullable. allow_nullable_key + // lets incremental unique_key columns work without wrapping every + // SELECT in assumeNotNull + if !strings.Contains(strings.ToUpper(orderByClause), "TUPLE()") { + settings = " SETTINGS allow_nullable_key = 1" + } + } + _, err = e.DbConn.Exec(g.F("CREATE TABLE %s %s %s%s AS (%s)", quoted, engineClause, orderByClause, settings, selectSQL)) + return err + } + + if isSQLServerFamily(dbType) { + // SQL Server has no CTAS; use SELECT INTO + _, err = e.DbConn.Exec(g.F("SELECT * INTO %s FROM (%s) AS _sling_src", quoted, selectSQL)) + return err + } + + // Standard CTAS (Postgres, MySQL, Snowflake, BigQuery, DuckDB, …) + _, err = e.DbConn.Exec(g.F("CREATE TABLE %s AS (%s)", quoted, selectSQL)) + return err +} + +// createOrReplaceTableAs atomically rebuilds a table from a SELECT when the +// dialect supports CREATE OR REPLACE TABLE. +func (e *Executor) createOrReplaceTableAs(fullName, selectSQL string, model *Model) error { + quoted, err := e.quoteFullTableName(fullName) + if err != nil { + return err + } + if e.isClickHouse() { + // ClickHouse: drop + create (atomic path uses rename elsewhere) + return e.createTableAs(fullName, selectSQL, model) + } + _, err = e.DbConn.Exec(g.F("CREATE OR REPLACE TABLE %s AS (%s)", quoted, selectSQL)) + return err +} + +// createOrReplaceView creates/replaces a view, dialect-aware. +func (e *Executor) createOrReplaceView(fullName, selectSQL string) error { + quoted, err := e.quoteFullTableName(fullName) + if err != nil { + return err + } + dbType := e.DbConn.GetType() + + if isSQLServerFamily(dbType) { + // SQL Server 2016+: CREATE OR ALTER VIEW + _, err = e.DbConn.Exec(g.F("CREATE OR ALTER VIEW %s AS %s", quoted, selectSQL)) + return err + } + + _, err = e.DbConn.Exec(g.F("CREATE OR REPLACE VIEW %s AS (%s)", quoted, selectSQL)) + return err +} + +// renameTable renames oldFull → newFull using the dialect template. +// newFull may be schema-qualified; for dialects that want a bare name in +// RENAME TO, only the name component is used. +func (e *Executor) renameTable(oldFull, newFull string) error { + oldQuoted, err := e.quoteFullTableName(oldFull) + if err != nil { + return err + } + newTable, err := database.ParseTableName(newFull, e.DbConn.GetType()) + if err != nil { + return g.Error(err, "could not parse table name '%s'", newFull) + } + + tpl := e.DbConn.GetTemplateValue("core.rename_table") + if strings.TrimSpace(tpl) == "" { + return g.Error("database %s does not support table rename", e.DbConn.GetType()) + } + + // ClickHouse RENAME TABLE a TO b wants fully-qualified both sides. + // Postgres ALTER TABLE a RENAME TO b wants bare new name. + var newRef string + if e.isClickHouse() { + newRef = newTable.FDQN() + } else { + newRef = e.DbConn.Quote(newTable.Name) + } + + sql := g.R(tpl, "table", oldQuoted, "new_table", newRef) + _, err = e.DbConn.Exec(sql) + return err +} + +// bareTableName extracts the unqualified table name from schema.table. +func bareTableName(fullName string) string { + if idx := strings.LastIndex(fullName, "."); idx >= 0 { + return fullName[idx+1:] + } + return fullName +} + +// schemaOf extracts the schema from schema.table. +func schemaOf(fullName string) string { + if idx := strings.LastIndex(fullName, "."); idx >= 0 { + return fullName[:idx] + } + return "" +} diff --git a/core/sling/build/executor.go b/core/sling/build/executor.go new file mode 100644 index 000000000..907650f0a --- /dev/null +++ b/core/sling/build/executor.go @@ -0,0 +1,1644 @@ +package build + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/flarco/g" + "github.com/slingdata-io/sling-cli/core/dbio" + "github.com/slingdata-io/sling-cli/core/dbio/connection" + "github.com/slingdata-io/sling-cli/core/dbio/database" + "github.com/slingdata-io/sling-cli/core/dbio/iop" + "github.com/slingdata-io/sling-cli/core/env" + "github.com/slingdata-io/sling-cli/core/sling" + "github.com/spf13/cast" +) + +// Executor runs a compiled Build against a target database. +type Executor struct { + Build *Build + ConnName string // resolved target connection name + DbConn database.Connection // database connection (nil until Connect) + Results []ExecutionResult // per-node results + RunID string // unique per Execute() for temp table isolation + connEntries connection.ConnEntries // pre-resolved connection entries (optional) + ctx *g.Context + failedSet map[string]bool // tracks failed nodes for skipping downstream + stopped bool // set when fail-fast triggers; prevents new dispatches +} + +// ExecutionResult holds the outcome of executing one node. +type ExecutionResult struct { + Name string + NodeType string // "seed" or "model" + Mode string // "full-refresh", "view", "truncate", "incremental", "append" + Duration time.Duration + Err error + Skipped bool +} + +// BuildState implements sling.RuntimeState for build model hooks. +type BuildState struct { + State map[string]map[string]any `json:"state,omitempty"` + Store map[string]any `json:"store,omitempty"` + Env map[string]any `json:"env,omitempty"` + Timestamp sling.DateTimeState `json:"timestamp,omitempty"` + Model BuildModelState `json:"model,omitempty"` + Target BuildTargetState `json:"target,omitempty"` +} + +// BuildModelState holds model metadata available in hooks. +type BuildModelState struct { + Name string `json:"name,omitempty"` + Schema string `json:"schema,omitempty"` + FullName string `json:"full_name,omitempty"` + Mode string `json:"mode,omitempty"` +} + +// BuildTargetState holds target connection metadata available in hooks. +type BuildTargetState struct { + Name string `json:"name,omitempty"` +} + +func (bs *BuildState) GetStore() map[string]any { return bs.Store } +func (bs *BuildState) SetStoreData(key string, value any, del bool) { + if del { + delete(bs.Store, key) + } else { + bs.Store[key] = value + } +} +func (bs *BuildState) SetStateData(id string, data map[string]any) { bs.State[id] = data } +func (bs *BuildState) SetStateKeyValue(id, key string, value any) { + if bs.State[id] == nil { + bs.State[id] = map[string]any{} + } + bs.State[id][key] = value +} +func (bs *BuildState) Marshall() string { return g.Marshal(bs) } +func (bs *BuildState) TaskExecution() *sling.TaskExecution { return nil } +func (bs *BuildState) StepExecution() *sling.PipelineStepExecution { return nil } + +// NewExecutor creates an Executor from a compiled Build. +func NewExecutor(b *Build) (*Executor, error) { + connName := b.GetTarget() + if connName == "" { + return nil, g.Error("no target connection specified") + } + return &Executor{ + Build: b, + ConnName: connName, + RunID: g.RandString(g.AlphaRunesLower, 6), + connEntries: b.connEntries, + failedSet: make(map[string]bool), + ctx: g.NewContext(context.Background()), + }, nil +} + +// Connect establishes a database connection to the target. +func (e *Executor) Connect() error { + entries := e.connEntries + if entries == nil { + entries = connection.GetLocalConns() + } + entry := entries.Get(e.ConnName) + if entry.Name == "" { + return g.Error("connection '%s' not found", e.ConnName) + } + + dbConn, err := entry.Connection.AsDatabase() + if err != nil { + return g.Error(err, "could not create database connection for '%s'", e.ConnName) + } + + if err := dbConn.Connect(); err != nil { + return g.Error(err, "could not connect to '%s'", e.ConnName) + } + + e.DbConn = dbConn + return nil +} + +// Close closes the database connection. +func (e *Executor) Close() { + if e.DbConn != nil { + e.DbConn.Close() + } +} + +// CreateSchemas creates all unique schemas needed by the selected nodes. +func (e *Executor) CreateSchemas() error { + schemas := make(map[string]bool) + for _, name := range e.Build.Selected { + node := e.Build.DAG.Nodes[name] + if node.Model != nil { + schemas[node.Model.Schema] = true + } + if node.Seed != nil { + schemas[node.Seed.Schema] = true + } + } + + for schema := range schemas { + sql := g.R( + e.DbConn.Template().Value("core.create_schema"), + "schema", e.DbConn.Quote(schema), + ) + if _, err := e.DbConn.Exec(sql); err != nil { + // Ignore "already exists" errors for databases without IF NOT EXISTS + errLower := strings.ToLower(err.Error()) + if !strings.Contains(errLower, "already exists") && !strings.Contains(errLower, "duplicate") { + return g.Error(err, "could not create schema '%s'", schema) + } + } + } + return nil +} + +// Execute runs all selected nodes with a ready-queue scheduler. +// A node is dispatched as soon as its selected dependencies complete (no level barriers). +func (e *Executor) Execute() error { + if err := e.Connect(); err != nil { + return err + } + defer e.Close() + + if err := e.CreateSchemas(); err != nil { + return err + } + + total := len(e.Build.Selected) + if total == 0 { + fmt.Println("No models selected.") + return nil + } + + threads := e.Build.Options.Threads + if threads < 1 { + threads = 1 + } + + if e.DbConn != nil && e.DbConn.GetType().IsSingleWriterDB() && threads > 1 { + g.Debug("capping build threads to 1 for %s target (single writer)", e.DbConn.GetType()) + threads = 1 + } + + selectedSet := make(map[string]bool, total) + for _, name := range e.Build.Selected { + selectedSet[name] = true + } + + // remaining[name] = count of selected deps not yet finished + remaining := make(map[string]int, total) + dependents := make(map[string][]string, total) + for _, name := range e.Build.Selected { + node := e.Build.DAG.Nodes[name] + if node == nil { + continue + } + count := 0 + for _, dep := range node.Dependencies { + if selectedSet[dep] { + count++ + dependents[dep] = append(dependents[dep], name) + } + } + remaining[name] = count + } + + var ( + mu sync.Mutex + ready []string + started = make(map[string]bool, total) + inFlight int + completed int + idx int + failFast = e.Build.Options.FailFast + cond = sync.NewCond(&mu) + ) + + for _, name := range e.Build.Selected { + if remaining[name] == 0 { + ready = append(ready, name) + } + } + + var wg sync.WaitGroup + worker := func() { + defer wg.Done() + for { + mu.Lock() + for len(ready) == 0 && !e.stopped && completed+inFlight < total { + cond.Wait() + } + // Exit when nothing left to do + if len(ready) == 0 { + mu.Unlock() + return + } + if e.stopped { + // Don't start new work under fail-fast + mu.Unlock() + return + } + + name := ready[0] + ready = ready[1:] + if started[name] { + mu.Unlock() + continue + } + started[name] = true + idx++ + nodeIdx := idx + inFlight++ + mu.Unlock() + + result := e.runNode(name, nodeIdx, total) + + mu.Lock() + e.Results = append(e.Results, result) + seedSkipped := result.Skipped && e.Build.Options.NoSeeds && + e.Build.DAG.Nodes[name] != nil && e.Build.DAG.Nodes[name].Seed != nil + // failedSet is also read under e.ctx.Mux in runNode — hold both + e.ctx.Mux.Lock() + if result.Err != nil || (result.Skipped && !seedSkipped) { + e.failedSet[name] = true + } + if result.Err != nil && failFast { + e.stopped = true + } + e.ctx.Mux.Unlock() + for _, depName := range dependents[name] { + remaining[depName]-- + if remaining[depName] == 0 && !started[depName] { + ready = append(ready, depName) + } + } + inFlight-- + completed++ + cond.Broadcast() + mu.Unlock() + + e.printProgress(nodeIdx, total, result, false) + } + } + + nWorkers := threads + if nWorkers > total { + nWorkers = total + } + wg.Add(nWorkers) + for i := 0; i < nWorkers; i++ { + go worker() + } + wg.Wait() + + // Mark any never-started nodes as skipped (fail-fast or dep failure cascade) + for _, name := range e.Build.Selected { + mu.Lock() + wasStarted := started[name] + mu.Unlock() + if wasStarted { + continue + } + result := ExecutionResult{Name: name, Skipped: true} + if node := e.Build.DAG.Nodes[name]; node != nil { + if node.Seed != nil { + result.NodeType = "seed" + result.Mode = "full-refresh" + result.Name = node.Seed.FullTableName + } else if node.Model != nil { + result.NodeType = "model" + result.Mode = e.Build.GetModelMode(node.Model) + result.Name = node.Model.FullTableName + } + } + e.Results = append(e.Results, result) + e.ctx.Mux.Lock() + e.failedSet[name] = true + e.ctx.Mux.Unlock() + } + + fmt.Println() + e.printSummary() + + var errs []error + for _, r := range e.Results { + if r.Err != nil { + errs = append(errs, g.Error(r.Err, "%s", r.Name)) + } + } + if len(errs) > 0 { + msgs := make([]string, len(errs)) + for i, err := range errs { + msgs[i] = err.Error() + } + return g.Error("build completed with %d error(s): %s", len(errs), strings.Join(msgs, "; ")) + } + return nil +} + +// runNode executes a single DAG node and returns its result. +func (e *Executor) runNode(nodeName string, nodeIndex, total int) ExecutionResult { + start := time.Now() + node := e.Build.DAG.Nodes[nodeName] + + var result ExecutionResult + result.Name = nodeName + if node == nil { + result.Err = g.Error("node '%s' not found in DAG", nodeName) + result.Duration = time.Since(start) + return result + } + + e.ctx.Mux.Lock() + skipped := e.isDownstreamOfFailed(nodeName) + e.ctx.Mux.Unlock() + + seedSkipped := !skipped && e.Build.Options.NoSeeds && node.Seed != nil + testOnly := e.Build.Options.Test + if testOnly && node.Seed != nil { + seedSkipped = true + } + + if skipped || seedSkipped { + result.Skipped = true + result.Duration = time.Since(start) + if node.Seed != nil { + result.NodeType = "seed" + result.Mode = "full-refresh" + result.Name = node.Seed.FullTableName + } else if node.Model != nil { + result.NodeType = "model" + result.Mode = e.Build.GetModelMode(node.Model) + result.Name = node.Model.FullTableName + } + return result + } + + if node.Seed != nil { + result.NodeType = "seed" + result.Mode = "full-refresh" + result.Name = node.Seed.FullTableName + e.printProgress(nodeIndex, total, result, true) + result.Err = e.executeSeed(node.Seed) + result.Duration = time.Since(start) + return result + } + + if node.Model != nil { + mode := e.Build.GetModelMode(node.Model) + result.NodeType = "model" + result.Mode = mode + if testOnly { + result.Mode = "test" + } + result.Name = node.Model.FullTableName + e.printProgress(nodeIndex, total, result, true) + if testOnly { + result.Err = e.executeModelTests(node.Model) + } else { + result.Err = e.executeModel(node.Model, mode) + } + result.Duration = time.Since(start) + } + return result +} + +// isDownstreamOfFailed checks if any upstream dependency of the node has failed. +// Must be called under e.mu lock. +func (e *Executor) isDownstreamOfFailed(name string) bool { + node := e.Build.DAG.Nodes[name] + if node == nil { + return false + } + for _, dep := range node.Dependencies { + if e.failedSet[dep] { + return true + } + } + return false +} + +// executeSeed loads a seed file using the sling task infrastructure. +func (e *Executor) executeSeed(seed *Seed) error { + // LoadSeed opens its own target connection. DuckDB-family files allow + // one writer, so release the executor handle first. + if e.DbConn != nil && e.DbConn.GetType().IsSingleWriterDB() { + e.Close() + defer func() { _ = e.Connect() }() + } + return LoadSeed(seed, e.ConnName, true) +} + +// parseModelHooks parses model hooks from the config into executable Hook objects. +func (e *Executor) parseModelHooks(model *Model, mode string) error { + if model.Config.Hooks.IsEmpty() { + return nil + } + + state := &BuildState{ + State: map[string]map[string]any{}, + Store: map[string]any{}, + Env: map[string]any{}, + Model: BuildModelState{ + Name: model.Name, + Schema: model.Schema, + FullName: model.FullTableName, + Mode: mode, + }, + Target: BuildTargetState{ + Name: e.ConnName, + }, + } + state.Timestamp.Update() + + // populate env from Build vars if available + if e.Build != nil && e.Build.Project != nil && e.Build.Project.Config != nil { + for k, v := range e.Build.Project.Config.Vars { + state.Env[k] = v + } + } + + ctx := g.NewContext(context.Background()) + + // parse start hooks + for i, hookRaw := range model.Config.Hooks.Start { + opts := sling.NewParseOptions(sling.HookStageStart, sling.HookKindHook, i, state, ctx) + hook, err := sling.ParseHook(hookRaw, opts) + if err != nil { + return g.Error(err, "error parsing start hook %d for model '%s'", i+1, model.Name) + } + if hook != nil { + model.startHooks = append(model.startHooks, hook) + } + } + + // parse end hooks + for i, hookRaw := range model.Config.Hooks.End { + opts := sling.NewParseOptions(sling.HookStageEnd, sling.HookKindHook, i, state, ctx) + hook, err := sling.ParseHook(hookRaw, opts) + if err != nil { + return g.Error(err, "error parsing end hook %d for model '%s'", i+1, model.Name) + } + if hook != nil { + model.endHooks = append(model.endHooks, hook) + } + } + + return nil +} + +// executeModel executes a single model based on its mode. +func (e *Executor) executeModel(model *Model, mode string) error { + // Parse and execute start hooks + if err := e.parseModelHooks(model, mode); err != nil { + return g.Error(err, "could not parse hooks for model '%s'", model.Name) + } + + if len(model.startHooks) > 0 { + g.Debug("running start hooks for %s", model.Name) + if err := model.startHooks.Execute(); err != nil { + return g.Error(err, "start hook failed for model '%s'", model.Name) + } + } + + // Pre-statements (from multi-statement SQL file) + for i, stmt := range model.PreStatements { + g.Debug("running pre-statement %d/%d for %s", i+1, len(model.PreStatements), model.Name) + if _, err := e.DbConn.Exec(stmt); err != nil { + return g.Error(err, "pre-statement %d failed for model '%s'", i+1, model.Name) + } + } + + var err error + switch mode { + case "full-refresh": + err = e.executeFullRefresh(model) + case "view": + err = e.executeView(model) + case "truncate": + err = e.executeTruncate(model) + case "incremental": + err = e.executeIncremental(model) + case "append", "snapshot": // snapshot is deprecated alias + err = e.executeAppend(model) + default: + err = g.Error("unknown mode '%s' for model '%s'", mode, model.Name) + } + + if err != nil { + return err + } + + // Post-statements (from multi-statement SQL file) + for i, stmt := range model.PostStatements { + g.Debug("running post-statement %d/%d for %s", i+1, len(model.PostStatements), model.Name) + if _, err := e.DbConn.Exec(stmt); err != nil { + return g.Error(err, "post-statement %d failed for model '%s'", i+1, model.Name) + } + } + + // Declarative data tests + if len(model.Config.Tests) > 0 { + if err := e.executeModelTests(model); err != nil { + return err + } + } + + // End hooks + if len(model.endHooks) > 0 { + g.Debug("running end hooks for %s", model.Name) + if err := model.endHooks.Execute(); err != nil { + return g.Error(err, "end hook failed for model '%s'", model.Name) + } + } + + return nil +} + +// executeFullRefresh rebuilds the table. Prefer atomic swap (tmp + rename) or +// CREATE OR REPLACE TABLE where supported so the target is never missing mid-build. +func (e *Executor) executeFullRefresh(model *Model) error { + sql := model.CompiledSQL + cascade := e.wantCascade(model) + + // Drop any existing view first (model may previously have been a view) + _ = e.dropView(model.FullTableName, cascade) + + // Path A: CREATE OR REPLACE TABLE (Snowflake, BigQuery, DuckDB, Databricks) + if supportsCreateOrReplaceTable(e.DbConn.GetType()) && !e.isClickHouse() { + return e.createOrReplaceTableAs(model.FullTableName, sql, model) + } + + // Path B: atomic temp + rename swap when rename is available + if e.supportsRenameTable() { + return e.executeFullRefreshAtomic(model, sql, cascade) + } + + // Path C: fallback — drop then CTAS (brief downtime window) + if err := e.dropTable(model.FullTableName, cascade); err != nil { + return err + } + return e.createTableAs(model.FullTableName, sql, model) +} + +// executeFullRefreshAtomic creates into a temp table, then renames into place. +func (e *Executor) executeFullRefreshAtomic(model *Model, sql string, cascade bool) error { + tmpFull := e.getTempTableName(model) + // Ensure temp is clean + _ = e.dropTable(tmpFull, false) + + if err := e.createTableAs(tmpFull, sql, model); err != nil { + _ = e.dropTable(tmpFull, false) + return g.Error(err, "could not create temp table for full-refresh of '%s'", model.Name) + } + + // Drop target (or rename aside if we want even less downtime — drop is fine + // once data is ready in tmp; window is only drop+rename, not CTAS duration) + if err := e.dropTable(model.FullTableName, cascade); err != nil { + _ = e.dropTable(tmpFull, false) + return err + } + + if err := e.renameTable(tmpFull, model.FullTableName); err != nil { + // Best-effort: leave tmp so data isn't lost + return g.Error(err, "could not rename temp table to '%s' (temp left at %s)", model.FullTableName, tmpFull) + } + return nil +} + +// executeView creates or replaces a view. +func (e *Executor) executeView(model *Model) error { + cascade := e.wantCascade(model) + + // Drop any existing table first (dirty fixtures plant tables that + // staging models then replace with views). CREATE OR REPLACE VIEW + // fails on Postgres when the name is already a table. + if err := e.dropTable(model.FullTableName, cascade); err != nil { + g.Debug("drop table before view creation for %s: %s", model.Name, err) + } + if err := e.dropView(model.FullTableName, cascade); err != nil { + g.Debug("drop view before view creation for %s: %s", model.Name, err) + } + + return e.createOrReplaceView(model.FullTableName, model.CompiledSQL) +} + +// executeTruncate creates the table on first run, truncates + inserts on subsequent runs. +func (e *Executor) executeTruncate(model *Model) error { + sql := model.CompiledSQL + + exists, err := e.tableExists(model) + if err != nil { + return err + } + + if !exists { + return e.createTableAs(model.FullTableName, sql, model) + } + + if err := e.truncateTable(model.FullTableName); err != nil { + return err + } + return e.insertSelect(model.FullTableName, sql) +} + +// executeIncremental dispatches to the correct incremental strategy based on the +// model's detected style. dbt-style models use executeLegacyIncremental (the +// original is_incremental() path); sling-style models use resolveRange + executeRange. +func (e *Executor) executeIncremental(model *Model) error { + uniqueKeys := getUniqueKeys(model) + if len(uniqueKeys) == 0 { + return g.Error("model '%s' uses incremental mode but has no unique_key defined in config()", model.Name) + } + + exists, err := e.tableExists(model) + if err != nil { + return err + } + if !exists || e.Build.Options.FullRefresh { + return e.executeFullRefresh(model) + } + + switch model.Style { + case StyleDbt: + if e.Build.Options.Range != nil { + return g.Error("model '%s' uses is_incremental() (dbt style); --range requires {incremental_where_cond} (sling style)", model.Name) + } + return e.executeLegacyIncremental(model) + case StyleSling: + r, err := e.resolveRange(model) + if err != nil { + return err + } + return e.executeRange(model, r) + default: + return g.Error("model '%s': unknown incremental style %d", model.Name, model.Style) + } +} + +// executeLegacyIncremental is the original dbt-style incremental path, preserved +// byte-for-byte. Recompiles with is_incremental()=true, stages into a temp table, +// and merges using the configured strategy. +func (e *Executor) executeLegacyIncremental(model *Model) error { + t := model.FullTableName + + // Subsequent run: recompile with is_incremental()=true + _, err := e.Build.Engine.CompileModel(model, &IncrementalContext{IsIncremental: true}) + if err != nil { + return g.Error(err, "could not compile incremental SQL for '%s'", model.Name) + } + + // Re-apply table reference rewriting after incremental recompilation + if model.Config.Rewrite == nil || *model.Config.Rewrite { + rewritten, _ := RewriteTableReferences(model.CompiledSQL, e.Build.Project, model.Name) + model.CompiledSQL = rewritten + } + + // Re-split after recompilation since CompiledSQL changed + result, splitErr := MakeModelSQL(model.CompiledSQL, e.DbConn.GetType()) + if splitErr != nil { + return g.Error(splitErr, "could not parse incremental SQL for '%s'", model.Name) + } + model.CompiledSQL = result.ModelQuery + incrementalSQL := model.CompiledSQL + + // Get merge strategy + strategy := e.getMergeStrategy(model) + + // Create temp table with incremental results + tempTable := e.getTempTableName(model) + defer func() { + if dropErr := e.dropTable(tempTable, false); dropErr != nil { + g.Debug("could not drop temp table %s: %s", tempTable, dropErr) + } + }() + + // ClickHouse temp staging uses Memory engine (model=nil → Memory default in createTableAs) + if e.isClickHouse() { + quoted, qErr := e.quoteFullTableName(tempTable) + if qErr != nil { + return qErr + } + _, err = e.DbConn.Exec(g.F("CREATE TABLE %s ENGINE = Memory AS (%s)", quoted, incrementalSQL)) + } else { + err = e.createTableAs(tempTable, incrementalSQL, nil) + } + if err != nil { + return g.Error(err, "could not create temp table for incremental merge on '%s'", model.Name) + } + + // Generate and execute merge SQL using sling's merge infrastructure + uniqueKeys := getUniqueKeys(model) + // Quote target/temp for merge; GenerateMergeSQL expects usable identifiers + tgtQuoted, err := e.quoteFullTableName(t) + if err != nil { + return err + } + tmpQuoted, err := e.quoteFullTableName(tempTable) + if err != nil { + return err + } + mergeSQL, err := e.DbConn.GenerateMergeSQLWithStrategy(tmpQuoted, tgtQuoted, uniqueKeys, &strategy) + if err != nil { + return g.Error(err, "could not generate merge SQL for '%s'", model.Name) + } + + _, err = e.DbConn.ExecMulti(mergeSQL) + if err != nil { + return g.Error(err, "could not execute incremental merge for '%s'", model.Name) + } + + return nil +} + +// executeAppend handles append-only mode (formerly "snapshot"). +// First run: CTAS. Subsequent: INSERT. +func (e *Executor) executeAppend(model *Model) error { + sql := model.CompiledSQL + + exists, err := e.tableExists(model) + if err != nil { + return err + } + + if !exists { + return e.createTableAs(model.FullTableName, sql, model) + } + + return e.insertSelect(model.FullTableName, sql) +} + +// tableExists checks whether the model's target table exists. +func (e *Executor) tableExists(model *Model) (bool, error) { + table, err := database.ParseTableName(model.FullTableName, e.DbConn.GetType()) + if err != nil { + return false, g.Error(err, "could not parse table name '%s'", model.FullTableName) + } + return e.DbConn.TableExists(table) +} + +// isClickHouse returns true if the target database is ClickHouse or Proton. +func (e *Executor) isClickHouse() bool { + return g.In(e.DbConn.GetType(), dbio.TypeDbClickhouse, dbio.TypeDbProton) +} + +// getEngineClause returns the ClickHouse ENGINE clause for a model. +func getEngineClause(model *Model) string { + engine := model.Config.Engine + if engine == "" { + engine = "MergeTree()" + } + return g.F("ENGINE = %s", engine) +} + +// getOrderByClause returns the ClickHouse ORDER BY clause for a model. +// quote, when non-nil, is applied to each key. +func getOrderByClause(model *Model, quote func(string) string) string { + keys := getUniqueKeys(model) + if len(keys) == 0 { + return "ORDER BY tuple()" + } + quoted := make([]string, len(keys)) + for i, k := range keys { + if quote != nil { + quoted[i] = quote(k) + } else { + quoted[i] = k + } + } + return g.F("ORDER BY (%s)", strings.Join(quoted, ", ")) +} + +// getUniqueKeys extracts the unique key(s) from the model config. +func getUniqueKeys(model *Model) []string { + if model.Config.UniqueKey == nil { + return nil + } + switch v := model.Config.UniqueKey.(type) { + case string: + if v == "" { + return nil + } + return []string{v} + case []string: + return v + case []interface{}: + keys := make([]string, 0, len(v)) + for _, k := range v { + keys = append(keys, fmt.Sprint(k)) + } + return keys + } + return nil +} + +// method wrappers that delegate to package-level functions +func (e *Executor) getEngineClause(model *Model) string { return getEngineClause(model) } +func (e *Executor) getOrderByClause(model *Model) string { + return getOrderByClause(model, e.DbConn.Quote) +} + +// getMergeStrategy maps the user-facing merge_strategy string to a database.MergeStrategy constant. +// For ClickHouse, it forces delete+insert since ClickHouse doesn't support UPDATE/MERGE. +func (e *Executor) getMergeStrategy(model *Model) database.MergeStrategy { + return getMergeStrategy(model, e.isClickHouse()) +} + +// getMergeStrategy is the package-level implementation for merge strategy resolution. +func getMergeStrategy(model *Model, isClickHouse bool) database.MergeStrategy { + userStrategy := model.Config.MergeStrategy + + // ClickHouse: force delete+insert + if isClickHouse { + if userStrategy != "" && userStrategy != "delete+insert" { + g.Warn("ClickHouse does not support '%s' merge strategy; using delete+insert instead", userStrategy) + } + return database.MergeStrategyDeleteInsert + } + + switch userStrategy { + case "delete+insert": + return database.MergeStrategyDeleteInsert + case "update+insert": + return database.MergeStrategyUpdateInsert + case "insert": + return database.MergeStrategyInsert + default: + return database.MergeStrategyDeleteInsert + } +} + +// getTempTableName returns a unique schema-qualified temp table for this run. +// The run ID prevents collisions across concurrent builds of the same model. +func (e *Executor) getTempTableName(model *Model) string { + return getTempTableName(model, e.RunID) +} + +// getTempTableName is the package-level implementation. +// Uses a schema-qualified name so that GetColumns() can find the table for merge SQL generation. +func getTempTableName(model *Model, runID string) string { + if runID == "" { + runID = "x" + } + // Keep identifier safe: alphanumeric + underscore only + safeName := strings.Map(func(r rune) rune { + if (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || (r >= '0' && r <= '9') || r == '_' { + return r + } + return '_' + }, model.Name) + tempName := g.F("_sling_build_tmp_%s_%s", safeName, runID) + return g.F("%s.%s", model.Schema, tempName) +} + +// printProgress prints a single line of execution progress. +// When started is true, prints the "START" line (no duration). +// When started is false, prints the final status with duration. +func (e *Executor) printProgress(index, total int, result ExecutionResult, started bool) { + nodeType := result.Mode + if result.NodeType == "seed" { + nodeType = "seed" + } + + // Format: [1/8] staging.country_codes (seed) ........... START + // Format: [1/8] staging.country_codes (seed) ........... OK (0.2s) + prefix := g.F("[%d/%d] %s (%s) ", index, total, result.Name, env.DarkGrayString(nodeType)) + + dotsLen := 70 - len(prefix) + if dotsLen < 3 { + dotsLen = 3 + } + dots := strings.Repeat(".", dotsLen) + + if started { + e.ctx.Info("%s%s %s", prefix, dots, "START") + return + } + + if result.Skipped { + e.ctx.Info("%s%s %s", prefix, dots, env.YellowString("SKIP")) + return + } + + durationStr := formatDuration(result.Duration) + status := env.GreenString("OK") + if result.Err != nil { + status = env.RedString("FAIL") + } + e.ctx.Info("%s%s %s (%s)", prefix, dots, status, durationStr) +} + +// formatDuration formats a duration as a human-readable string. +func formatDuration(d time.Duration) string { + if d < time.Second { + return g.F("%dms", d.Milliseconds()) + } + return g.F("%.1fs", d.Seconds()) +} + +// formatChunkProgressLine returns the indented sub-line for a single range chunk. +// Used for DEBUG-level per-chunk progress during multi-chunk --range backfills. +// Pure function — returns a string; caller decides log level and writer. +func formatChunkProgressLine(idx, total int, chunk RangeChunk, updateKey string, dur time.Duration, failed bool) string { + status := env.GreenString("OK") + if failed { + status = env.RedString("FAIL") + } + return fmt.Sprintf(" chunk %d/%d %s %s (%s)", + idx, total, chunk.Describe(updateKey), status, formatDuration(dur)) +} + +// formatResumeCommand returns the --range value for the resume hint. +// The failed chunk's lower bound becomes the new start; the original +// last chunk's upper bound remains the end. +func formatResumeCommand(failed, last RangeChunk, step string) string { + raw := failed.LowerRaw + "," + last.UpperRaw + if step != "" { + raw = raw + "," + step + } + return raw +} + +// printSummary prints the final execution summary. +func (e *Executor) printSummary() { + var successes, failures, skipped int + var totalDuration time.Duration + var failedResults []ExecutionResult + for _, r := range e.Results { + totalDuration += r.Duration + if r.Skipped { + skipped++ + } else if r.Err != nil { + failures++ + failedResults = append(failedResults, r) + } else { + successes++ + } + } + + successStr := env.GreenString(g.F("%d Successes", successes)) + failureStr := g.F("%d Failures", failures) + if failures > 0 { + failureStr = env.RedString(failureStr) + } else { + failureStr = env.GreenString(failureStr) + } + skippedStr := "" + if skipped > 0 { + skippedStr = g.F(" | %s", env.YellowString(g.F("%d Skipped", skipped))) + } + + g.Info("Build Completed in %s | %s | %s%s\n", g.DurationString(totalDuration), successStr, failureStr, skippedStr) + + // Print errors section + if len(failedResults) > 0 { + fmt.Println(env.RedString("Errors:")) + for _, r := range failedResults { + errMsg := strings.ReplaceAll(strings.TrimSpace(g.ErrMsgSimple(r.Err)), "\n", "\n ") + fmt.Printf(" - %s:\n %s\n", r.Name, env.RedString(errMsg)) + } + } +} + +// Range is the resolved set of chunks for a single build model execution. +type Range struct { + Chunks []RangeChunk // ordered; 0 chunks = no-op + UpdateState bool // advance SLING_STATE after last chunk succeeds + FromCLI bool // came from --range (print resume hint on failure) + CLIRaw string // original raw --range value (for resume hint) + Step string // parsed step for resume hint, may be "" +} + +// RangeChunk is a single [lower, upper) window for one merge pass. +type RangeChunk struct { + Lower string // already-quoted SQL literal, or "" for unbounded + Upper string // already-quoted SQL literal, or "" for unbounded + LowerInclusive bool // true → use >= for lower + ColType iop.ColumnType // for state writes + LowerRaw string // raw display form for logs/resume hint + UpperRaw string // raw display form for logs/resume hint +} + +// WhereCond returns the WHERE clause body for this chunk. +func (c RangeChunk) WhereCond(updateKey string, quote func(string) string) string { + qKey := quote(updateKey) + hasLower := c.Lower != "" && c.Lower != "null" + hasUpper := c.Upper != "" && c.Upper != "null" + switch { + case !hasLower && !hasUpper: + return "1=1" + case !hasLower: + return fmt.Sprintf("%s < %s", qKey, c.Upper) + case !hasUpper: + if c.LowerInclusive { + return fmt.Sprintf("%s >= %s", qKey, c.Lower) + } + return fmt.Sprintf("%s > %s", qKey, c.Lower) + default: + lowerOp := ">" + if c.LowerInclusive { + lowerOp = ">=" + } + return fmt.Sprintf("%s %s %s AND %s < %s", qKey, lowerOp, c.Lower, qKey, c.Upper) + } +} + +// Describe is used for log lines. +func (c RangeChunk) Describe(updateKey string) string { + lower := c.LowerRaw + if lower == "" { + lower = "∅" + } + upper := c.UpperRaw + if upper == "" { + upper = "now" + } + bracket := "(" + if c.LowerInclusive { + bracket = "[" + } + return fmt.Sprintf("%s=%s%s, %s)", updateKey, bracket, lower, upper) +} + +// parseValueAsTime turns a state value into time.Time. Only valid for +// date/datetime column types (enforced by caller). +func parseValueAsTime(value string, colType iop.ColumnType) (time.Time, error) { + if value == "" { + return time.Time{}, g.Error("empty state value cannot be parsed as time") + } + if colType != "" && !colType.IsDatetime() && !colType.IsDate() { + return time.Time{}, g.Error("range.advance/lookback requires datetime/date update_key, got %q", string(colType)) + } + t, err := cast.ToTimeE(value) + if err != nil { + return time.Time{}, g.Error(err, "could not parse value %q as time", value) + } + return t, nil +} + +// quoteValue formats a value as a SQL literal for the given DB. +// Delegates to iop.FormatValue (core/dbio/iop/datatype.go:1643). +func quoteValue(value any, colType iop.ColumnType, dbType dbio.Type) string { + if value == nil { + return "null" + } + if t, ok := value.(time.Time); ok { + if colType == "" { + colType = iop.TimestampType + } + return iop.FormatValue(t, colType, dbType) + } + if colType == "" { + colType = iop.StringType + } + return iop.FormatValue(value, colType, dbType) +} + +// parseCLIRange parses --range "start,end[,step]". +func parseCLIRange(raw string) (start, end, step string, hasStep bool, err error) { + parts := strings.Split(raw, ",") + if len(parts) < 2 || len(parts) > 3 { + return "", "", "", false, g.Error("invalid --range %q: expected 'start,end' or 'start,end,step'", raw) + } + start = strings.TrimSpace(parts[0]) + end = strings.TrimSpace(parts[1]) + if start == "" || end == "" { + return "", "", "", false, g.Error("invalid --range %q: start and end must be non-empty", raw) + } + if len(parts) == 3 { + step = strings.TrimSpace(parts[2]) + if step != "" { + if _, err := parseBuildDuration(step); err != nil { + return "", "", "", false, g.Error(err, "invalid --range step") + } + hasStep = true + } + } + return +} + +// splitCLIRange produces chunks from a raw --range string. colType="" +// makes quoteValue treat values as string literals (ISO dates implicit-cast +// correctly on every dialect). +func splitCLIRange(raw string, dbType dbio.Type, colType iop.ColumnType) (*Range, error) { + start, end, stepStr, hasStep, err := parseCLIRange(raw) + if err != nil { + return nil, err + } + r := &Range{UpdateState: false, FromCLI: true, CLIRaw: raw, Step: stepStr} + if !hasStep { + r.Chunks = []RangeChunk{{ + Lower: quoteValue(start, colType, dbType), + Upper: quoteValue(end, colType, dbType), + LowerInclusive: true, // CLI backfills are inclusive-start + ColType: colType, + LowerRaw: start, + UpperRaw: end, + }} + return r, nil + } + step, _ := parseBuildDuration(stepStr) + startT, err := cast.ToTimeE(start) + if err != nil { + return nil, g.Error(err, "--range with step requires ISO date/timestamp start") + } + endT, err := cast.ToTimeE(end) + if err != nil { + return nil, g.Error(err, "--range with step requires ISO date/timestamp end") + } + for cur := startT; cur.Before(endT); cur = cur.Add(step) { + upper := cur.Add(step) + if upper.After(endT) { + upper = endT + } + r.Chunks = append(r.Chunks, RangeChunk{ + Lower: quoteValue(cur, colType, dbType), + Upper: quoteValue(upper, colType, dbType), + LowerInclusive: true, + ColType: colType, + LowerRaw: cur.Format(time.RFC3339), + UpperRaw: upper.Format(time.RFC3339), + }) + } + return r, nil +} + +// resolveRange decides which range-resolution strategy to use and returns the +// ordered set of chunks to execute. Returns a Range with 0 chunks as a no-op. +func (e *Executor) resolveRange(model *Model) (*Range, error) { + // --range CLI flag wins over all automatic resolution + if e.Build.Options.Range != nil { + return e.resolveCLIRange(model) + } + + rc := model.Config.Range + if rc != nil && rc.HasAdvance() { + return e.resolveAdvanceRange(model) + } + + return e.resolveIncrementalRange(model) +} + +// resolveCLIRange parses the --range flag and produces one or more chunks. +// State is never advanced for CLI backfills. +func (e *Executor) resolveCLIRange(model *Model) (*Range, error) { + raw := *e.Build.Options.Range + dbType := e.DbConn.GetType() + return splitCLIRange(raw, dbType, "") +} + +// resolveIncrementalRange resolves the watermark via tier A/B/C and applies +// optional lookback. This is used when there is no step (plain incremental). +// +// - Tier A: SLING_STATE is configured → read from state store +// - Tier B: target table has rows → SELECT MAX(update_key) +// - Tier C: first run → unbounded lower (full-refresh semantics) +func (e *Executor) resolveIncrementalRange(model *Model) (*Range, error) { + updateKey := model.Config.UpdateKey + if updateKey == "" { + return nil, g.Error("model '%s': sling-style incremental requires update_key in config()", model.Name) + } + + dbType := e.DbConn.GetType() + + var lowerRaw string + var colType iop.ColumnType + lowerInclusive := false + + // Tier A: state store + if sling.IsStateConfigured() { + rec, err := sling.ReadState(model.Name, model.FullTableName) + if err != nil { + return nil, g.Error(err, "could not read SLING_STATE for model '%s'", model.Name) + } + if rec != nil && rec.IsValid() { + lowerRaw = rec.Value + colType = rec.ColumnType + g.Debug("build[%s]: tier A — state value %q", model.Name, lowerRaw) + } + } + + // Tier B: query target table max + if lowerRaw == "" { + maxVal, maxType, err := e.queryTargetMax(model, updateKey) + if err != nil { + g.Warn("build[%s]: tier B probe failed (%s); falling through to first-run", model.Name, err) + } else if maxVal != "" { + lowerRaw = maxVal + colType = maxType + g.Debug("build[%s]: tier B — target MAX %q", model.Name, lowerRaw) + } + } + + // Tier C: first run — unbounded lower + if lowerRaw == "" { + g.Debug("build[%s]: tier C — first run, no lower bound", model.Name) + r := &Range{ + UpdateState: true, + Chunks: []RangeChunk{{ + Lower: "", + Upper: "", + LowerInclusive: false, + ColType: colType, + }}, + } + return r, nil + } + + // Apply lookback + rc := model.Config.Range + if rc != nil && rc.HasLookback() { + dur, err := parseBuildDuration(rc.Lookback) + if err != nil { + return nil, g.Error(err, "model '%s': invalid range.lookback", model.Name) + } + t, err := parseValueAsTime(lowerRaw, colType) + if err != nil { + return nil, g.Error(err, "model '%s': could not apply lookback to state value", model.Name) + } + lowerRaw = t.Add(-dur).Format(time.RFC3339) + lowerInclusive = true + } + + lower := quoteValue(lowerRaw, colType, dbType) + return &Range{ + UpdateState: true, + Chunks: []RangeChunk{{ + Lower: lower, + Upper: "", + LowerInclusive: lowerInclusive, + ColType: colType, + LowerRaw: lowerRaw, + }}, + }, nil +} + +// resolveAdvanceRange resolves a range that moves forward one "advance" window +// per run. Requires SLING_STATE. On first run, probes source for MIN(update_key) +// (or uses range.start). On subsequent runs, advances by one advance-window from +// the last state value. +func (e *Executor) resolveAdvanceRange(model *Model) (*Range, error) { + if !sling.IsStateConfigured() { + return nil, g.Error("model '%s': range.advance requires SLING_STATE to be configured", model.Name) + } + + rc := model.Config.Range + advance, err := parseBuildDuration(rc.Advance) + if err != nil { + return nil, g.Error(err, "model '%s': invalid range.advance", model.Name) + } + + updateKey := model.Config.UpdateKey + if updateKey == "" { + return nil, g.Error("model '%s': range.advance requires update_key in config()", model.Name) + } + + dbType := e.DbConn.GetType() + var colType iop.ColumnType + + // Read existing state + rec, err := sling.ReadState(model.Name, model.FullTableName) + if err != nil { + return nil, g.Error(err, "could not read SLING_STATE for model '%s'", model.Name) + } + + now := time.Now().UTC() + + if rec != nil && rec.IsValid() { + // Subsequent run: advance from last state + stateT, err := parseValueAsTime(rec.Value, rec.ColumnType) + if err != nil { + return nil, g.Error(err, "model '%s': could not parse state value for advance range", model.Name) + } + + lower := stateT + if rc.HasLookback() { + lb, err := parseBuildDuration(rc.Lookback) + if err != nil { + return nil, g.Error(err, "model '%s': invalid range.lookback", model.Name) + } + lower = stateT.Add(-lb) + } + upper := stateT.Add(advance) + if upper.After(now) { + upper = now + } + + if !lower.Before(upper) { + g.Debug("build[%s]: advance range caught up (lower >= upper), no-op", model.Name) + return &Range{UpdateState: false, Chunks: nil}, nil + } + + lowerInclusive := rc.HasLookback() + colType = rec.ColumnType + return &Range{ + UpdateState: true, + Step: rc.Advance, + Chunks: []RangeChunk{{ + Lower: quoteValue(lower, colType, dbType), + Upper: quoteValue(upper, colType, dbType), + LowerInclusive: lowerInclusive, + ColType: colType, + LowerRaw: lower.Format(time.RFC3339), + UpperRaw: upper.Format(time.RFC3339), + }}, + }, nil + } + + // First run: resolve origin + var originT time.Time + if rc.Start != "" { + originT, err = parseValueAsTime(rc.Start, "") + if err != nil { + return nil, g.Error(err, "model '%s': could not parse range.start", model.Name) + } + colType = iop.TimestampType + } else { + // Probe source for MIN(update_key) + minVal, minType, err := e.probeSourceMin(model, updateKey) + if err != nil { + return nil, g.Error(err, "model '%s': could not probe source for range origin", model.Name) + } + if minVal == "" { + g.Debug("build[%s]: advance first-run probe returned empty source; no-op", model.Name) + return &Range{UpdateState: false, Chunks: nil}, nil + } + colType = minType + originT, err = parseValueAsTime(minVal, colType) + if err != nil { + return nil, g.Error(err, "model '%s': could not parse probed origin", model.Name) + } + } + + // Cache origin in state immediately (idempotent on crash/retry) + if err := sling.WriteState(model.Name, model.FullTableName, originT.Format(time.RFC3339), colType); err != nil { + g.Warn("build[%s]: could not cache advance origin in SLING_STATE: %s", model.Name, err) + } + + upper := originT.Add(advance) + if upper.After(now) { + upper = now + } + if !originT.Before(upper) { + return &Range{UpdateState: false, Chunks: nil}, nil + } + + return &Range{ + UpdateState: true, + Step: rc.Advance, + Chunks: []RangeChunk{{ + Lower: quoteValue(originT, colType, dbType), + Upper: quoteValue(upper, colType, dbType), + LowerInclusive: true, // first-run: inclusive of origin + ColType: colType, + LowerRaw: originT.Format(time.RFC3339), + UpperRaw: upper.Format(time.RFC3339), + }}, + }, nil +} + +// probeSourceMin compiles the model with default context, rewrites refs, builds +// the model SQL, and runs SELECT MIN(update_key) FROM () __sling_probe. +func (e *Executor) probeSourceMin(model *Model, updateKey string) (string, iop.ColumnType, error) { + // Save and restore CompiledSQL so the probe doesn't pollute model state + savedSQL := model.CompiledSQL + + _, err := e.Build.Engine.CompileModel(model, DefaultIncrementalContext()) + if err != nil { + model.CompiledSQL = savedSQL + return "", "", g.Error(err, "probeSourceMin: could not compile model '%s'", model.Name) + } + + rewritten, _ := RewriteTableReferences(model.CompiledSQL, e.Build.Project, model.Name) + result, err := MakeModelSQL(rewritten, e.DbConn.GetType()) + model.CompiledSQL = savedSQL // always restore + + if err != nil { + return "", "", g.Error(err, "probeSourceMin: could not parse SQL for model '%s'", model.Name) + } + + probeSQL := fmt.Sprintf("SELECT MIN(%s) FROM (%s) __sling_probe", + e.DbConn.Quote(updateKey), result.ModelQuery) + + data, err := e.DbConn.Query(probeSQL) + if err != nil { + return "", "", g.Error(err, "probeSourceMin: query failed for model '%s'", model.Name) + } + if len(data.Rows) == 0 || len(data.Rows[0]) == 0 || data.Rows[0][0] == nil { + return "", "", nil + } + + var colType iop.ColumnType + if len(data.Columns) > 0 { + colType = data.Columns[0].Type + } + if colType == "" { + colType = iop.TimestampType + } + + return fmt.Sprint(data.Rows[0][0]), colType, nil +} + +// queryTargetMax runs SELECT MAX(update_key) FROM full_table_name. +func (e *Executor) queryTargetMax(model *Model, updateKey string) (string, iop.ColumnType, error) { + quoted, err := e.quoteFullTableName(model.FullTableName) + if err != nil { + return "", "", err + } + sql := fmt.Sprintf("SELECT MAX(%s) FROM %s", + e.DbConn.Quote(updateKey), quoted) + + data, err := e.DbConn.Query(sql) + if err != nil { + return "", "", err + } + if len(data.Rows) == 0 || len(data.Rows[0]) == 0 || data.Rows[0][0] == nil { + return "", "", nil + } + + var colType iop.ColumnType + if len(data.Columns) > 0 { + colType = data.Columns[0].Type + } + if colType == "" { + colType = iop.TimestampType + } + + return fmt.Sprint(data.Rows[0][0]), colType, nil +} + +// executeRange iterates over the chunks in r and runs a merge for each one. +// On any chunk failure it calls handleChunkError and returns. +// On all-success it conditionally advances SLING_STATE. +func (e *Executor) executeRange(model *Model, r *Range) error { + if len(r.Chunks) == 0 { + g.Debug("build[%s]: range resolved to 0 chunks; skipping", model.Name) + return nil + } + + updateKey := model.Config.UpdateKey + if updateKey == "" { + return g.Error("model '%s': sling-style incremental requires update_key in config()", model.Name) + } + + multi := len(r.Chunks) > 1 + for i, chunk := range r.Chunks { + whereCond := chunk.WhereCond(updateKey, e.DbConn.Quote) + valueLit := chunk.Lower + if valueLit == "" || valueLit == "null" { + valueLit = "null" + } + + incCtx := &IncrementalContext{ + IsIncremental: true, + WhereCond: whereCond, + Value: valueLit, + } + + chunkStart := time.Now() + chunkErr := e.runMergeForChunk(model, incCtx) + chunkDur := time.Since(chunkStart) + + if multi { + g.Debug("%s", formatChunkProgressLine(i+1, len(r.Chunks), chunk, updateKey, chunkDur, chunkErr != nil)) + } else { + g.Debug("build[%s]: chunk %d/%d %s (%s)", + model.Name, i+1, len(r.Chunks), chunk.Describe(updateKey), formatDuration(chunkDur)) + } + + if chunkErr != nil { + return e.handleChunkError(model, r, i, chunkErr) + } + } + + // Advance state after successful run + if r.UpdateState && sling.IsStateConfigured() { + if err := e.advanceStateAfterRange(model, r, updateKey); err != nil { + g.Warn("build[%s]: could not advance SLING_STATE: %s", model.Name, err) + } + } + + return nil +} + +// runMergeForChunk compiles the model with incCtx, rewrites refs, and executes +// the temp-table + merge strategy. This is factored from executeLegacyIncremental. +func (e *Executor) runMergeForChunk(model *Model, incCtx *IncrementalContext) error { + t := model.FullTableName + uniqueKeys := getUniqueKeys(model) + + _, err := e.Build.Engine.CompileModel(model, incCtx) + if err != nil { + return g.Error(err, "could not compile incremental SQL for '%s'", model.Name) + } + + // Honor rewrite: false + if model.Config.Rewrite == nil || *model.Config.Rewrite { + rewritten, _ := RewriteTableReferences(model.CompiledSQL, e.Build.Project, model.Name) + model.CompiledSQL = rewritten + } + + result, splitErr := MakeModelSQL(model.CompiledSQL, e.DbConn.GetType()) + if splitErr != nil { + return g.Error(splitErr, "could not parse incremental SQL for '%s'", model.Name) + } + model.CompiledSQL = result.ModelQuery + incrementalSQL := model.CompiledSQL + + strategy := e.getMergeStrategy(model) + tempTable := e.getTempTableName(model) + defer func() { + if dropErr := e.dropTable(tempTable, false); dropErr != nil { + g.Debug("could not drop temp table %s: %s", tempTable, dropErr) + } + }() + + if e.isClickHouse() { + quoted, qErr := e.quoteFullTableName(tempTable) + if qErr != nil { + return qErr + } + _, err = e.DbConn.Exec(g.F("CREATE TABLE %s ENGINE = Memory AS (%s)", quoted, incrementalSQL)) + } else { + err = e.createTableAs(tempTable, incrementalSQL, nil) + } + if err != nil { + return g.Error(err, "could not create temp table for incremental merge on '%s'", model.Name) + } + + tgtQuoted, err := e.quoteFullTableName(t) + if err != nil { + return err + } + tmpQuoted, err := e.quoteFullTableName(tempTable) + if err != nil { + return err + } + mergeSQL, err := e.DbConn.GenerateMergeSQLWithStrategy(tmpQuoted, tgtQuoted, uniqueKeys, &strategy) + if err != nil { + return g.Error(err, "could not generate merge SQL for '%s'", model.Name) + } + + _, err = e.DbConn.ExecMulti(mergeSQL) + if err != nil { + return g.Error(err, "could not execute incremental merge for '%s'", model.Name) + } + + return nil +} + +// advanceStateAfterRange writes the final watermark to SLING_STATE. +// For bounded upper (paged): writes last chunk's UpperRaw. +// For unbounded upper (plain incremental): queries target MAX post-merge. +func (e *Executor) advanceStateAfterRange(model *Model, r *Range, updateKey string) error { + last := r.Chunks[len(r.Chunks)-1] + + if last.Upper != "" && last.Upper != "null" { + // Paged / bounded — advance to upper bound + return sling.WriteState(model.Name, model.FullTableName, last.UpperRaw, last.ColType) + } + + // Unbounded upper — query actual max from target + maxVal, maxType, err := e.queryTargetMax(model, updateKey) + if err != nil { + return g.Error(err, "advanceStateAfterRange: queryTargetMax failed") + } + if maxVal == "" { + g.Warn("build[%s]: target table appears empty after merge; leaving state unchanged", model.Name) + return nil + } + + colType := last.ColType + if colType == "" { + colType = maxType + } + return sling.WriteState(model.Name, model.FullTableName, maxVal, colType) +} + +// handleChunkError formats an error from a failed chunk and optionally prints +// a resume hint when the range came from --range. +func (e *Executor) handleChunkError(model *Model, r *Range, failedIdx int, err error) error { + chunk := r.Chunks[failedIdx] + if r.FromCLI { + e.printResumeHint(model, r, failedIdx, chunk) + } + return g.Error(err, "build[%s]: chunk %d failed (%s)", + model.Name, failedIdx+1, chunk.Describe(model.Config.UpdateKey)) +} + +// printResumeHint prints a sling build --range command the user can re-run to +// resume from the failed chunk. Emits at INFO level (regardless of --debug) so +// operators can always see the recovery command in a wall of logs. +func (e *Executor) printResumeHint(model *Model, r *Range, failedIdx int, failed RangeChunk) { + last := r.Chunks[len(r.Chunks)-1] + raw := formatResumeCommand(failed, last, r.Step) + e.ctx.Info("%s chunk %d/%d failed — resume with:", + env.YellowString("▶"), failedIdx+1, len(r.Chunks)) + e.ctx.Info(" sling build --range '%s' -s %s", raw, model.Name) +} diff --git a/core/sling/build/executor_test.go b/core/sling/build/executor_test.go new file mode 100644 index 000000000..7d6c8fac1 --- /dev/null +++ b/core/sling/build/executor_test.go @@ -0,0 +1,533 @@ +package build + +import ( + "regexp" + "strings" + "testing" + "time" + + "github.com/slingdata-io/sling-cli/core/dbio" + "github.com/slingdata-io/sling-cli/core/dbio/database" + "github.com/slingdata-io/sling-cli/core/dbio/iop" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNewExecutor(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{Target: "POSTGRES"}) + require.NoError(t, err) + require.NoError(t, b.Compile()) + + exec, err := NewExecutor(b) + require.NoError(t, err) + assert.Equal(t, "POSTGRES", exec.ConnName) + assert.NotNil(t, exec.Build) + assert.NotNil(t, exec.failedSet) +} + +func TestNewExecutorNoTarget(t *testing.T) { + dir := t.TempDir() + + b, err := NewBuild(dir, BuildOptions{}) + require.NoError(t, err) + + _, err = NewExecutor(b) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no target connection specified") +} + +func TestGetUniqueKeysString(t *testing.T) { + model := &Model{Config: ModelConfig{UniqueKey: "id"}} + keys := getUniqueKeys(model) + assert.Equal(t, []string{"id"}, keys) +} + +func TestGetUniqueKeysStringSlice(t *testing.T) { + model := &Model{Config: ModelConfig{UniqueKey: []string{"id", "name"}}} + keys := getUniqueKeys(model) + assert.Equal(t, []string{"id", "name"}, keys) +} + +func TestGetUniqueKeysInterfaceSlice(t *testing.T) { + model := &Model{Config: ModelConfig{UniqueKey: []interface{}{"id", "name"}}} + keys := getUniqueKeys(model) + assert.Equal(t, []string{"id", "name"}, keys) +} + +func TestGetUniqueKeysNil(t *testing.T) { + model := &Model{Config: ModelConfig{UniqueKey: nil}} + keys := getUniqueKeys(model) + assert.Nil(t, keys) +} + +func TestGetUniqueKeysEmptyString(t *testing.T) { + model := &Model{Config: ModelConfig{UniqueKey: ""}} + keys := getUniqueKeys(model) + assert.Nil(t, keys) +} + +func TestGetEngineClauseDefault(t *testing.T) { + model := &Model{Config: ModelConfig{}} + clause := getEngineClause(model) + assert.Equal(t, "ENGINE = MergeTree()", clause) +} + +func TestGetEngineClauseCustom(t *testing.T) { + model := &Model{Config: ModelConfig{Engine: "ReplacingMergeTree(updated_at)"}} + clause := getEngineClause(model) + assert.Equal(t, "ENGINE = ReplacingMergeTree(updated_at)", clause) +} + +func TestGetOrderByClauseNoKeys(t *testing.T) { + model := &Model{Config: ModelConfig{}} + clause := getOrderByClause(model, nil) + assert.Equal(t, "ORDER BY tuple()", clause) +} + +func TestGetOrderByClauseSingleKey(t *testing.T) { + model := &Model{Config: ModelConfig{UniqueKey: "id"}} + clause := getOrderByClause(model, nil) + assert.Equal(t, "ORDER BY (id)", clause) +} + +func TestGetOrderByClauseMultipleKeys(t *testing.T) { + model := &Model{Config: ModelConfig{UniqueKey: []string{"id", "name"}}} + clause := getOrderByClause(model, nil) + assert.Equal(t, "ORDER BY (id, name)", clause) +} + +func TestClickHouseCreateTableAllowsNullableKeys(t *testing.T) { + model := &Model{Config: ModelConfig{UniqueKey: []string{"order_key", "line_number"}}} + clause := getOrderByClause(model, nil) + assert.Equal(t, "ORDER BY (order_key, line_number)", clause) + assert.NotContains(t, strings.ToUpper(clause), "TUPLE()") +} + +func TestGetOrderByClauseQuoted(t *testing.T) { + model := &Model{Config: ModelConfig{UniqueKey: "id"}} + clause := getOrderByClause(model, func(s string) string { return `"` + s + `"` }) + assert.Equal(t, `ORDER BY ("id")`, clause) +} + +func TestFormatDurationMilliseconds(t *testing.T) { + d := 250 * time.Millisecond + assert.Equal(t, "250ms", formatDuration(d)) +} + +func TestFormatDurationSeconds(t *testing.T) { + d := 1500 * time.Millisecond + assert.Equal(t, "1.5s", formatDuration(d)) +} + +func TestFormatDurationZero(t *testing.T) { + d := time.Duration(0) + assert.Equal(t, "0ms", formatDuration(d)) +} + +func TestExecutionResultTracking(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{Target: "POSTGRES"}) + require.NoError(t, err) + require.NoError(t, b.Compile()) + + exec, err := NewExecutor(b) + require.NoError(t, err) + + // Add results manually + exec.Results = append(exec.Results, ExecutionResult{ + Name: "stg_orders", + NodeType: "model", + Mode: "full-refresh", + Duration: 200 * time.Millisecond, + }) + exec.Results = append(exec.Results, ExecutionResult{ + Name: "fct_orders", + NodeType: "model", + Mode: "incremental", + Duration: 500 * time.Millisecond, + Err: assert.AnError, + }) + + assert.Len(t, exec.Results, 2) + assert.Nil(t, exec.Results[0].Err) + assert.NotNil(t, exec.Results[1].Err) +} + +func TestGetMergeStrategyDeleteInsert(t *testing.T) { + model := &Model{Config: ModelConfig{MergeStrategy: "delete+insert"}} + strategy := getMergeStrategy(model, false) + assert.Equal(t, database.MergeStrategyDeleteInsert, strategy) +} + +func TestGetMergeStrategyUpdateInsert(t *testing.T) { + model := &Model{Config: ModelConfig{MergeStrategy: "update+insert"}} + strategy := getMergeStrategy(model, false) + assert.Equal(t, database.MergeStrategyUpdateInsert, strategy) +} + +func TestGetMergeStrategyInsert(t *testing.T) { + model := &Model{Config: ModelConfig{MergeStrategy: "insert"}} + strategy := getMergeStrategy(model, false) + assert.Equal(t, database.MergeStrategyInsert, strategy) +} + +func TestGetMergeStrategyDefault(t *testing.T) { + // Empty strategy defaults to delete+insert + model := &Model{Config: ModelConfig{}} + strategy := getMergeStrategy(model, false) + assert.Equal(t, database.MergeStrategyDeleteInsert, strategy) +} + +func TestGetMergeStrategyClickHouseOverride(t *testing.T) { + // ClickHouse forces delete+insert regardless of user setting + model := &Model{Config: ModelConfig{MergeStrategy: "update+insert"}} + strategy := getMergeStrategy(model, true) + assert.Equal(t, database.MergeStrategyDeleteInsert, strategy) + + // delete+insert is kept as-is for ClickHouse + model2 := &Model{Config: ModelConfig{MergeStrategy: "delete+insert"}} + strategy2 := getMergeStrategy(model2, true) + assert.Equal(t, database.MergeStrategyDeleteInsert, strategy2) + + // Empty also defaults to delete+insert for ClickHouse + model3 := &Model{Config: ModelConfig{}} + strategy3 := getMergeStrategy(model3, true) + assert.Equal(t, database.MergeStrategyDeleteInsert, strategy3) +} + +func TestGetTempTableName(t *testing.T) { + model := &Model{Name: "fct_orders", Schema: "marts"} + + // Always schema-qualified with run ID for concurrent-run isolation + tempName := getTempTableName(model, "abc123") + assert.Equal(t, "marts._sling_build_tmp_fct_orders_abc123", tempName) + + tempName2 := getTempTableName(model, "xyz789") + assert.Equal(t, "marts._sling_build_tmp_fct_orders_xyz789", tempName2) + assert.NotEqual(t, tempName, tempName2) +} + +func TestNormalizeMode(t *testing.T) { + m, warn := normalizeMode("snapshot") + assert.Equal(t, "append", m) + assert.NotEmpty(t, warn) + + m, warn = normalizeMode("table") + assert.Equal(t, "full-refresh", m) + assert.Empty(t, warn) + + m, _ = normalizeMode("append") + assert.Equal(t, "append", m) +} + +func TestMapMaterialized(t *testing.T) { + m, err := mapMaterialized("table") + require.NoError(t, err) + assert.Equal(t, "full-refresh", m) + + m, err = mapMaterialized("view") + require.NoError(t, err) + assert.Equal(t, "view", m) + + m, err = mapMaterialized("incremental") + require.NoError(t, err) + assert.Equal(t, "incremental", m) + + _, err = mapMaterialized("ephemeral") + assert.Error(t, err) +} + +func TestCompileDataTestNotNull(t *testing.T) { + sql, label, err := compileDataTest( + map[string]any{"not_null": []any{"id", "name"}}, + `"public"."orders"`, + func(s string) string { return `"` + s + `"` }, + ) + require.NoError(t, err) + assert.Contains(t, label, "not_null") + assert.Contains(t, sql, `"id" IS NULL`) + assert.Contains(t, sql, `"name" IS NULL`) +} + +func TestCompileDataTestUnique(t *testing.T) { + sql, label, err := compileDataTest( + map[string]any{"unique": "id"}, + `"public"."orders"`, + func(s string) string { return `"` + s + `"` }, + ) + require.NoError(t, err) + assert.Contains(t, label, "unique") + assert.Contains(t, sql, "GROUP BY") + assert.Contains(t, sql, "HAVING count(*) > 1") +} + +func TestCompileDataTestExpr(t *testing.T) { + sql, label, err := compileDataTest( + map[string]any{"expr": "sum(amount) >= 0"}, + `"public"."orders"`, + func(s string) string { return `"` + s + `"` }, + ) + require.NoError(t, err) + assert.Contains(t, label, "expr") + assert.Contains(t, sql, "sum(amount) >= 0") +} + +func TestContainsSemicolonDollarQuote(t *testing.T) { + // Semicolon inside dollar-quoted body should not count + sql := `CREATE FUNCTION f() RETURNS void AS $$ BEGIN PERFORM 1; END; $$ LANGUAGE plpgsql` + assert.False(t, containsSemicolon(sql)) + + // Real statement separator outside dollar quotes + sql2 := `SELECT 1; SELECT 2` + assert.True(t, containsSemicolon(sql2)) +} + +func TestIncrementalRequiresUniqueKey(t *testing.T) { + // Model with incremental mode but no unique_key should error + model := &Model{ + Name: "bad_model", + FullTableName: "public.bad_model", + Config: ModelConfig{Mode: "incremental"}, + CompiledSQL: "SELECT 1", + } + + dir := getTestFixturePath("sample_project") + b, err := NewBuild(dir, BuildOptions{Target: "POSTGRES"}) + require.NoError(t, err) + require.NoError(t, b.Compile()) + + exec, err := NewExecutor(b) + require.NoError(t, err) + + err = exec.executeIncremental(model) + assert.Error(t, err) + assert.Contains(t, err.Error(), "no unique_key defined") +} + +func TestIsDownstreamOfFailed(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{Target: "POSTGRES"}) + require.NoError(t, err) + require.NoError(t, b.Compile()) + + exec, err := NewExecutor(b) + require.NoError(t, err) + + // No failures yet — fct_orders depends on stg_orders + assert.False(t, exec.isDownstreamOfFailed("fct_orders")) + + // Mark stg_orders as failed + exec.failedSet["stg_orders"] = true + + // Now fct_orders should be downstream of a failure + assert.True(t, exec.isDownstreamOfFailed("fct_orders")) + + // stg_customers is independent, should not be affected + assert.False(t, exec.isDownstreamOfFailed("stg_customers")) +} + +// ============================================================================= +// Range tests +// ============================================================================= + +func quote(s string) string { return `"` + s + `"` } + +func TestRangeChunk_WhereCond_Unbounded(t *testing.T) { + c := RangeChunk{Lower: "", Upper: ""} + assert.Equal(t, "1=1", c.WhereCond("col", quote)) +} + +func TestRangeChunk_WhereCond_LowerOnly_Exclusive(t *testing.T) { + c := RangeChunk{Lower: "'2024-01-01'", Upper: "", LowerInclusive: false} + assert.Equal(t, `"col" > '2024-01-01'`, c.WhereCond("col", quote)) +} + +func TestRangeChunk_WhereCond_LowerOnly_Inclusive(t *testing.T) { + c := RangeChunk{Lower: "'2024-01-01'", Upper: "", LowerInclusive: true} + assert.Equal(t, `"col" >= '2024-01-01'`, c.WhereCond("col", quote)) +} + +func TestRangeChunk_WhereCond_UpperOnly(t *testing.T) { + c := RangeChunk{Lower: "", Upper: "'2024-12-31'"} + assert.Equal(t, `"col" < '2024-12-31'`, c.WhereCond("col", quote)) +} + +func TestRangeChunk_WhereCond_Bounded_Exclusive(t *testing.T) { + c := RangeChunk{Lower: "'2024-01-01'", Upper: "'2024-02-01'", LowerInclusive: false} + assert.Equal(t, `"col" > '2024-01-01' AND "col" < '2024-02-01'`, c.WhereCond("col", quote)) +} + +func TestRangeChunk_WhereCond_Bounded_Inclusive(t *testing.T) { + c := RangeChunk{Lower: "'2024-01-01'", Upper: "'2024-02-01'", LowerInclusive: true} + assert.Equal(t, `"col" >= '2024-01-01' AND "col" < '2024-02-01'`, c.WhereCond("col", quote)) +} + +func TestRangeChunk_WhereCond_NullLower_TreatedUnbounded(t *testing.T) { + c := RangeChunk{Lower: "null", Upper: "'2024-02-01'"} + assert.Equal(t, `"col" < '2024-02-01'`, c.WhereCond("col", quote)) +} + +func TestParseCLIRange_StartEnd(t *testing.T) { + start, end, step, hasStep, err := parseCLIRange("2024-01-01,2024-12-31") + require.NoError(t, err) + assert.Equal(t, "2024-01-01", start) + assert.Equal(t, "2024-12-31", end) + assert.Equal(t, "", step) + assert.False(t, hasStep) +} + +func TestParseCLIRange_StartEndStep(t *testing.T) { + start, end, step, hasStep, err := parseCLIRange("2024-01-01,2024-04-01,1mo") + require.NoError(t, err) + assert.Equal(t, "2024-01-01", start) + assert.Equal(t, "2024-04-01", end) + assert.Equal(t, "1mo", step) + assert.True(t, hasStep) +} + +func TestParseCLIRange_TooFewParts(t *testing.T) { + _, _, _, _, err := parseCLIRange("2024-01-01") + assert.Error(t, err) +} + +func TestParseCLIRange_TooManyParts(t *testing.T) { + _, _, _, _, err := parseCLIRange("a,b,c,d") + assert.Error(t, err) +} + +func TestParseCLIRange_EmptyStart(t *testing.T) { + _, _, _, _, err := parseCLIRange(",2024-12-31") + assert.Error(t, err) +} + +func TestParseCLIRange_BadStep(t *testing.T) { + _, _, _, _, err := parseCLIRange("2024-01-01,2024-12-31,notaduration") + assert.Error(t, err) +} + +func TestSplitCLIRange_NoStep_SingleChunk(t *testing.T) { + r, err := splitCLIRange("2024-01-01,2024-12-31", dbio.TypeDbDuckDb, "") + require.NoError(t, err) + require.Len(t, r.Chunks, 1) + assert.True(t, r.FromCLI) + assert.False(t, r.UpdateState) + assert.Equal(t, "2024-01-01", r.Chunks[0].LowerRaw) + assert.Equal(t, "2024-12-31", r.Chunks[0].UpperRaw) + assert.True(t, r.Chunks[0].LowerInclusive) +} + +func TestSplitCLIRange_WithStep_MultipleChunks(t *testing.T) { + // 1mo = 30d; Jan1→Apr1 = 91 days → 4 chunks (ceiling) + r, err := splitCLIRange("2024-01-01,2024-04-01,1mo", dbio.TypeDbDuckDb, iop.TimestampType) + require.NoError(t, err) + assert.Greater(t, len(r.Chunks), 1) + for _, c := range r.Chunks { + assert.True(t, c.LowerInclusive) + } +} + +func TestSplitCLIRange_WithStep_FinalChunkClamped(t *testing.T) { + // 2 full months + remainder + r, err := splitCLIRange("2024-01-01,2024-03-15,1mo", dbio.TypeDbDuckDb, iop.TimestampType) + require.NoError(t, err) + assert.Equal(t, 3, len(r.Chunks)) + // Last chunk upper should be clamped to end (2024-03-15) + lastChunk := r.Chunks[len(r.Chunks)-1] + assert.Contains(t, lastChunk.UpperRaw, "2024-03-15") +} + +func TestSplitCLIRange_WithStep_BadStartTime(t *testing.T) { + _, err := splitCLIRange("not-a-date,2024-04-01,1mo", dbio.TypeDbDuckDb, iop.TimestampType) + assert.Error(t, err) +} + +func TestQuoteValue_Nil(t *testing.T) { + result := quoteValue(nil, "", dbio.TypeDbDuckDb) + assert.Equal(t, "null", result) +} + +func TestQuoteValue_StringFallback(t *testing.T) { + result := quoteValue("hello", iop.StringType, dbio.TypeDbDuckDb) + assert.Contains(t, result, "hello") +} + +func TestQuoteValue_TimestampForDuckDB(t *testing.T) { + // iop.FormatValue wraps timestamp values in quotes for DuckDB + result := quoteValue("2024-01-15", iop.StringType, dbio.TypeDbDuckDb) + assert.NotEmpty(t, result) +} + +func TestParseValueAsTime_ISO8601(t *testing.T) { + t1, err := parseValueAsTime("2024-01-15T00:00:00Z", iop.TimestampType) + require.NoError(t, err) + assert.Equal(t, 2024, t1.Year()) + assert.Equal(t, 1, int(t1.Month())) + assert.Equal(t, 15, t1.Day()) +} + +func TestParseValueAsTime_ISO8601_DateOnly(t *testing.T) { + t1, err := parseValueAsTime("2024-03-01", iop.DateType) + require.NoError(t, err) + assert.Equal(t, 2024, t1.Year()) + assert.Equal(t, 3, int(t1.Month())) +} + +func TestParseValueAsTime_Empty(t *testing.T) { + _, err := parseValueAsTime("", iop.TimestampType) + assert.Error(t, err) +} + +func TestParseValueAsTime_NotDatetime(t *testing.T) { + _, err := parseValueAsTime("123", iop.IntegerType) + assert.Error(t, err) +} + +// stripANSI removes ANSI color codes from a string for substring matching. +var ansiEscapeRe = regexp.MustCompile(`\x1b\[[0-9;]*[mGKHJA-Z]`) + +func stripANSI(s string) string { return ansiEscapeRe.ReplaceAllString(s, "") } + +func TestFormatChunkProgressLine_OK(t *testing.T) { + c := RangeChunk{ + Lower: "'2024-01-01'", + Upper: "'2024-02-01'", + LowerRaw: "2024-01-01", + UpperRaw: "2024-02-01", + LowerInclusive: true, + } + line := stripANSI(formatChunkProgressLine(3, 12, c, "created_at", 1400*time.Millisecond, false)) + assert.Contains(t, line, "chunk 3/12") + assert.Contains(t, line, "created_at=[2024-01-01, 2024-02-01)") + assert.Contains(t, line, "OK") + assert.Contains(t, line, "1.4s") +} + +func TestFormatChunkProgressLine_Failed(t *testing.T) { + c := RangeChunk{ + Lower: "'2024-02-01'", + Upper: "'2024-03-01'", + LowerRaw: "2024-02-01", + UpperRaw: "2024-03-01", + LowerInclusive: true, + } + line := stripANSI(formatChunkProgressLine(2, 3, c, "ts", 250*time.Millisecond, true)) + assert.Contains(t, line, "chunk 2/3") + assert.Contains(t, line, "FAIL") + assert.Contains(t, line, "250ms") +} + +func TestFormatResumeCommand_WithStep(t *testing.T) { + failed := RangeChunk{LowerRaw: "2024-02-01"} + last := RangeChunk{UpperRaw: "2024-04-01"} + assert.Equal(t, "2024-02-01,2024-04-01,1mo", formatResumeCommand(failed, last, "1mo")) +} + +func TestFormatResumeCommand_WithoutStep(t *testing.T) { + failed := RangeChunk{LowerRaw: "2024-02-01"} + last := RangeChunk{UpperRaw: "2024-04-01"} + assert.Equal(t, "2024-02-01,2024-04-01", formatResumeCommand(failed, last, "")) +} diff --git a/core/sling/build/hook_runner.go b/core/sling/build/hook_runner.go new file mode 100644 index 000000000..606187b61 --- /dev/null +++ b/core/sling/build/hook_runner.go @@ -0,0 +1,140 @@ +package build + +import ( + "strings" + + "github.com/flarco/g" + "github.com/slingdata-io/sling-cli/core/sling" +) + +func init() { + // Register so type: build steps work without an import cycle + // (build imports sling; sling never imports build). + sling.HookRunBuild = RunForHook +} + +// RunForHook compiles and executes a build project for a pipeline/replication hook. +// Returns a state map with per-node results for state..results. +func RunForHook(path string, opts sling.HookBuildRunOptions) (map[string]any, error) { + buildOpts := BuildOptions{ + Target: opts.Target, + Select: opts.Select, + Exclude: opts.Exclude, + Vars: opts.Vars, + FailFast: opts.FailFast, + FullRefresh: opts.FullRefresh, + Threads: opts.Threads, + Schema: opts.Schema, + Prod: opts.Prod, + NoSeeds: opts.NoSeeds, + Recursive: opts.Recursive, + Test: opts.Test, + } + if buildOpts.Threads < 1 { + buildOpts.Threads = DefaultThreads + } + if opts.Range != "" { + buildOpts.Range = g.String(opts.Range) + } + + b, err := NewBuild(path, buildOpts) + if err != nil { + return nil, g.Error(err, "could not load build project from %s", path) + } + + if err := b.Compile(); err != nil { + return nil, g.Error(err, "could not compile build project") + } + + // Multi-target / recursive sub-projects use Build.Execute (no per-node results). + if len(b.Project.SubProjects) > 0 { + if err := b.Execute(); err != nil { + return g.M( + "path", path, + "target", b.GetTarget(), + "sub_projects", len(b.Project.SubProjects), + ), g.Error(err, "build failed") + } + return g.M( + "path", path, + "target", b.GetTarget(), + "sub_projects", len(b.Project.SubProjects), + "results", []map[string]any{}, + "total", 0, + "ok", 0, + "failed", 0, + "skipped", 0, + ), nil + } + + executor, err := NewExecutor(b) + if err != nil { + return nil, err + } + + runErr := executor.Execute() + data := resultsToState(path, b.GetTarget(), executor.Results) + if runErr != nil { + return data, g.Error(runErr, "build failed") + } + return data, nil +} + +func resultsToState(path, target string, results []ExecutionResult) map[string]any { + rows := make([]map[string]any, 0, len(results)) + ok, failed, skipped := 0, 0, 0 + for _, r := range results { + status := "success" + errMsg := "" + switch { + case r.Skipped: + status = "skipped" + skipped++ + case r.Err != nil: + status = "error" + errMsg = r.Err.Error() + failed++ + default: + ok++ + } + rows = append(rows, g.M( + "name", r.Name, + "type", r.NodeType, + "mode", r.Mode, + "duration", r.Duration.Seconds(), + "status", status, + "error", errMsg, + )) + } + + // Stable order for downstream checks (execution order can be parallel) + // — keep as-is; callers can count by status. + + return g.M( + "path", path, + "target", target, + "results", rows, + "total", len(rows), + "ok", ok, + "failed", failed, + "skipped", skipped, + // Convenience: comma-joined names of successful models/seeds + "ok_names", joinResultNames(results, "success"), + ) +} + +func joinResultNames(results []ExecutionResult, wantStatus string) string { + var names []string + for _, r := range results { + status := "success" + if r.Skipped { + status = "skipped" + } else if r.Err != nil { + status = "error" + } + if status == wantStatus { + names = append(names, r.Name) + } + } + return strings.Join(names, ",") +} diff --git a/core/sling/build/project.go b/core/sling/build/project.go new file mode 100644 index 000000000..df72a341c --- /dev/null +++ b/core/sling/build/project.go @@ -0,0 +1,1329 @@ +package build + +import ( + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + "github.com/flarco/g" + "github.com/samber/lo" + "github.com/slingdata-io/sling-cli/core/sling" + "github.com/spf13/cast" + "gopkg.in/yaml.v3" +) + +// BuildProject represents a sling build project discovered from a directory. +type BuildProject struct { + Dir string + Config *BuildConfig // from sling_build.yml (nil if missing) + Models map[string]*Model // keyed by unique model name + Seeds map[string]*Seed // keyed by unique seed name + Macros []*MacroFile // collected .macros.sql files + Mode string // "dev" or "prod" + SchemaOverride string // dev mode schema + DefaultSchema string // default schema for root-level files (default: "public") + ChildConfigs map[string]*BuildConfig // child sling_build.yml configs keyed by relative dir + SubProjects []*BuildProject // independent build projects (when no root yml) + Recursive bool // CLI --recursive: keep immediate child projects +} + +// Model represents a SQL model file in the project. +type Model struct { + Name string // e.g., "dim_customers" + FilePath string // absolute path + RelPath string // relative path from project root + Schema string // derived from folder or override + Prefix string // underscore-joined nested folder names + FullTableName string // schema.prefix_name (current mode) + ProdFullTableName string // schema.prefix_name (always prod-mode, for SQL matching) + RawSQL string // raw file content (frontmatter stripped) + CompiledSQL string // after Jinja rendering + PreStatements []string // SQL statements before the model query (from multi-statement splitting) + PostStatements []string // SQL statements after the model query (from multi-statement splitting) + Config ModelConfig // from YAML frontmatter or config() block + HasFrontmatter bool // true if config was set via YAML frontmatter (config() becomes no-op) + Style Style // detected incremental pattern: StyleDbt or StyleSling (populated at load) + Refs []string // ref() dependencies + Sources []string // src() references + DependsOn []string // all DAG dependencies (refs + bare refs + auto-detected) + startHooks sling.Hooks // parsed start hooks (populated at execution time) + endHooks sling.Hooks // parsed end hooks (populated at execution time) +} + +// ModelConfig holds configuration extracted from the config() block in a SQL model. +type ModelConfig struct { + Mode string `yaml:"mode,omitempty"` + Materialized string `yaml:"materialized,omitempty"` // dbt alias for mode + UniqueKey any `yaml:"unique_key,omitempty"` // string or []string + MergeStrategy string `yaml:"merge_strategy,omitempty"` + UpdateKey string `yaml:"update_key,omitempty"` + Tags []string `yaml:"tags,omitempty"` + Hooks sling.HookMap `yaml:"hooks,omitempty"` + PreHook string `yaml:"pre_hook,omitempty"` // deprecated: kept for validation only + PostHook string `yaml:"post_hook,omitempty"` // deprecated: kept for validation only + Schema string `yaml:"schema,omitempty"` + Enabled *bool `yaml:"enabled,omitempty"` + Engine string `yaml:"engine,omitempty"` + Range *RangeConfig `yaml:"range,omitempty"` + DropCascade *bool `yaml:"drop_cascade,omitempty"` // default false; CASCADE on DROP when true + Rewrite *bool `yaml:"rewrite,omitempty"` // default true; set false to skip bare-name rewrite + Tests []any `yaml:"tests,omitempty"` // declarative data tests +} + +// RangeConfig holds the range block from model front-matter. It drives the +// unified incremental / lookback / paged-backfill behavior (sling style only). +type RangeConfig struct { + Start string `yaml:"start,omitempty"` // literal value; parsed lazily at execution time + Advance string `yaml:"advance,omitempty"` // duration (5m, 5h, 5d, 5w, 1mo, 1y) — per-run forward advance + Lookback string `yaml:"lookback,omitempty"` // duration +} + +// HasAdvance returns true if paged-advance mode is enabled. +func (r *RangeConfig) HasAdvance() bool { return r != nil && r.Advance != "" } + +// HasLookback returns true if a lookback window is configured. +func (r *RangeConfig) HasLookback() bool { return r != nil && r.Lookback != "" } + +// durationRegex matches a positive integer followed by a unit: ms, s, m, h, d, w, mo, y. +// Longer units must come first in the alternation so "mo" wins over "m". +var durationRegex = regexp.MustCompile(`^(\d+)(ms|mo|s|m|h|d|w|y)$`) + +// parseBuildDuration parses a duration literal as used in range.advance / range.lookback. +// Supported units: ms, s, m, h, d (24h), w (7d), mo (30d), y (365d). +// The return is approximate for calendar units — phase 4 will re-parse if calendar +// accuracy is needed (e.g., month-by-month stepping via time.AddDate). +func parseBuildDuration(s string) (time.Duration, error) { + m := durationRegex.FindStringSubmatch(strings.TrimSpace(s)) + if m == nil { + return 0, g.Error("invalid duration '%s': expected with unit in ms,s,m,h,d,w,mo,y", s) + } + n, err := strconv.Atoi(m[1]) + if err != nil { + return 0, g.Error(err, "invalid duration '%s'", s) + } + unit := m[2] + switch unit { + case "ms": + return time.Duration(n) * time.Millisecond, nil + case "s": + return time.Duration(n) * time.Second, nil + case "m": + return time.Duration(n) * time.Minute, nil + case "h": + return time.Duration(n) * time.Hour, nil + case "d": + return time.Duration(n) * 24 * time.Hour, nil + case "w": + return time.Duration(n) * 7 * 24 * time.Hour, nil + case "mo": + return time.Duration(n) * 30 * 24 * time.Hour, nil + case "y": + return time.Duration(n) * 365 * 24 * time.Hour, nil + } + return 0, g.Error("invalid duration unit '%s'", unit) +} + +// validateModel runs load-time validation on a Model's configuration. +// It is called from addModel() after frontmatter parse and style detection. +func validateModel(m *Model) error { + if err := applyModeAliases(&m.Config, m.Name); err != nil { + return err + } + + // mode: incremental requires update_key + if m.Config.Mode == "incremental" && m.Config.UpdateKey == "" { + return g.Error("model '%s': mode 'incremental' requires update_key", m.Name) + } + + r := m.Config.Range + if r == nil { + return nil + } + + if r.Start != "" && r.Advance == "" { + return g.Error("model '%s': range.start requires range.advance", m.Name) + } + if (r.Advance != "" || r.Lookback != "") && m.Config.Mode != "incremental" { + return g.Error("model '%s': range.* requires mode: incremental", m.Name) + } + if r.Advance != "" && m.Config.UpdateKey == "" { + return g.Error("model '%s': range.advance requires update_key", m.Name) + } + if r.Advance != "" { + if _, err := parseBuildDuration(r.Advance); err != nil { + return g.Error(err, "model '%s': invalid range.advance", m.Name) + } + } + if r.Lookback != "" { + if _, err := parseBuildDuration(r.Lookback); err != nil { + return g.Error(err, "model '%s': invalid range.lookback", m.Name) + } + } + + // range.* features require owning the WHERE clause (sling style) + if (r.Advance != "" || r.Lookback != "") && m.Style == StyleDbt { + return g.Error("model '%s': range.* requires {incremental_where_cond} (sling style); is_incremental() is not compatible with range.*", m.Name) + } + + return nil +} + +// Seed represents a seed file (CSV, JSON, Parquet) in the project. +type Seed struct { + Name string // e.g., "country_codes" + FilePath string // absolute path + RelPath string // relative path from project root + Schema string + Prefix string + FullTableName string // schema.prefix_name (current mode) + ProdFullTableName string // schema.prefix_name (always prod-mode, for SQL matching) + Format string // csv, json, parquet +} + +// BuildConfig represents the contents of sling_build.yml. +type BuildConfig struct { + Target string `yaml:"target"` + Dev *DevConfig `yaml:"dev,omitempty"` + DbtProject any `yaml:"dbt_project,omitempty"` + Vars map[string]any `yaml:"vars,omitempty"` + Defaults BuildDefaults `yaml:"defaults,omitempty"` +} + +// DevConfig holds dev-mode settings in sling_build.yml. +// When present, dev mode is the default (override with --prod). +type DevConfig struct { + Target string `yaml:"target,omitempty"` // optional, falls back to top-level target + Schema string `yaml:"schema"` // mandatory for dev mode +} + +// DbtProjectConfig holds dbt project compatibility settings. +type DbtProjectConfig struct { + ModelsPath string `yaml:"models_path,omitempty"` // default: "models" + SeedsPath string `yaml:"seeds_path,omitempty"` // default: "seeds" +} + +// BuildDefaults holds default settings for models. +type BuildDefaults struct { + Mode string `yaml:"mode,omitempty"` + Schema string `yaml:"schema,omitempty"` + Tags []string `yaml:"tags,omitempty"` // additive across nesting + UniqueKey any `yaml:"unique_key,omitempty"` + UpdateKey string `yaml:"update_key,omitempty"` + MergeStrategy string `yaml:"merge_strategy,omitempty"` + Enabled *bool `yaml:"enabled,omitempty"` + Hooks sling.HookMap `yaml:"hooks,omitempty"` // additive across nesting + DropCascade *bool `yaml:"drop_cascade,omitempty"` +} + +// BuildOptions holds CLI-provided overrides. +type BuildOptions struct { + Target string + Schema string + Prod bool + Vars map[string]any + FullRefresh bool + Select []string + Exclude []string + Compile bool + Threads int + FailFast bool + List bool + NoSeeds bool + Range *string // CLI --range: "start,end[,step]" + Recursive bool // CLI --recursive/-R: discover sling_build.yml in immediate subdirectories + Test bool // CLI --test: run data tests only (no materialization) + JSON bool // CLI --json: machine-readable compile/list output +} + +// DefaultThreads is the default parallelism for model execution. +const DefaultThreads = 4 + +// ValidModes are the recognized materialization modes. +var ValidModes = map[string]bool{ + "full-refresh": true, + "view": true, + "truncate": true, + "incremental": true, + "append": true, + "snapshot": true, // deprecated alias for append +} + +// normalizeMode maps aliases and deprecated names to canonical modes. +// Returns the canonical mode and an optional warning message. +func normalizeMode(mode string) (string, string) { + switch strings.ToLower(strings.TrimSpace(mode)) { + case "": + return "", "" + case "snapshot": + return "append", "'snapshot' mode is deprecated and means append-only insert; use 'append'. 'snapshot' will mean SCD2 in a future release" + case "table": + return "full-refresh", "" + case "ephemeral": + return "ephemeral", "ephemeral models are not supported in sling build; materialize as a view or table instead" + default: + return strings.ToLower(strings.TrimSpace(mode)), "" + } +} + +// mapMaterialized converts a dbt materialized= value to a sling mode. +func mapMaterialized(materialized string) (string, error) { + switch strings.ToLower(strings.TrimSpace(materialized)) { + case "table": + return "full-refresh", nil + case "view": + return "view", nil + case "incremental": + return "incremental", nil + case "ephemeral": + return "", g.Error("materialized='ephemeral' is not supported in sling build; use view or table") + case "materialized_view", "materializedview": + return "", g.Error("materialized='%s' is not supported in sling build; use view or table", materialized) + case "": + return "", nil + default: + return "", g.Error("unknown materialized value '%s'; expected table, view, or incremental", materialized) + } +} + +// applyModeAliases resolves mode/materialized aliases on a ModelConfig. +// Prefer explicit mode over materialized. Warns on deprecated names. +func applyModeAliases(cfg *ModelConfig, modelName string) error { + if cfg.Materialized != "" { + mapped, err := mapMaterialized(cfg.Materialized) + if err != nil { + return g.Error(err, "model '%s'", modelName) + } + // Explicit mode wins over materialized + if cfg.Mode == "" { + cfg.Mode = mapped + } + cfg.Materialized = "" // consumed + } + if cfg.Mode != "" { + canonical, warn := normalizeMode(cfg.Mode) + if warn != "" { + g.Warn("model '%s': %s", modelName, warn) + } + if canonical == "ephemeral" { + return g.Error("model '%s': ephemeral models are not supported; use view or table", modelName) + } + cfg.Mode = canonical + } + return nil +} + +// ConfigFileName is the standard config file name. +const ConfigFileName = "sling_build.yml" + +// seedExtensions are recognized seed file extensions. +var seedExtensions = map[string]string{ + ".csv": "csv", + ".json": "json", + ".parquet": "parquet", +} + +// LoadProject loads a build project from the given directory. +func LoadProject(dir string, opts ...BuildOptions) (*BuildProject, error) { + absDir, err := filepath.Abs(dir) + if err != nil { + return nil, g.Error(err, "could not resolve project directory") + } + + info, err := os.Stat(absDir) + if err != nil { + return nil, g.Error(err, "could not access project directory: %s", absDir) + } + if !info.IsDir() { + return nil, g.Error("path is not a directory: %s", absDir) + } + + var cliOpts BuildOptions + if len(opts) > 0 { + cliOpts = opts[0] + } + + project := &BuildProject{ + Dir: absDir, + Models: make(map[string]*Model), + Seeds: make(map[string]*Seed), + DefaultSchema: "public", + ChildConfigs: make(map[string]*BuildConfig), + Recursive: cliOpts.Recursive, + } + + // Load root config if present + rootConfigPath := filepath.Join(absDir, ConfigFileName) + if _, err := os.Stat(rootConfigPath); err == nil { + cfg, err := loadConfig(rootConfigPath) + if err != nil { + return nil, g.Error(err, "could not load %s", rootConfigPath) + } + project.Config = cfg + } + + // Discover nested sling_build.yml files (only when --recursive is set) + if cliOpts.Recursive { + if err := discoverNestedConfigs(project); err != nil { + return nil, g.Error(err, "could not discover nested configs") + } + } + + // If no root config but children have configs, these are independent builds + if project.Config == nil && len(project.ChildConfigs) > 0 { + return loadIndependentBuilds(project, cliOpts) + } + + // Apply CLI overrides + applyCliOverrides(project, cliOpts) + + // Discover files + if err := discoverFiles(project); err != nil { + return nil, g.Error(err, "could not discover files") + } + + // Warn about macro name shadows + warnMacroShadows(project) + + // Validate unique names + if err := validateUniqueNames(project); err != nil { + return nil, err + } + + return project, nil +} + +// loadConfig reads and parses a sling_build.yml file. +func loadConfig(path string) (*BuildConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, g.Error(err, "could not read config file") + } + + cfg := &BuildConfig{} + if err := yaml.Unmarshal(data, cfg); err != nil { + return nil, g.Error(err, "could not parse config file") + } + + return cfg, nil +} + +// mergeHookMaps produces a HookMap whose slices are the ordered concatenation +// of parent then child slices for each stage. All six stages are merged so +// this helper stays useful if more stages get wired into build execution. +func mergeHookMaps(parent, child sling.HookMap) sling.HookMap { + appendAny := func(a, b []any) []any { + if len(a) == 0 { + return b + } + if len(b) == 0 { + return a + } + out := make([]any, 0, len(a)+len(b)) + out = append(out, a...) + out = append(out, b...) + return out + } + return sling.HookMap{ + Start: appendAny(parent.Start, child.Start), + End: appendAny(parent.End, child.End), + Pre: appendAny(parent.Pre, child.Pre), + Post: appendAny(parent.Post, child.Post), + PreMerge: appendAny(parent.PreMerge, child.PreMerge), + PostMerge: appendAny(parent.PostMerge, child.PostMerge), + } +} + +// mergeConfigs merges a child config into a parent, returning a new merged config. +// vars are deep-merged, defaults are shallow-merged, all other fields replaced by child. +func mergeConfigs(parent, child *BuildConfig) *BuildConfig { + if parent == nil { + return child + } + if child == nil { + return parent + } + + merged := &BuildConfig{ + Target: parent.Target, + Dev: parent.Dev, + DbtProject: parent.DbtProject, + Defaults: parent.Defaults, + } + + // Deep merge vars + merged.Vars = make(map[string]any) + for k, v := range parent.Vars { + merged.Vars[k] = v + } + for k, v := range child.Vars { + merged.Vars[k] = v + } + + // Child overrides non-empty fields + if child.Target != "" { + merged.Target = child.Target + } + if child.Dev != nil { + merged.Dev = child.Dev + } + if child.DbtProject != nil { + merged.DbtProject = child.DbtProject + } + + // Field-by-field defaults merge. + // Scalars: child replaces if set. Tags: union+dedupe. Hooks: append (parent first). + if child.Defaults.Mode != "" { + merged.Defaults.Mode = child.Defaults.Mode + } + if child.Defaults.Schema != "" { + merged.Defaults.Schema = child.Defaults.Schema + } + if child.Defaults.UniqueKey != nil { + merged.Defaults.UniqueKey = child.Defaults.UniqueKey + } + if child.Defaults.UpdateKey != "" { + merged.Defaults.UpdateKey = child.Defaults.UpdateKey + } + if child.Defaults.MergeStrategy != "" { + merged.Defaults.MergeStrategy = child.Defaults.MergeStrategy + } + if child.Defaults.Enabled != nil { + merged.Defaults.Enabled = child.Defaults.Enabled + } + if child.Defaults.DropCascade != nil { + merged.Defaults.DropCascade = child.Defaults.DropCascade + } + if len(parent.Defaults.Tags)+len(child.Defaults.Tags) > 0 { + merged.Defaults.Tags = lo.Uniq(append(append([]string(nil), parent.Defaults.Tags...), child.Defaults.Tags...)) + } + merged.Defaults.Hooks = mergeHookMaps(parent.Defaults.Hooks, child.Defaults.Hooks) + + return merged +} + +// discoverNestedConfigs finds child sling_build.yml files in subdirectories. +func discoverNestedConfigs(project *BuildProject) error { + entries, err := os.ReadDir(project.Dir) + if err != nil { + return g.Error(err, "could not read directory") + } + + for _, entry := range entries { + if !entry.IsDir() || strings.HasPrefix(entry.Name(), ".") { + continue + } + + childDir := filepath.Join(project.Dir, entry.Name()) + childConfigPath := filepath.Join(childDir, ConfigFileName) + + if _, err := os.Stat(childConfigPath); err == nil { + cfg, err := loadConfig(childConfigPath) + if err != nil { + return g.Error(err, "could not load %s", childConfigPath) + } + project.ChildConfigs[entry.Name()] = cfg + } + } + + return nil +} + +// loadIndependentBuilds creates sub-projects when no root yml exists but children do. +func loadIndependentBuilds(project *BuildProject, cliOpts BuildOptions) (*BuildProject, error) { + for childDir, childCfg := range project.ChildConfigs { + subDir := filepath.Join(project.Dir, childDir) + subProject := &BuildProject{ + Dir: subDir, + Config: childCfg, + Models: make(map[string]*Model), + Seeds: make(map[string]*Seed), + DefaultSchema: "public", + ChildConfigs: make(map[string]*BuildConfig), + } + + applyCliOverrides(subProject, cliOpts) + + if err := discoverFiles(subProject); err != nil { + return nil, g.Error(err, "could not discover files in %s", subDir) + } + + warnMacroShadows(subProject) + + if err := validateUniqueNames(subProject); err != nil { + return nil, err + } + + project.SubProjects = append(project.SubProjects, subProject) + } + + return project, nil +} + +// applyCliOverrides applies CLI flags over project config. +func applyCliOverrides(project *BuildProject, opts BuildOptions) { + // Determine mode from config: dev block present → dev, otherwise prod + project.Mode = "prod" + if project.Config != nil && project.Config.Dev != nil { + project.Mode = "dev" + project.SchemaOverride = project.Config.Dev.Schema + } + + // CLI overrides + if opts.Prod { + project.Mode = "prod" + project.SchemaOverride = "" + } + if opts.Schema != "" { + project.Mode = "dev" + project.SchemaOverride = opts.Schema + } + + // Resolve target: in dev mode, use dev.target if set + if project.Config != nil && project.Mode == "dev" && project.Config.Dev != nil && project.Config.Dev.Target != "" { + project.Config.Target = project.Config.Dev.Target + } + + // --target overrides everything + if opts.Target != "" && project.Config != nil { + project.Config.Target = opts.Target + } else if opts.Target != "" && project.Config == nil { + project.Config = &BuildConfig{Target: opts.Target} + } + + // Merge vars + if project.Config != nil && len(opts.Vars) > 0 { + if project.Config.Vars == nil { + project.Config.Vars = make(map[string]any) + } + for k, v := range opts.Vars { + project.Config.Vars[k] = v + } + } +} + +// getDbtProjectConfig parses the dbt_project field which can be bool or DbtProjectConfig. +func getDbtProjectConfig(cfg *BuildConfig) *DbtProjectConfig { + if cfg == nil || cfg.DbtProject == nil { + return nil + } + + switch v := cfg.DbtProject.(type) { + case bool: + if v { + return &DbtProjectConfig{ + ModelsPath: "models", + SeedsPath: "seeds", + } + } + return nil + case map[string]any: + dbtCfg := &DbtProjectConfig{ + ModelsPath: "models", + SeedsPath: "seeds", + } + if mp, ok := v["models_path"]; ok { + dbtCfg.ModelsPath = cast.ToString(mp) + } + if sp, ok := v["seeds_path"]; ok { + dbtCfg.SeedsPath = cast.ToString(sp) + } + return dbtCfg + } + + return nil +} + +// discoverFiles walks the project directory and populates Models and Seeds. +func discoverFiles(project *BuildProject) error { + dbtCfg := getDbtProjectConfig(project.Config) + + if dbtCfg != nil { + // dbt_project mode: scan models_path for .sql, seeds_path for seed files + modelsDir := filepath.Join(project.Dir, dbtCfg.ModelsPath) + seedsDir := filepath.Join(project.Dir, dbtCfg.SeedsPath) + + if err := walkForModels(project, modelsDir, modelsDir); err != nil { + return err + } + if err := walkForSeeds(project, seedsDir, seedsDir); err != nil { + return err + } + } else { + // Flat mode: walk project root, classify by extension + if err := walkFlat(project); err != nil { + return err + } + } + + return nil +} + +// skipNestedBuildDir reports whether dir is a nested project that parent +// discovery must ignore. Immediate children stay in the walk when -R is set +// so ChildConfigs still apply. Deeper nested projects and any nested project +// without -R are skipped (e.31 leftover probe/ dirs). +func skipNestedBuildDir(project *BuildProject, dir string) bool { + if project == nil || dir == "" || dir == project.Dir { + return false + } + if _, err := os.Stat(filepath.Join(dir, ConfigFileName)); err != nil { + return false + } + if !project.Recursive { + return true + } + rel, err := filepath.Rel(project.Dir, dir) + if err != nil || rel == "." { + return false + } + if !strings.Contains(rel, string(os.PathSeparator)) { + return false + } + return true +} + +// walkFlat discovers models and seeds in flat directory structure. +func walkFlat(project *BuildProject) error { + return filepath.Walk(project.Dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Skip hidden files/dirs + if strings.HasPrefix(info.Name(), ".") { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + + // Skip directories (we'll process their contents) + if info.IsDir() { + if skipNestedBuildDir(project, path) { + return filepath.SkipDir + } + return nil + } + + // Skip config files + if info.Name() == ConfigFileName { + return nil + } + + relPath, err := filepath.Rel(project.Dir, path) + if err != nil { + return g.Error(err, "could not get relative path") + } + + ext := strings.ToLower(filepath.Ext(info.Name())) + + // Collect .macros.sql files + if strings.HasSuffix(strings.ToLower(info.Name()), ".macros.sql") { + relDir := filepath.ToSlash(filepath.Dir(relPath)) + return collectMacro(project, path, relDir) + } + + if ext == ".sql" { + return addModel(project, path, relPath) + } + + if format, ok := seedExtensions[ext]; ok { + return addSeed(project, path, relPath, format) + } + + return nil + }) +} + +// walkForModels discovers .sql model files under the given root. +func walkForModels(project *BuildProject, walkRoot, baseDir string) error { + if _, err := os.Stat(walkRoot); os.IsNotExist(err) { + return nil + } + + return filepath.Walk(walkRoot, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + if strings.HasPrefix(info.Name(), ".") { + return filepath.SkipDir + } + if skipNestedBuildDir(project, path) { + return filepath.SkipDir + } + return nil + } + if strings.HasPrefix(info.Name(), ".") || info.Name() == ConfigFileName { + return nil + } + + if strings.HasSuffix(strings.ToLower(info.Name()), ".macros.sql") { + relPath, err := filepath.Rel(baseDir, path) + if err != nil { + return g.Error(err, "could not get relative path for macro") + } + relDir := filepath.ToSlash(filepath.Dir(relPath)) + return collectMacro(project, path, relDir) + } + + ext := strings.ToLower(filepath.Ext(info.Name())) + if ext == ".sql" { + relPath, err := filepath.Rel(baseDir, path) + if err != nil { + return g.Error(err, "could not get relative path") + } + return addModel(project, path, relPath) + } + + return nil + }) +} + +// walkForSeeds discovers seed files under the given root. +func walkForSeeds(project *BuildProject, walkRoot, baseDir string) error { + if _, err := os.Stat(walkRoot); os.IsNotExist(err) { + return nil + } + + return filepath.Walk(walkRoot, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + if strings.HasPrefix(info.Name(), ".") { + return filepath.SkipDir + } + if skipNestedBuildDir(project, path) { + return filepath.SkipDir + } + return nil + } + if strings.HasPrefix(info.Name(), ".") || info.Name() == ConfigFileName { + return nil + } + + ext := strings.ToLower(filepath.Ext(info.Name())) + if format, ok := seedExtensions[ext]; ok { + relPath, err := filepath.Rel(baseDir, path) + if err != nil { + return g.Error(err, "could not get relative path") + } + return addSeed(project, path, relPath, format) + } + + return nil + }) +} + +// parseYAMLFrontmatter extracts YAML frontmatter from the leading comment of a +// SQL model file. The comment may use any of these styles: +// +// /** name: my graph **/ (doc-block, plain YAML) +// /* {"name": "my graph"} */ (block comment, JSON/flow) +// -- {name: my graph} (single-line, flow) +// -- { \n -- name: my graph \n -- } (multi-line line-comment, flow) +// +// For the /** ... **/ style any valid YAML mapping is accepted (canonical form). +// For other styles, the comment must contain a YAML/JSON mapping starting with +// '{' so plain prose comments aren't misread as frontmatter (e.g. a "-- Pre- +// statement: setup" line that incidentally parses as a YAML map). +// +// On success the YAML is parsed into ModelConfig and the SQL after the comment +// is returned. If no frontmatter is detected the original SQL is returned +// unchanged. +func parseYAMLFrontmatter(sql string) (config ModelConfig, remainingSQL string, hasFrontmatter bool, err error) { + trimmed := strings.TrimLeft(sql, " \t\r\n") + if trimmed == "" { + return ModelConfig{}, sql, false, nil + } + + var yamlContent string + var afterComment string + // requireBraceObject is true for comment styles that don't have a strong + // frontmatter signal (-- and /* */). Those need explicit '{' to be treated + // as frontmatter, otherwise they're just regular comments. + var requireBraceObject bool + var detected bool + + switch { + case strings.HasPrefix(trimmed, "/**"): + // /** ... **/ doc-block: canonical, plain YAML allowed. + closingIdx := strings.Index(trimmed[3:], "**/") + if closingIdx < 0 { + return ModelConfig{}, sql, false, nil + } + yamlContent = trimmed[3 : 3+closingIdx] + afterComment = trimmed[3+closingIdx+3:] + detected = true + requireBraceObject = false + + case strings.HasPrefix(trimmed, "/*"): + // /* ... */ block comment: must contain a {} object to be frontmatter. + closingIdx := strings.Index(trimmed[2:], "*/") + if closingIdx < 0 { + return ModelConfig{}, sql, false, nil + } + yamlContent = trimmed[2 : 2+closingIdx] + afterComment = trimmed[2+closingIdx+2:] + detected = true + requireBraceObject = true + + case strings.HasPrefix(trimmed, "--"): + // Consecutive `-- ...` lines: must contain a {} object to be frontmatter. + lines := strings.Split(trimmed, "\n") + var contentLines []string + consumed := 0 + for i, line := range lines { + ltrim := strings.TrimLeft(line, " \t") + if !strings.HasPrefix(ltrim, "--") { + consumed = i + break + } + body := strings.TrimPrefix(ltrim, "--") + body = strings.TrimPrefix(body, " ") + contentLines = append(contentLines, body) + consumed = i + 1 + } + yamlContent = strings.Join(contentLines, "\n") + afterComment = strings.Join(lines[consumed:], "\n") + detected = true + requireBraceObject = true + } + + if !detected { + return ModelConfig{}, sql, false, nil + } + + contentStripped := strings.TrimSpace(yamlContent) + if contentStripped == "" { + // Empty /** **/ → frontmatter present but empty config (preserves + // existing behavior). Empty /* */ or -- block → not frontmatter. + if requireBraceObject { + return ModelConfig{}, sql, false, nil + } + return ModelConfig{}, stripLeadingNewline(afterComment), true, nil + } + + if requireBraceObject && !strings.HasPrefix(contentStripped, "{") { + // Looks like a regular comment — pass through unchanged. + return ModelConfig{}, sql, false, nil + } + + // Validate that the content is actually a YAML mapping (not a scalar or + // list). For required-brace styles a parse failure means "regular comment", + // not frontmatter. + var asMap map[string]any + if mapErr := yaml.Unmarshal([]byte(yamlContent), &asMap); mapErr != nil { + if requireBraceObject { + return ModelConfig{}, sql, false, nil + } + return ModelConfig{}, sql, false, g.Error(mapErr, "could not parse YAML frontmatter") + } + if asMap == nil { + // Scalar/null — treat as regular comment for require-brace styles, or + // empty config for /** **/ style. + if requireBraceObject { + return ModelConfig{}, sql, false, nil + } + return ModelConfig{}, stripLeadingNewline(afterComment), true, nil + } + + cfg := ModelConfig{} + if err := yaml.Unmarshal([]byte(yamlContent), &cfg); err != nil { + return ModelConfig{}, sql, false, g.Error(err, "could not parse YAML frontmatter") + } + + if cfg.PreHook != "" || cfg.PostHook != "" { + return ModelConfig{}, sql, false, g.Error("pre_hook/post_hook are not supported in sling build frontmatter. Use hooks.start/hooks.end instead.\nSee https://docs.slingdata.io/concepts/sling-build for details") + } + + return cfg, stripLeadingNewline(afterComment), true, nil +} + +// stripLeadingNewline removes a single leading \n or \r\n. +func stripLeadingNewline(s string) string { + if strings.HasPrefix(s, "\r\n") { + return s[2:] + } + if strings.HasPrefix(s, "\n") { + return s[1:] + } + return s +} + +// addModel creates a Model from a file and adds it to the project. +func addModel(project *BuildProject, absPath, relPath string) error { + rawSQL, err := os.ReadFile(absPath) + if err != nil { + return g.Error(err, "could not read model file: %s", absPath) + } + + schema, prefix, name, fullTableName := resolveTableName(relPath, project.Mode, project.SchemaOverride, project.DefaultSchema) + _, _, _, prodFullTableName := resolveTableName(relPath, "prod", "", project.DefaultSchema) + + // Apply merged defaults (root + child sling_build.yml) for this file's location. + defaults := effectiveDefaults(project, relPath) + modelConfig := ModelConfig{ + Mode: defaults.Mode, + Schema: defaults.Schema, + Tags: append([]string(nil), defaults.Tags...), + UniqueKey: defaults.UniqueKey, + UpdateKey: defaults.UpdateKey, + MergeStrategy: defaults.MergeStrategy, + Enabled: defaults.Enabled, + Hooks: defaults.Hooks, + DropCascade: defaults.DropCascade, + } + + // Parse YAML frontmatter (canonical config declaration) + sqlContent := string(rawSQL) + hasFrontmatter := false + fmConfig, remaining, hasFM, fmErr := parseYAMLFrontmatter(sqlContent) + if fmErr != nil { + return g.Error(fmErr, "model '%s'", name) + } + if hasFM { + hasFrontmatter = true + sqlContent = remaining + // Frontmatter overrides defaults (non-zero fields only). + // Tags and Hooks layer additively on top of defaults instead of replacing. + if fmConfig.Mode != "" { + modelConfig.Mode = fmConfig.Mode + } + if fmConfig.UniqueKey != nil { + modelConfig.UniqueKey = fmConfig.UniqueKey + } + if fmConfig.MergeStrategy != "" { + modelConfig.MergeStrategy = fmConfig.MergeStrategy + } + if fmConfig.UpdateKey != "" { + modelConfig.UpdateKey = fmConfig.UpdateKey + } + if len(fmConfig.Tags) > 0 { + modelConfig.Tags = lo.Uniq(append(modelConfig.Tags, fmConfig.Tags...)) + } + modelConfig.Hooks = mergeHookMaps(modelConfig.Hooks, fmConfig.Hooks) + if fmConfig.Schema != "" { + modelConfig.Schema = fmConfig.Schema + } + if fmConfig.Enabled != nil { + modelConfig.Enabled = fmConfig.Enabled + } + if fmConfig.Engine != "" { + modelConfig.Engine = fmConfig.Engine + } + if fmConfig.Range != nil { + modelConfig.Range = fmConfig.Range + } + if fmConfig.DropCascade != nil { + modelConfig.DropCascade = fmConfig.DropCascade + } + if fmConfig.Rewrite != nil { + modelConfig.Rewrite = fmConfig.Rewrite + } + if len(fmConfig.Tests) > 0 { + modelConfig.Tests = fmConfig.Tests + } + if fmConfig.Materialized != "" { + modelConfig.Materialized = fmConfig.Materialized + } + } + + // Schema override from defaults or frontmatter — recompute FullTableName. + // (Before this change, a frontmatter `schema:` was silently ignored at the + // table-name level; only modelConfig.Schema was set.) + // ProdFullTableName is intentionally NOT rewritten: it exists specifically + // as the prod-mode reference for SQL ref() matching. + if modelConfig.Schema != "" { + schema = modelConfig.Schema + qualifiedName := name + if prefix != "" { + qualifiedName = prefix + "_" + name + } + fullTableName = schema + "." + qualifiedName + } + + // Detect incremental pattern (dbt-style vs sling-native). Errors at load time + // if the model mixes both patterns. + style, styleErr := detectModelStyle(sqlContent) + if styleErr != nil { + return g.Error(styleErr, "model '%s'", name) + } + + model := &Model{ + Name: name, + FilePath: absPath, + RelPath: relPath, + Schema: schema, + Prefix: prefix, + FullTableName: fullTableName, + ProdFullTableName: prodFullTableName, + RawSQL: sqlContent, + Config: modelConfig, + HasFrontmatter: hasFrontmatter, + Style: style, + } + + if err := validateModel(model); err != nil { + return err + } + + if existing, ok := project.Models[name]; ok { + return g.Error("duplicate model name '%s': found in both '%s' and '%s'", name, existing.RelPath, relPath) + } + + project.Models[name] = model + return nil +} + +// addSeed creates a Seed from a file and adds it to the project. +func addSeed(project *BuildProject, absPath, relPath, format string) error { + schema, prefix, name, fullTableName := resolveTableName(relPath, project.Mode, project.SchemaOverride, project.DefaultSchema) + _, _, _, prodFullTableName := resolveTableName(relPath, "prod", "", project.DefaultSchema) + + // Seeds only honor defaults.schema from the merged config — no tags, + // enabled, hooks, or unique_key semantics apply to seeds today. + if defaults := effectiveDefaults(project, relPath); defaults.Schema != "" { + schema = defaults.Schema + qualifiedName := name + if prefix != "" { + qualifiedName = prefix + "_" + name + } + fullTableName = schema + "." + qualifiedName + } + + seed := &Seed{ + Name: name, + FilePath: absPath, + RelPath: relPath, + Schema: schema, + Prefix: prefix, + FullTableName: fullTableName, + ProdFullTableName: prodFullTableName, + Format: format, + } + + if existing, ok := project.Seeds[name]; ok { + return g.Error("duplicate seed name '%s': found in both '%s' and '%s'", name, existing.RelPath, relPath) + } + + project.Seeds[name] = seed + return nil +} + +// resolveTableName determines the schema, prefix, name, and full table name from a relative path. +func resolveTableName(relPath, mode, schemaOverride, defaultSchema string) (schema, prefix, name, fullTableName string) { + // Normalize path separators + relPath = filepath.ToSlash(relPath) + + // Split path into parts + parts := strings.Split(relPath, "/") + + // Extract filename and remove extension + fileName := parts[len(parts)-1] + ext := filepath.Ext(fileName) + name = strings.TrimSuffix(fileName, ext) + + // Get directory parts (excluding filename) + dirParts := parts[:len(parts)-1] + + if mode == "dev" { + // Dev mode: all folder parts become prefix, use override schema + schema = schemaOverride + if len(dirParts) > 0 { + prefix = strings.Join(dirParts, "_") + } + } else { + // Prod mode: 1st folder = schema, remaining = prefix + if len(dirParts) == 0 { + // Root-level file + schema = defaultSchema + } else { + schema = dirParts[0] + if len(dirParts) > 1 { + prefix = strings.Join(dirParts[1:], "_") + } + } + } + + // Build full table name + qualifiedName := name + if prefix != "" { + qualifiedName = prefix + "_" + name + } + fullTableName = schema + "." + qualifiedName + + return +} + +// validateUniqueNames checks that there are no duplicate names across models and seeds. +func validateUniqueNames(project *BuildProject) error { + seen := make(map[string]string) // name -> "model" or "seed" + + for name := range project.Models { + if existing, ok := seen[name]; ok { + return g.Error("duplicate name '%s': found as both %s and model", name, existing) + } + seen[name] = "model" + } + + for name := range project.Seeds { + if existing, ok := seen[name]; ok { + return g.Error("duplicate name '%s': found as both %s and seed", name, existing) + } + seen[name] = "seed" + } + + return nil +} + +// effectiveDefaults returns the merged BuildDefaults that apply to the file at relPath. +// It is the single source of truth for "what defaults apply to this file" and is +// reusable by both addModel and addSeed. +func effectiveDefaults(project *BuildProject, relPath string) BuildDefaults { + relDir := filepath.Dir(relPath) + if relDir == "." { + relDir = "" + } + cfg := project.GetEffectiveConfig(relDir) + if cfg == nil { + return BuildDefaults{} + } + return cfg.Defaults +} + +// GetEffectiveConfig returns the merged config for the project, applying child overrides. +func (p *BuildProject) GetEffectiveConfig(dir string) *BuildConfig { + if dir == "" || dir == "." { + return p.Config + } + + parts := strings.Split(dir, string(filepath.Separator)) + if len(parts) > 0 { + if childCfg, ok := p.ChildConfigs[parts[0]]; ok { + return mergeConfigs(p.Config, childCfg) + } + } + + return p.Config +} + +// AllNames returns a sorted list of all model and seed names. +func (p *BuildProject) AllNames() []string { + names := make([]string, 0, len(p.Models)+len(p.Seeds)) + for name := range p.Models { + names = append(names, name) + } + for name := range p.Seeds { + names = append(names, name) + } + return lo.Uniq(names) +} + +// LookupFullTableName returns the full table name for a given model or seed name. +func (p *BuildProject) LookupFullTableName(name string) (string, bool) { + if m, ok := p.Models[name]; ok { + return m.FullTableName, true + } + if s, ok := p.Seeds[name]; ok { + return s.FullTableName, true + } + return "", false +} + +// prodNameEntry maps a prod-mode name to its model/seed name and current-mode FullTableName. +type prodNameEntry struct { + Name string // model or seed name + FullTableName string // current-mode full table name +} + +// BuildProdNameIndex builds a lookup map from lowercased prod FullTableName (and unqualified name) +// to the model/seed entry. Qualified names take priority over unqualified for the same key. +func (p *BuildProject) BuildProdNameIndex() map[string]prodNameEntry { + index := make(map[string]prodNameEntry) + + // First pass: add unqualified names (lower priority) + for _, m := range p.Models { + key := strings.ToLower(m.Name) + index[key] = prodNameEntry{Name: m.Name, FullTableName: m.FullTableName} + } + for _, s := range p.Seeds { + key := strings.ToLower(s.Name) + index[key] = prodNameEntry{Name: s.Name, FullTableName: s.FullTableName} + } + + // Second pass: add qualified prod names (higher priority, overwrites unqualified if same key) + for _, m := range p.Models { + key := strings.ToLower(m.ProdFullTableName) + index[key] = prodNameEntry{Name: m.Name, FullTableName: m.FullTableName} + } + for _, s := range p.Seeds { + key := strings.ToLower(s.ProdFullTableName) + index[key] = prodNameEntry{Name: s.Name, FullTableName: s.FullTableName} + } + + return index +} + +// ============================================================================= +// Seed Loading +// ============================================================================= + +// LoadSeed loads a seed file into the target database using the existing +// sling task infrastructure. This gets CSV/JSON/Parquet parsing, type inference, +// bulk loading, and all 30+ connectors for free. Seeds always use full-refresh. +func LoadSeed(seed *Seed, connName string, fullRefresh bool) error { + _ = fullRefresh // seeds always full-refresh; kept for call-site clarity + + // Build source connection using file:// prefix for the directory + sourceDir := filepath.Dir(seed.FilePath) + sourceFile := filepath.Base(seed.FilePath) + + cfg := &sling.Config{ + Source: sling.Source{ + Conn: "file://" + sourceDir, + Stream: sourceFile, + Options: &sling.SourceOptions{}, + }, + Target: sling.Target{ + Conn: connName, + Object: seed.FullTableName, + }, + Mode: sling.FullRefreshMode, + } + + task := sling.NewTask("", cfg) + if task.Err != nil { + return g.Error(task.Err, "could not create task for seed '%s'", seed.Name) + } + + if err := task.Execute(); err != nil { + return g.Error(err, "could not load seed '%s' into %s", seed.Name, seed.FullTableName) + } + + return nil +} + +// MakeSeedConfig creates a sling.Config for loading a seed file +// without executing it. Useful for testing and compile mode. +func MakeSeedConfig(seed *Seed, connName string) *sling.Config { + sourceDir := filepath.Dir(seed.FilePath) + sourceFile := filepath.Base(seed.FilePath) + + return &sling.Config{ + Source: sling.Source{ + Conn: "file://" + sourceDir, + Stream: sourceFile, + Options: &sling.SourceOptions{}, + }, + Target: sling.Target{ + Conn: connName, + Object: seed.FullTableName, + }, + Mode: sling.FullRefreshMode, + } +} diff --git a/core/sling/build/project_test.go b/core/sling/build/project_test.go new file mode 100644 index 000000000..fc7667c75 --- /dev/null +++ b/core/sling/build/project_test.go @@ -0,0 +1,1516 @@ +package build + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/slingdata-io/sling-cli/core/sling" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func getTestFixturePath(name string) string { + // Tests run from core/sling/build/, fixtures are at tests/build/ + // We need to find the project root first + wd, _ := os.Getwd() + // Walk up to find go.mod + dir := wd + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + break + } + parent := filepath.Dir(dir) + if parent == dir { + // Fallback to relative path + return filepath.Join("../../../tests/build", name) + } + dir = parent + } + return filepath.Join(dir, "tests/build", name) +} + +func TestResolveTableName(t *testing.T) { + tests := []struct { + name string + relPath string + mode string + schemaOverride string + defaultSchema string + wantSchema string + wantPrefix string + wantName string + wantFull string + }{ + { + name: "prod mode - first level folder is schema", + relPath: "staging/stg_orders.sql", + mode: "prod", + defaultSchema: "public", + wantSchema: "staging", + wantPrefix: "", + wantName: "stg_orders", + wantFull: "staging.stg_orders", + }, + { + name: "prod mode - nested folder becomes prefix", + relPath: "marts/core/dim_customers.sql", + mode: "prod", + defaultSchema: "public", + wantSchema: "marts", + wantPrefix: "core", + wantName: "dim_customers", + wantFull: "marts.core_dim_customers", + }, + { + name: "prod mode - deeply nested folders", + relPath: "marts/core/finance/revenue.sql", + mode: "prod", + defaultSchema: "public", + wantSchema: "marts", + wantPrefix: "core_finance", + wantName: "revenue", + wantFull: "marts.core_finance_revenue", + }, + { + name: "prod mode - root level file uses default schema", + relPath: "raw.sql", + mode: "prod", + defaultSchema: "public", + wantSchema: "public", + wantPrefix: "", + wantName: "raw", + wantFull: "public.raw", + }, + { + name: "dev mode - all folders become prefix", + relPath: "staging/stg_orders.sql", + mode: "dev", + schemaOverride: "dev_fritz", + defaultSchema: "public", + wantSchema: "dev_fritz", + wantPrefix: "staging", + wantName: "stg_orders", + wantFull: "dev_fritz.staging_stg_orders", + }, + { + name: "dev mode - nested folders all become prefix", + relPath: "marts/core/dim_customers.sql", + mode: "dev", + schemaOverride: "dev_fritz", + defaultSchema: "public", + wantSchema: "dev_fritz", + wantPrefix: "marts_core", + wantName: "dim_customers", + wantFull: "dev_fritz.marts_core_dim_customers", + }, + { + name: "dev mode - root level file", + relPath: "raw.sql", + mode: "dev", + schemaOverride: "dev_fritz", + defaultSchema: "public", + wantSchema: "dev_fritz", + wantPrefix: "", + wantName: "raw", + wantFull: "dev_fritz.raw", + }, + { + name: "csv seed file", + relPath: "staging/country_codes.csv", + mode: "prod", + defaultSchema: "public", + wantSchema: "staging", + wantPrefix: "", + wantName: "country_codes", + wantFull: "staging.country_codes", + }, + { + name: "json seed file in nested dir", + relPath: "seeds/status_map.json", + mode: "prod", + defaultSchema: "public", + wantSchema: "seeds", + wantPrefix: "", + wantName: "status_map", + wantFull: "seeds.status_map", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + schema, prefix, name, fullTableName := resolveTableName(tt.relPath, tt.mode, tt.schemaOverride, tt.defaultSchema) + assert.Equal(t, tt.wantSchema, schema, "schema") + assert.Equal(t, tt.wantPrefix, prefix, "prefix") + assert.Equal(t, tt.wantName, name, "name") + assert.Equal(t, tt.wantFull, fullTableName, "fullTableName") + }) + } +} + +func TestDiscoverFilesFlatMode(t *testing.T) { + dir := getTestFixturePath("sample_project") + + project := &BuildProject{ + Dir: dir, + Models: make(map[string]*Model), + Seeds: make(map[string]*Seed), + Mode: "prod", + DefaultSchema: "public", + ChildConfigs: make(map[string]*BuildConfig), + } + project.Config, _ = loadConfig(filepath.Join(dir, ConfigFileName)) + + err := discoverFiles(project) + require.NoError(t, err) + + // Check models + expectedModels := []string{"stg_orders", "stg_customers", "dim_customers", "fct_orders", "revenue", "raw"} + assert.Len(t, project.Models, len(expectedModels)) + for _, name := range expectedModels { + assert.Contains(t, project.Models, name, "expected model: %s", name) + } + + // Check seeds + expectedSeeds := []string{"country_codes", "status_map"} + assert.Len(t, project.Seeds, len(expectedSeeds)) + for _, name := range expectedSeeds { + assert.Contains(t, project.Seeds, name, "expected seed: %s", name) + } + + // Check specific model properties + stgOrders := project.Models["stg_orders"] + assert.Equal(t, "staging", stgOrders.Schema) + assert.Equal(t, "", stgOrders.Prefix) + assert.Equal(t, "staging.stg_orders", stgOrders.FullTableName) + assert.NotEmpty(t, stgOrders.RawSQL) + + dimCustomers := project.Models["dim_customers"] + assert.Equal(t, "marts", dimCustomers.Schema) + assert.Equal(t, "core", dimCustomers.Prefix) + assert.Equal(t, "marts.core_dim_customers", dimCustomers.FullTableName) + + revenue := project.Models["revenue"] + assert.Equal(t, "marts", revenue.Schema) + assert.Equal(t, "finance", revenue.Prefix) + assert.Equal(t, "marts.finance_revenue", revenue.FullTableName) + + raw := project.Models["raw"] + assert.Equal(t, "public", raw.Schema) + assert.Equal(t, "", raw.Prefix) + assert.Equal(t, "public.raw", raw.FullTableName) + + // Check seed properties + countryCodes := project.Seeds["country_codes"] + assert.Equal(t, "staging", countryCodes.Schema) + assert.Equal(t, "csv", countryCodes.Format) + assert.Equal(t, "staging.country_codes", countryCodes.FullTableName) + + statusMap := project.Seeds["status_map"] + assert.Equal(t, "seeds", statusMap.Schema) + assert.Equal(t, "json", statusMap.Format) + + // Check macros are collected (not counted as models) + assert.Len(t, project.Macros, 2) // utils.macros.sql + staging_helpers.macros.sql +} + +func TestDiscoverFilesDbtProjectMode(t *testing.T) { + dir := getTestFixturePath("dbt_compat_project") + + project := &BuildProject{ + Dir: dir, + Models: make(map[string]*Model), + Seeds: make(map[string]*Seed), + Mode: "prod", + DefaultSchema: "public", + ChildConfigs: make(map[string]*BuildConfig), + } + project.Config, _ = loadConfig(filepath.Join(dir, ConfigFileName)) + + err := discoverFiles(project) + require.NoError(t, err) + + // Check models - should only find SQL in models/ dir + assert.Len(t, project.Models, 1) + assert.Contains(t, project.Models, "stg_orders") + + // Check model naming: staging/ is first folder under models/, so schema=staging + stgOrders := project.Models["stg_orders"] + assert.Equal(t, "staging", stgOrders.Schema) + assert.Equal(t, "staging.stg_orders", stgOrders.FullTableName) + + // Check seeds - should only find seeds in seeds/ dir + assert.Len(t, project.Seeds, 1) + assert.Contains(t, project.Seeds, "country_codes") + + countryCodes := project.Seeds["country_codes"] + assert.Equal(t, "staging", countryCodes.Schema) + assert.Equal(t, "csv", countryCodes.Format) +} + +func TestLoadProjectSampleProject(t *testing.T) { + dir := getTestFixturePath("sample_project") + + project, err := LoadProject(dir) + require.NoError(t, err) + + assert.Equal(t, "prod", project.Mode) + assert.NotNil(t, project.Config) + assert.Equal(t, "POSTGRES", project.Config.Target) + assert.Len(t, project.Models, 6) + assert.Len(t, project.Seeds, 2) + assert.NotEmpty(t, project.Macros) +} + +func TestLoadProjectDbtCompat(t *testing.T) { + dir := getTestFixturePath("dbt_compat_project") + + project, err := LoadProject(dir) + require.NoError(t, err) + + assert.NotNil(t, project.Config) + assert.Equal(t, true, project.Config.DbtProject) + assert.Len(t, project.Models, 1) + assert.Len(t, project.Seeds, 1) +} + +func TestLoadProjectWithCliOverrides(t *testing.T) { + dir := getTestFixturePath("sample_project") + + // Test dev mode override via --schema + project, err := LoadProject(dir, BuildOptions{Schema: "dev_test"}) + require.NoError(t, err) + + assert.Equal(t, "dev", project.Mode) + assert.Equal(t, "dev_test", project.SchemaOverride) + + // In dev mode, all models should use the override schema + for _, model := range project.Models { + assert.Equal(t, "dev_test", model.Schema, "model %s should use dev schema", model.Name) + } + for _, seed := range project.Seeds { + assert.Equal(t, "dev_test", seed.Schema, "seed %s should use dev schema", seed.Name) + } +} + +func TestLoadProjectWithProdOverride(t *testing.T) { + dir := getTestFixturePath("sample_project") + + // Test --prod flag + project, err := LoadProject(dir, BuildOptions{Prod: true}) + require.NoError(t, err) + + assert.Equal(t, "prod", project.Mode) +} + +func TestNestedProjectIsolation(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(dir, "sling_build.yml"), []byte("target: POSTGRES\n"), 0644)) + require.NoError(t, os.MkdirAll(filepath.Join(dir, "staging"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "staging", "stg_ok.sql"), []byte("select 1 as id\n"), 0644)) + + probe := filepath.Join(dir, "probe") + require.NoError(t, os.MkdirAll(filepath.Join(probe, "staging"), 0755)) + require.NoError(t, os.WriteFile(filepath.Join(probe, "sling_build.yml"), []byte("target: DUCKDB\n"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(probe, "staging", "stg_broken.sql"), []byte("select not_a_column from nowhere\n"), 0644)) + + project, err := LoadProject(dir) + require.NoError(t, err) + require.NotNil(t, project.Models["stg_ok"]) + require.Nil(t, project.Models["stg_broken"], "nested probe project must not join parent discovery") + + rec, err := LoadProject(dir, BuildOptions{Recursive: true}) + require.NoError(t, err) + require.NotNil(t, rec.Models["stg_ok"]) + require.NotNil(t, rec.Models["stg_broken"], "-R keeps immediate child models") +} + +func TestNestedConfigInheritance(t *testing.T) { + dir := getTestFixturePath("nested_yml_project") + + // Nested sling_build.yml overrides require --recursive + project, err := LoadProject(dir, BuildOptions{Recursive: true}) + require.NoError(t, err) + + assert.NotNil(t, project.Config) + assert.Equal(t, "POSTGRES", project.Config.Target) + assert.Equal(t, "full-refresh", project.Config.Defaults.Mode) + + // The staging child config should override defaults.mode to "truncate" + assert.Contains(t, project.ChildConfigs, "staging") + assert.Equal(t, "truncate", project.ChildConfigs["staging"].Defaults.Mode) + + // Models in staging should get the overridden mode + stgOrders := project.Models["stg_orders"] + require.NotNil(t, stgOrders) + assert.Equal(t, "truncate", stgOrders.Config.Mode) + + // Models in marts should get the root default mode + dimCustomers := project.Models["dim_customers"] + require.NotNil(t, dimCustomers) + assert.Equal(t, "full-refresh", dimCustomers.Config.Mode) +} + +func TestMultiTargetIndependentBuilds(t *testing.T) { + dir := getTestFixturePath("multi_target_project") + + // Independent sub-projects require --recursive to discover child ymls + project, err := LoadProject(dir, BuildOptions{Recursive: true}) + require.NoError(t, err) + + // Should have no root config + assert.Nil(t, project.Config) + + // Should have sub-projects + assert.Len(t, project.SubProjects, 2) + + // Verify each sub-project + targets := make(map[string]bool) + for _, sub := range project.SubProjects { + require.NotNil(t, sub.Config) + targets[sub.Config.Target] = true + assert.Len(t, sub.Models, 1) // each has one model + } + + assert.True(t, targets["POSTGRES"]) + assert.True(t, targets["CLICKHOUSE"]) +} + +func TestMergeConfigs(t *testing.T) { + parent := &BuildConfig{ + Target: "POSTGRES", + Dev: &DevConfig{Schema: "dev_schema"}, + Vars: map[string]any{ + "start_date": "2024-01-01", + "environment": "prod", + }, + Defaults: BuildDefaults{ + Mode: "full-refresh", + }, + } + + child := &BuildConfig{ + Vars: map[string]any{ + "environment": "staging", // override + "new_var": "value", // new + }, + Defaults: BuildDefaults{ + Mode: "truncate", // override + }, + } + + merged := mergeConfigs(parent, child) + + assert.Equal(t, "POSTGRES", merged.Target) // inherited + assert.Equal(t, "dev_schema", merged.Dev.Schema) // inherited + assert.Equal(t, "truncate", merged.Defaults.Mode) + assert.Equal(t, "2024-01-01", merged.Vars["start_date"]) // inherited + assert.Equal(t, "staging", merged.Vars["environment"]) // overridden + assert.Equal(t, "value", merged.Vars["new_var"]) // new +} + +func TestMergeConfigsNil(t *testing.T) { + cfg := &BuildConfig{Target: "PG"} + assert.Equal(t, cfg, mergeConfigs(nil, cfg)) + assert.Equal(t, cfg, mergeConfigs(cfg, nil)) +} + +func TestValidateUniqueNames(t *testing.T) { + // No duplicates - should pass + project := &BuildProject{ + Models: map[string]*Model{"model_a": {Name: "model_a"}}, + Seeds: map[string]*Seed{"seed_a": {Name: "seed_a"}}, + } + assert.NoError(t, validateUniqueNames(project)) + + // Model-seed collision - should fail + project = &BuildProject{ + Models: map[string]*Model{"same_name": {Name: "same_name"}}, + Seeds: map[string]*Seed{"same_name": {Name: "same_name"}}, + } + err := validateUniqueNames(project) + assert.Error(t, err) + assert.Contains(t, err.Error(), "duplicate name") +} + +func TestLoadProjectEmptyDir(t *testing.T) { + dir := t.TempDir() + + project, err := LoadProject(dir) + require.NoError(t, err) + + assert.Nil(t, project.Config) + assert.Len(t, project.Models, 0) + assert.Len(t, project.Seeds, 0) +} + +func TestLoadProjectNonExistentDir(t *testing.T) { + _, err := LoadProject("/nonexistent/path") + assert.Error(t, err) +} + +func TestGetDbtProjectConfig(t *testing.T) { + // false + cfg := &BuildConfig{DbtProject: false} + assert.Nil(t, getDbtProjectConfig(cfg)) + + // true - uses defaults + cfg = &BuildConfig{DbtProject: true} + dbtCfg := getDbtProjectConfig(cfg) + require.NotNil(t, dbtCfg) + assert.Equal(t, "models", dbtCfg.ModelsPath) + assert.Equal(t, "seeds", dbtCfg.SeedsPath) + + // Custom paths + cfg = &BuildConfig{DbtProject: map[string]any{ + "models_path": "custom_models", + "seeds_path": "custom_seeds", + }} + dbtCfg = getDbtProjectConfig(cfg) + require.NotNil(t, dbtCfg) + assert.Equal(t, "custom_models", dbtCfg.ModelsPath) + assert.Equal(t, "custom_seeds", dbtCfg.SeedsPath) + + // nil + assert.Nil(t, getDbtProjectConfig(nil)) + assert.Nil(t, getDbtProjectConfig(&BuildConfig{})) +} + +func TestLookupFullTableName(t *testing.T) { + project := &BuildProject{ + Models: map[string]*Model{ + "stg_orders": {Name: "stg_orders", FullTableName: "staging.stg_orders"}, + }, + Seeds: map[string]*Seed{ + "country_codes": {Name: "country_codes", FullTableName: "staging.country_codes"}, + }, + } + + name, ok := project.LookupFullTableName("stg_orders") + assert.True(t, ok) + assert.Equal(t, "staging.stg_orders", name) + + name, ok = project.LookupFullTableName("country_codes") + assert.True(t, ok) + assert.Equal(t, "staging.country_codes", name) + + _, ok = project.LookupFullTableName("nonexistent") + assert.False(t, ok) +} + +// ============================================================================= +// Seed Tests +// ============================================================================= + +func TestMakeSeedConfigCSV(t *testing.T) { + seed := &Seed{ + Name: "country_codes", + FilePath: "/tmp/project/staging/country_codes.csv", + RelPath: "staging/country_codes.csv", + Schema: "staging", + FullTableName: "staging.country_codes", + Format: "csv", + } + + cfg := MakeSeedConfig(seed, "POSTGRES") + + assert.Equal(t, "file:///tmp/project/staging", cfg.Source.Conn) + assert.Equal(t, "country_codes.csv", cfg.Source.Stream) + assert.Equal(t, "POSTGRES", cfg.Target.Conn) + assert.Equal(t, "staging.country_codes", cfg.Target.Object) + assert.Equal(t, sling.FullRefreshMode, cfg.Mode) +} + +func TestMakeSeedConfigJSON(t *testing.T) { + seed := &Seed{ + Name: "status_map", + FilePath: "/tmp/project/seeds/status_map.json", + RelPath: "seeds/status_map.json", + Schema: "seeds", + FullTableName: "seeds.status_map", + Format: "json", + } + + cfg := MakeSeedConfig(seed, "CLICKHOUSE") + + assert.Equal(t, "file:///tmp/project/seeds", cfg.Source.Conn) + assert.Equal(t, "status_map.json", cfg.Source.Stream) + assert.Equal(t, "CLICKHOUSE", cfg.Target.Conn) + assert.Equal(t, "seeds.status_map", cfg.Target.Object) + assert.Equal(t, sling.FullRefreshMode, cfg.Mode) +} + +func TestMakeSeedConfigParquet(t *testing.T) { + seed := &Seed{ + Name: "large_data", + FilePath: "/data/warehouse/large_data.parquet", + RelPath: "warehouse/large_data.parquet", + Schema: "warehouse", + FullTableName: "warehouse.large_data", + Format: "parquet", + } + + cfg := MakeSeedConfig(seed, "SNOWFLAKE") + + assert.Equal(t, "file:///data/warehouse", cfg.Source.Conn) + assert.Equal(t, "large_data.parquet", cfg.Source.Stream) + assert.Equal(t, "SNOWFLAKE", cfg.Target.Conn) + assert.Equal(t, "warehouse.large_data", cfg.Target.Object) + assert.Equal(t, sling.FullRefreshMode, cfg.Mode) +} + +func TestMakeSeedConfigSourceOptions(t *testing.T) { + seed := &Seed{ + Name: "test_seed", + FilePath: "/tmp/test_seed.csv", + RelPath: "test_seed.csv", + Schema: "public", + FullTableName: "public.test_seed", + Format: "csv", + } + + cfg := MakeSeedConfig(seed, "MY_DB") + + // Source options should be initialized (non-nil) + assert.NotNil(t, cfg.Source.Options) +} + +// ============================================================================= +// YAML Frontmatter Tests +// ============================================================================= + +func TestParseYAMLFrontmatter_Basic(t *testing.T) { + sql := `/** +mode: incremental +unique_key: id +update_key: updated_at +**/ +SELECT * FROM orders` + + cfg, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.True(t, hasFM) + assert.Equal(t, "incremental", cfg.Mode) + assert.Equal(t, "id", cfg.UniqueKey) + assert.Equal(t, "updated_at", cfg.UpdateKey) + assert.Equal(t, "SELECT * FROM orders", strings.TrimSpace(remaining)) +} + +func TestParseYAMLFrontmatter_AllFields(t *testing.T) { + sql := `/** +mode: incremental +unique_key: + - id + - tenant_id +merge_strategy: delete+insert +update_key: updated_at +tags: + - daily + - finance +hooks: + start: + - type: log + message: "Starting model build" + end: + - type: log + message: "Model build complete" +schema: analytics +enabled: false +engine: MergeTree() +**/ +SELECT 1` + + cfg, _, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.True(t, hasFM) + assert.Equal(t, "incremental", cfg.Mode) + assert.Equal(t, []interface{}{"id", "tenant_id"}, cfg.UniqueKey) + assert.Equal(t, "delete+insert", cfg.MergeStrategy) + assert.Equal(t, "updated_at", cfg.UpdateKey) + assert.Equal(t, []string{"daily", "finance"}, cfg.Tags) + assert.Len(t, cfg.Hooks.Start, 1) + assert.Len(t, cfg.Hooks.End, 1) + assert.Equal(t, "analytics", cfg.Schema) + assert.NotNil(t, cfg.Enabled) + assert.False(t, *cfg.Enabled) + assert.Equal(t, "MergeTree()", cfg.Engine) +} + +func TestParseYAMLFrontmatter_PreHookError(t *testing.T) { + sql := `/** +mode: full-refresh +pre_hook: "CREATE TEMP TABLE tmp AS SELECT 1" +**/ +SELECT 1` + + _, _, _, err := parseYAMLFrontmatter(sql) + require.Error(t, err) + assert.Contains(t, err.Error(), "pre_hook/post_hook are not supported") + assert.Contains(t, err.Error(), "hooks.start/hooks.end") +} + +func TestParseYAMLFrontmatter_PostHookError(t *testing.T) { + sql := `/** +mode: full-refresh +post_hook: "DROP TABLE IF EXISTS tmp" +**/ +SELECT 1` + + _, _, _, err := parseYAMLFrontmatter(sql) + require.Error(t, err) + assert.Contains(t, err.Error(), "pre_hook/post_hook are not supported") +} + +func TestParseYAMLFrontmatter_WithHooks(t *testing.T) { + sql := `/** +mode: incremental +unique_key: id +hooks: + start: + - type: query + connection: postgres + query: "REFRESH MATERIALIZED VIEW upstream_mv" + - type: log + message: "Starting model build" + end: + - type: check + check: execution.status.error == 0 + - type: log + message: "Model build complete" +**/ +SELECT * FROM {{ ref('stg_orders') }}` + + cfg, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.True(t, hasFM) + assert.Equal(t, "incremental", cfg.Mode) + assert.Equal(t, "id", cfg.UniqueKey) + assert.False(t, cfg.Hooks.IsEmpty()) + assert.Len(t, cfg.Hooks.Start, 2) + assert.Len(t, cfg.Hooks.End, 2) + assert.Contains(t, remaining, "SELECT * FROM") +} + +func TestParseYAMLFrontmatter_NoFrontmatter(t *testing.T) { + sql := `SELECT * FROM orders` + + cfg, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.False(t, hasFM) + assert.Equal(t, ModelConfig{}, cfg) + assert.Equal(t, sql, remaining) +} + +func TestParseYAMLFrontmatter_JinjaConfigNotFrontmatter(t *testing.T) { + sql := `{%- config(mode='incremental') -%} +SELECT * FROM orders` + + _, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.False(t, hasFM) + assert.Equal(t, sql, remaining) +} + +func TestParseYAMLFrontmatter_UnclosedDelimiter(t *testing.T) { + sql := `/** +mode: incremental +SELECT * FROM orders` + + _, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.False(t, hasFM) + assert.Equal(t, sql, remaining) +} + +func TestParseYAMLFrontmatter_RegularComment(t *testing.T) { + // A regular /* ... */ comment should NOT be treated as frontmatter + sql := `/* this is a regular comment */ +SELECT * FROM orders` + + _, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.False(t, hasFM) + assert.Equal(t, sql, remaining) +} + +func TestParseYAMLFrontmatter_LeadingWhitespace(t *testing.T) { + sql := ` +/** +mode: view +**/ +SELECT 1` + + cfg, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.True(t, hasFM) + assert.Equal(t, "view", cfg.Mode) + assert.Equal(t, "SELECT 1", strings.TrimSpace(remaining)) +} + +func TestParseYAMLFrontmatter_InvalidYAML(t *testing.T) { + sql := `/** +mode: [invalid yaml +**/ +SELECT 1` + + _, _, _, err := parseYAMLFrontmatter(sql) + assert.Error(t, err) +} + +func TestParseYAMLFrontmatter_EmptyFrontmatter(t *testing.T) { + sql := `/** +**/ +SELECT 1` + + cfg, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.True(t, hasFM) + assert.Equal(t, ModelConfig{}, cfg) + assert.Equal(t, "SELECT 1", strings.TrimSpace(remaining)) +} + +func TestParseYAMLFrontmatter_InlineOpening(t *testing.T) { + // /** on same line as first YAML key + sql := `/** mode: view +**/ +SELECT 1` + + cfg, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.True(t, hasFM) + assert.Equal(t, "view", cfg.Mode) + assert.Equal(t, "SELECT 1", strings.TrimSpace(remaining)) +} + +// ============================================================================= +// Flexible Frontmatter Format Tests +// ============================================================================= + +func TestParseYAMLFrontmatter_LineCommentInlineFlow(t *testing.T) { + sql := `-- {mode: view, schema: analytics} +SELECT 1` + + cfg, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.True(t, hasFM) + assert.Equal(t, "view", cfg.Mode) + assert.Equal(t, "analytics", cfg.Schema) + assert.Equal(t, "SELECT 1", strings.TrimSpace(remaining)) +} + +func TestParseYAMLFrontmatter_LineCommentMultilineFlow(t *testing.T) { + sql := `-- { +-- mode: incremental, +-- unique_key: id, +-- update_key: updated_at +-- } +SELECT * FROM raw_orders` + + cfg, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.True(t, hasFM) + assert.Equal(t, "incremental", cfg.Mode) + assert.Equal(t, "id", cfg.UniqueKey) + assert.Equal(t, "updated_at", cfg.UpdateKey) + assert.Equal(t, "SELECT * FROM raw_orders", strings.TrimSpace(remaining)) +} + +func TestParseYAMLFrontmatter_BlockCommentJSON(t *testing.T) { + sql := `/* {"mode": "view", "schema": "analytics"} */ +SELECT 1` + + cfg, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.True(t, hasFM) + assert.Equal(t, "view", cfg.Mode) + assert.Equal(t, "analytics", cfg.Schema) + assert.Equal(t, "SELECT 1", strings.TrimSpace(remaining)) +} + +func TestParseYAMLFrontmatter_BlockCommentMultilineFlow(t *testing.T) { + sql := `/* { + "mode": "incremental", + "unique_key": "id" +} */ +SELECT 1` + + cfg, _, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.True(t, hasFM) + assert.Equal(t, "incremental", cfg.Mode) + assert.Equal(t, "id", cfg.UniqueKey) +} + +func TestParseYAMLFrontmatter_LineCommentRegularComment(t *testing.T) { + // A `--` line comment without a {} object should NOT be detected as frontmatter. + sql := `-- Pre-statement: create a temp table for staging +CREATE TEMP TABLE tmp AS SELECT 1 AS id;` + + _, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.False(t, hasFM) + assert.Equal(t, sql, remaining) +} + +func TestParseYAMLFrontmatter_LineCommentMultipleRegular(t *testing.T) { + sql := `-- This is a comment +-- explaining the next query +SELECT 1` + + _, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.False(t, hasFM) + assert.Equal(t, sql, remaining) +} + +func TestParseYAMLFrontmatter_BlockCommentNoBraces(t *testing.T) { + // Plain prose in /* */ should remain a regular comment. + sql := `/* this is a regular comment about the query */ +SELECT 1` + + _, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.False(t, hasFM) + assert.Equal(t, sql, remaining) +} + +func TestParseYAMLFrontmatter_LineCommentSimpleName(t *testing.T) { + // Single-key flow object on one line — common minimal frontmatter. + sql := `-- {schema: marts} +SELECT 1` + + cfg, _, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.True(t, hasFM) + assert.Equal(t, "marts", cfg.Schema) +} + +func TestParseYAMLFrontmatter_LineCommentWithHooks(t *testing.T) { + sql := `-- { +-- mode: incremental, +-- unique_key: id, +-- tags: [daily, finance] +-- } +SELECT 1` + + cfg, _, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.True(t, hasFM) + assert.Equal(t, "incremental", cfg.Mode) + assert.Equal(t, []string{"daily", "finance"}, cfg.Tags) +} + +func TestParseYAMLFrontmatter_LineCommentInvalidFlowFallsBack(t *testing.T) { + // `-- { not valid yaml` shouldn't crash; should be treated as regular comment. + sql := `-- { not valid yaml here +SELECT 1` + + _, remaining, hasFM, err := parseYAMLFrontmatter(sql) + require.NoError(t, err) + assert.False(t, hasFM) + assert.Equal(t, sql, remaining) +} + +func TestParseYAMLFrontmatter_BlockCommentPreHookErrors(t *testing.T) { + sql := `/* {"mode": "full-refresh", "pre_hook": "x"} */ +SELECT 1` + + _, _, _, err := parseYAMLFrontmatter(sql) + require.Error(t, err) + assert.Contains(t, err.Error(), "pre_hook/post_hook are not supported") +} + +func TestAddModelWithFrontmatter(t *testing.T) { + dir := t.TempDir() + sqlContent := `/** +mode: incremental +unique_key: id +update_key: updated_at +**/ +SELECT * FROM raw_orders` + + modelPath := filepath.Join(dir, "orders.sql") + require.NoError(t, os.WriteFile(modelPath, []byte(sqlContent), 0644)) + + project := &BuildProject{ + Dir: dir, + Mode: "prod", + DefaultSchema: "public", + Models: make(map[string]*Model), + Seeds: make(map[string]*Seed), + ChildConfigs: make(map[string]*BuildConfig), + } + + err := addModel(project, modelPath, "orders.sql") + require.NoError(t, err) + + model := project.Models["orders"] + require.NotNil(t, model) + assert.True(t, model.HasFrontmatter) + assert.Equal(t, "incremental", model.Config.Mode) + assert.Equal(t, "id", model.Config.UniqueKey) + assert.Equal(t, "updated_at", model.Config.UpdateKey) + assert.Equal(t, "SELECT * FROM raw_orders", strings.TrimSpace(model.RawSQL)) +} + +func TestSeedInBuildProject(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{Target: "POSTGRES"}) + require.NoError(t, err) + require.NoError(t, b.Compile()) + + // Seeds should be at depth 0 in DAG + countryCodes := b.DAG.Nodes["country_codes"] + require.NotNil(t, countryCodes) + assert.NotNil(t, countryCodes.Seed) + assert.Nil(t, countryCodes.Model) + assert.Equal(t, 0, countryCodes.Depth) + assert.Empty(t, countryCodes.Dependencies) + + statusMap := b.DAG.Nodes["status_map"] + require.NotNil(t, statusMap) + assert.NotNil(t, statusMap.Seed) + assert.Equal(t, 0, statusMap.Depth) + + // Seeds should appear in execution order + levels := b.DAG.GetExecutionLevels() + require.NotEmpty(t, levels) + + // Level 0 should contain seeds and models with no deps + level0 := levels[0] + assert.Contains(t, level0, "country_codes") + assert.Contains(t, level0, "status_map") +} + +// ============================================================================= +// Phase 2: RangeConfig + validateModel tests +// ============================================================================= + +// addModelFromFrontmatter is a test helper that writes a model SQL file containing +// the given YAML frontmatter and body, then calls addModel() on a fresh project. +// Returns the resulting error (nil on success) and the Model if it was added. +func addModelFromFrontmatter(t *testing.T, frontmatter, body string) (*Model, error) { + t.Helper() + dir := t.TempDir() + content := "/**\n" + frontmatter + "\n**/\n" + body + modelPath := filepath.Join(dir, "m.sql") + require.NoError(t, os.WriteFile(modelPath, []byte(content), 0644)) + + project := &BuildProject{ + Dir: dir, + Mode: "prod", + DefaultSchema: "public", + Models: make(map[string]*Model), + Seeds: make(map[string]*Seed), + ChildConfigs: make(map[string]*BuildConfig), + } + + if err := addModel(project, modelPath, "m.sql"); err != nil { + return nil, err + } + return project.Models["m"], nil +} + +func TestFrontmatterMaterializedAlias(t *testing.T) { + m, err := addModelFromFrontmatter(t, + "materialized: table\nunique_key: id", + "SELECT 1 as id") + require.NoError(t, err) + assert.Equal(t, "full-refresh", m.Config.Mode) +} + +func TestFrontmatterSnapshotRenamedToAppend(t *testing.T) { + m, err := addModelFromFrontmatter(t, + "mode: snapshot", + "SELECT 1 as id") + require.NoError(t, err) + assert.Equal(t, "append", m.Config.Mode) +} + +func TestFrontmatterDataTests(t *testing.T) { + m, err := addModelFromFrontmatter(t, + "mode: full-refresh\ntests:\n - not_null: [id]\n - unique: id\n - expr: sum(id) > 0", + "SELECT 1 as id") + require.NoError(t, err) + require.Len(t, m.Config.Tests, 3) +} + +func TestFrontmatterRewriteFalse(t *testing.T) { + m, err := addModelFromFrontmatter(t, + "mode: full-refresh\nrewrite: false", + "SELECT * FROM external_orders") + require.NoError(t, err) + require.NotNil(t, m.Config.Rewrite) + assert.False(t, *m.Config.Rewrite) +} + +func TestFrontmatterDropCascade(t *testing.T) { + m, err := addModelFromFrontmatter(t, + "mode: full-refresh\ndrop_cascade: true", + "SELECT 1 as id") + require.NoError(t, err) + require.NotNil(t, m.Config.DropCascade) + assert.True(t, *m.Config.DropCascade) +} + +func TestRangeConfig_Valid_AbsentMeansPlainIncremental(t *testing.T) { + m, err := addModelFromFrontmatter(t, + "mode: incremental\nunique_key: id\nupdate_key: updated_at", + "SELECT id, updated_at FROM raw WHERE {incremental_where_cond}") + require.NoError(t, err) + require.NotNil(t, m) + assert.Nil(t, m.Config.Range) + assert.Equal(t, StyleSling, m.Style) +} + +func TestRangeConfig_Valid_AdvanceOnly(t *testing.T) { + m, err := addModelFromFrontmatter(t, + "mode: incremental\nunique_key: id\nupdate_key: updated_at\nrange:\n advance: 7d", + "SELECT id, updated_at FROM raw WHERE {incremental_where_cond}") + require.NoError(t, err) + require.NotNil(t, m.Config.Range) + assert.Equal(t, "7d", m.Config.Range.Advance) + assert.True(t, m.Config.Range.HasAdvance()) + assert.False(t, m.Config.Range.HasLookback()) +} + +func TestRangeConfig_Valid_StartAndAdvance(t *testing.T) { + m, err := addModelFromFrontmatter(t, + "mode: incremental\nunique_key: id\nupdate_key: updated_at\nrange:\n start: '2020-01-01'\n advance: 1mo", + "SELECT id, updated_at FROM raw WHERE {incremental_where_cond}") + require.NoError(t, err) + require.NotNil(t, m.Config.Range) + assert.Equal(t, "2020-01-01", m.Config.Range.Start) + assert.Equal(t, "1mo", m.Config.Range.Advance) +} + +func TestRangeConfig_Valid_AdvanceAndLookback(t *testing.T) { + m, err := addModelFromFrontmatter(t, + "mode: incremental\nunique_key: id\nupdate_key: updated_at\nrange:\n advance: 7d\n lookback: 2d", + "SELECT id, updated_at FROM raw WHERE {incremental_where_cond}") + require.NoError(t, err) + require.NotNil(t, m.Config.Range) + assert.Equal(t, "7d", m.Config.Range.Advance) + assert.Equal(t, "2d", m.Config.Range.Lookback) + assert.True(t, m.Config.Range.HasLookback()) +} + +func TestRangeConfig_Valid_LookbackOnly(t *testing.T) { + m, err := addModelFromFrontmatter(t, + "mode: incremental\nunique_key: id\nupdate_key: updated_at\nrange:\n lookback: 3h", + "SELECT id, updated_at FROM raw WHERE {incremental_where_cond}") + require.NoError(t, err) + require.NotNil(t, m.Config.Range) + assert.Equal(t, "", m.Config.Range.Advance) + assert.Equal(t, "3h", m.Config.Range.Lookback) +} + +func TestRangeConfig_Invalid_StartWithoutAdvance(t *testing.T) { + _, err := addModelFromFrontmatter(t, + "mode: incremental\nunique_key: id\nupdate_key: updated_at\nrange:\n start: '2020-01-01'", + "SELECT id, updated_at FROM raw WHERE {incremental_where_cond}") + require.Error(t, err) + assert.Contains(t, err.Error(), "range.start requires range.advance") +} + +func TestRangeConfig_Invalid_AdvanceWithoutUpdateKey(t *testing.T) { + _, err := addModelFromFrontmatter(t, + "mode: incremental\nunique_key: id\nrange:\n advance: 7d", + "SELECT id FROM raw WHERE {incremental_where_cond}") + require.Error(t, err) + // incremental mode without update_key fires first + assert.Contains(t, err.Error(), "update_key") +} + +func TestRangeConfig_Invalid_RangeWithoutIncrementalMode(t *testing.T) { + _, err := addModelFromFrontmatter(t, + "mode: view\nrange:\n advance: 7d", + "SELECT id FROM raw") + require.Error(t, err) + assert.Contains(t, err.Error(), "range.* requires mode: incremental") +} + +func TestRangeConfig_Invalid_BadAdvanceDuration(t *testing.T) { + _, err := addModelFromFrontmatter(t, + "mode: incremental\nunique_key: id\nupdate_key: updated_at\nrange:\n advance: seven_days", + "SELECT id, updated_at FROM raw WHERE {incremental_where_cond}") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid range.advance") +} + +func TestRangeConfig_Invalid_BadLookbackDuration(t *testing.T) { + _, err := addModelFromFrontmatter(t, + "mode: incremental\nunique_key: id\nupdate_key: updated_at\nrange:\n lookback: later", + "SELECT id, updated_at FROM raw WHERE {incremental_where_cond}") + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid range.lookback") +} + +func TestIncrementalWithoutUpdateKey(t *testing.T) { + _, err := addModelFromFrontmatter(t, + "mode: incremental\nunique_key: id", + "SELECT id FROM raw") + require.Error(t, err) + assert.Contains(t, err.Error(), "mode 'incremental' requires update_key") +} + +func TestParseBuildDuration(t *testing.T) { + // All valid units should parse without error. + cases := map[string]time.Duration{ + "500ms": 500 * time.Millisecond, + "30s": 30 * time.Second, + "15m": 15 * time.Minute, + "3h": 3 * time.Hour, + "2d": 2 * 24 * time.Hour, + "1w": 7 * 24 * time.Hour, + "1mo": 30 * 24 * time.Hour, + "1y": 365 * 24 * time.Hour, + } + for input, expected := range cases { + got, err := parseBuildDuration(input) + require.NoError(t, err, "input=%s", input) + assert.Equal(t, expected, got, "input=%s", input) + } + + // Invalid inputs + for _, bad := range []string{"", "foo", "5", "d5", "5x", "-1d"} { + _, err := parseBuildDuration(bad) + assert.Error(t, err, "expected error for input=%q", bad) + } +} + +func TestValidateModel_RangeWithDbtStyleErrors(t *testing.T) { + // A model using is_incremental() + range.advance errors at load. + _, err := addModelFromFrontmatter(t, + "mode: incremental\nunique_key: id\nupdate_key: updated_at\nrange:\n advance: 7d", + `SELECT id, updated_at FROM raw +{% if is_incremental() %}WHERE updated_at > (SELECT MAX(updated_at) FROM {{ this }}){% endif %}`) + require.Error(t, err) + assert.Contains(t, err.Error(), "range.* requires {incremental_where_cond}") +} + +func TestValidateModel_MixedStyleErrors(t *testing.T) { + // A model containing both is_incremental() AND {incremental_where_cond} errors. + _, err := addModelFromFrontmatter(t, + "mode: incremental\nunique_key: id\nupdate_key: updated_at", + `SELECT id, updated_at FROM raw WHERE {incremental_where_cond} +{% if is_incremental() %}AND updated_at > (SELECT MAX(updated_at) FROM {{ this }}){% endif %}`) + require.Error(t, err) + assert.Contains(t, err.Error(), "cannot mix is_incremental() and {incremental_where_cond}") +} + +// ============================================================================= +// Expanded BuildDefaults — tests for layered defaults, additive merging, +// schema overrides, and enabled:false DAG skip. +// ============================================================================= + +func TestMergeConfigsAdditiveTagsAndHooks(t *testing.T) { + parent := &BuildConfig{ + Defaults: BuildDefaults{ + Tags: []string{"a", "b"}, + Hooks: sling.HookMap{ + Start: []any{map[string]any{"type": "log", "message": "parent_start"}}, + End: []any{map[string]any{"type": "log", "message": "parent_end"}}, + }, + }, + } + child := &BuildConfig{ + Defaults: BuildDefaults{ + Tags: []string{"b", "c"}, // "b" duplicates parent — should be deduped + Hooks: sling.HookMap{ + Start: []any{map[string]any{"type": "log", "message": "child_start"}}, + }, + }, + } + + merged := mergeConfigs(parent, child) + + // Tags: union + dedupe, parent order preserved first + assert.Equal(t, []string{"a", "b", "c"}, merged.Defaults.Tags) + + // Hooks: parent first, child appended + require.Len(t, merged.Defaults.Hooks.Start, 2) + assert.Equal(t, "parent_start", merged.Defaults.Hooks.Start[0].(map[string]any)["message"]) + assert.Equal(t, "child_start", merged.Defaults.Hooks.Start[1].(map[string]any)["message"]) + + // End only had parent entry — should survive + require.Len(t, merged.Defaults.Hooks.End, 1) + assert.Equal(t, "parent_end", merged.Defaults.Hooks.End[0].(map[string]any)["message"]) +} + +func TestMergeHookMapsHandlesEmpty(t *testing.T) { + // Both empty + result := mergeHookMaps(sling.HookMap{}, sling.HookMap{}) + assert.True(t, result.IsEmpty()) + + // Parent empty, child has entries + child := sling.HookMap{Start: []any{"a"}} + result = mergeHookMaps(sling.HookMap{}, child) + assert.Equal(t, []any{"a"}, result.Start) + + // Parent has entries, child empty + parent := sling.HookMap{End: []any{"b"}} + result = mergeHookMaps(parent, sling.HookMap{}) + assert.Equal(t, []any{"b"}, result.End) +} + +// buildDefaultsProject returns a fresh project rooted at a tempdir with the given +// root + child configs and models. Each model value is raw SQL content (with or +// without frontmatter). Returns a loaded project in prod mode. +func buildDefaultsProject(t *testing.T, rootCfg string, childCfgs map[string]string, models map[string]string, seeds map[string]string) *BuildProject { + t.Helper() + dir := t.TempDir() + + if rootCfg != "" { + require.NoError(t, os.WriteFile(filepath.Join(dir, "sling_build.yml"), []byte(rootCfg), 0644)) + } + for subDir, cfg := range childCfgs { + subPath := filepath.Join(dir, subDir) + require.NoError(t, os.MkdirAll(subPath, 0755)) + require.NoError(t, os.WriteFile(filepath.Join(subPath, "sling_build.yml"), []byte(cfg), 0644)) + } + for relPath, content := range models { + fullPath := filepath.Join(dir, relPath) + require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0755)) + require.NoError(t, os.WriteFile(fullPath, []byte(content), 0644)) + } + for relPath, content := range seeds { + fullPath := filepath.Join(dir, relPath) + require.NoError(t, os.MkdirAll(filepath.Dir(fullPath), 0755)) + require.NoError(t, os.WriteFile(fullPath, []byte(content), 0644)) + } + + // Recursive so child sling_build.yml defaults are discovered + project, err := LoadProject(dir, BuildOptions{Recursive: true}) + require.NoError(t, err) + return project +} + +func TestEffectiveDefaultsNested(t *testing.T) { + project := buildDefaultsProject(t, + `target: POSTGRES +defaults: + schema: raw +`, + map[string]string{ + "staging": `defaults: + mode: truncate +`, + }, + map[string]string{ + "staging/orders.sql": "SELECT 1 as id", + "marts/fct.sql": "SELECT 2 as id", + }, + nil, + ) + + orders := project.Models["orders"] + require.NotNil(t, orders) + assert.Equal(t, "truncate", orders.Config.Mode, "child defaults.mode should apply") + assert.Equal(t, "raw", orders.Schema, "root defaults.schema should override folder") + // In prod mode, a single folder becomes the schema, so prefix is empty. + assert.Equal(t, "raw.orders", orders.FullTableName) + + fct := project.Models["fct"] + require.NotNil(t, fct) + assert.Equal(t, "", fct.Config.Mode, "marts has no child defaults.mode") + assert.Equal(t, "raw", fct.Schema, "root defaults.schema applies everywhere") + assert.Equal(t, "raw.fct", fct.FullTableName) +} + +func TestDefaultsSchemaOverride(t *testing.T) { + project := buildDefaultsProject(t, + `target: POSTGRES +defaults: + schema: analytics +`, + nil, + map[string]string{ + "staging/orders.sql": "SELECT 1 as id", + }, + nil, + ) + + orders := project.Models["orders"] + require.NotNil(t, orders) + assert.Equal(t, "analytics", orders.Schema) + assert.Equal(t, "analytics.orders", orders.FullTableName) + + // ProdFullTableName is intentionally not rewritten — used for ref() matching. + assert.Equal(t, "staging.orders", orders.ProdFullTableName) +} + +func TestDefaultsEnabledFalseSkippedInDAG(t *testing.T) { + project := buildDefaultsProject(t, + `target: POSTGRES +`, + nil, + map[string]string{ + "staging/stg_active.sql": "SELECT 1", + "staging/stg_disabled.sql": "/**\nenabled: false\n**/\nSELECT 2", + }, + nil, + ) + + // Both models are loaded into project.Models + assert.Contains(t, project.Models, "stg_active") + assert.Contains(t, project.Models, "stg_disabled") + + // Confirm enabled state was parsed from frontmatter + disabled := project.Models["stg_disabled"] + require.NotNil(t, disabled.Config.Enabled) + assert.False(t, *disabled.Config.Enabled) + + // Build the DAG — disabled model should NOT be a node. + // No SQL refs between models, so DependsOn is empty and no compile needed. + dag, err := BuildDAG(project) + require.NoError(t, err) + + assert.Contains(t, dag.Nodes, "stg_active") + assert.NotContains(t, dag.Nodes, "stg_disabled") +} + +func TestDefaultsEnabledFalseViaDefaults(t *testing.T) { + // Enabled:false set via defaults (not frontmatter) on a child folder. + project := buildDefaultsProject(t, + `target: POSTGRES +`, + map[string]string{ + "archive": `defaults: + enabled: false +`, + }, + map[string]string{ + "staging/stg_active.sql": "SELECT 1", + "archive/old_model.sql": "SELECT 2", + }, + nil, + ) + + oldModel := project.Models["old_model"] + require.NotNil(t, oldModel) + require.NotNil(t, oldModel.Config.Enabled) + assert.False(t, *oldModel.Config.Enabled) + + dag, err := BuildDAG(project) + require.NoError(t, err) + assert.Contains(t, dag.Nodes, "stg_active") + assert.NotContains(t, dag.Nodes, "old_model") +} + +func TestFrontmatterSchemaOverridesFolder(t *testing.T) { + // Regression test for the latent bug fixed alongside defaults.schema: + // a frontmatter `schema:` must override the folder-derived Schema + FullTableName. + project := buildDefaultsProject(t, + `target: POSTGRES +`, + nil, + map[string]string{ + "staging/orders.sql": "/**\nschema: custom\n**/\nSELECT 1", + }, + nil, + ) + + orders := project.Models["orders"] + require.NotNil(t, orders) + assert.Equal(t, "custom", orders.Schema) + assert.Equal(t, "custom.orders", orders.FullTableName) + // Prod name is untouched — ref() matches against this. + assert.Equal(t, "staging.orders", orders.ProdFullTableName) +} + +func TestDefaultsHooksMergedWithFrontmatter(t *testing.T) { + project := buildDefaultsProject(t, + `target: POSTGRES +defaults: + hooks: + start: + - type: log + message: default_start +`, + nil, + map[string]string{ + "staging/orders.sql": `/** +hooks: + start: + - type: log + message: fm_start +**/ +SELECT 1`, + }, + nil, + ) + + orders := project.Models["orders"] + require.NotNil(t, orders) + require.Len(t, orders.Config.Hooks.Start, 2, "defaults + frontmatter hooks should merge") + assert.Equal(t, "default_start", orders.Config.Hooks.Start[0].(map[string]any)["message"]) + assert.Equal(t, "fm_start", orders.Config.Hooks.Start[1].(map[string]any)["message"]) +} + +func TestDefaultsTagsMergedWithFrontmatter(t *testing.T) { + project := buildDefaultsProject(t, + `target: POSTGRES +defaults: + tags: [a, b] +`, + nil, + map[string]string{ + "staging/orders.sql": `/** +tags: [b, c] +**/ +SELECT 1`, + }, + nil, + ) + + orders := project.Models["orders"] + require.NotNil(t, orders) + assert.Equal(t, []string{"a", "b", "c"}, orders.Config.Tags) +} + +func TestDefaultsAppliedToSeedSchema(t *testing.T) { + project := buildDefaultsProject(t, + `target: POSTGRES +defaults: + schema: raw +`, + nil, + nil, + map[string]string{ + "staging/country_codes.csv": "code,name\nUS,United States\n", + }, + ) + + seed := project.Seeds["country_codes"] + require.NotNil(t, seed) + assert.Equal(t, "raw", seed.Schema) + assert.Equal(t, "raw.country_codes", seed.FullTableName) +} + +func TestDefaultsUniqueKeyAndMergeStrategy(t *testing.T) { + project := buildDefaultsProject(t, + `target: POSTGRES +defaults: + mode: incremental + unique_key: id + update_key: updated_at + merge_strategy: delete+insert +`, + nil, + map[string]string{ + "staging/orders.sql": "SELECT id, updated_at FROM raw WHERE {incremental_where_cond}", + }, + nil, + ) + + orders := project.Models["orders"] + require.NotNil(t, orders) + assert.Equal(t, "incremental", orders.Config.Mode) + assert.Equal(t, "id", orders.Config.UniqueKey) + assert.Equal(t, "updated_at", orders.Config.UpdateKey) + assert.Equal(t, "delete+insert", orders.Config.MergeStrategy) +} diff --git a/core/sling/build/selector.go b/core/sling/build/selector.go new file mode 100644 index 000000000..e142bd896 --- /dev/null +++ b/core/sling/build/selector.go @@ -0,0 +1,666 @@ +package build + +import ( + "path/filepath" + "sort" + "strconv" + "strings" + + "github.com/flarco/g" + "github.com/gobwas/glob" + "github.com/samber/lo" +) + +// ============================================================================= +// DAG +// ============================================================================= + +// DAG represents a directed acyclic graph of models and seeds. +type DAG struct { + Nodes map[string]*DAGNode + Order []string // topological sort result +} + +// DAGNode represents a node in the dependency graph. +type DAGNode struct { + Name string + Model *Model // nil for seeds + Seed *Seed // nil for models + Dependencies []string + Dependents []string + Depth int +} + +// BuildDAG constructs a dependency graph from a project's models and seeds. +// Models must have been compiled first (DependsOn populated). +func BuildDAG(project *BuildProject) (*DAG, error) { + dag := &DAG{ + Nodes: make(map[string]*DAGNode), + } + + // Add seed nodes (depth 0, no dependencies) + for name, seed := range project.Seeds { + dag.Nodes[name] = &DAGNode{ + Name: name, + Seed: seed, + } + } + + // isDisabled returns true for models that are excluded from DAG execution. + isDisabled := func(name string) bool { + m, ok := project.Models[name] + if !ok { + return false + } + return m.Config.Enabled != nil && !*m.Config.Enabled + } + + // Add model nodes with dependencies + for name, model := range project.Models { + // Skip disabled models — they stay in project.Models (for compile/list/ + // template) but are excluded from DAG-based execution. Dependents of a + // disabled model silently drop the edge (matching the existing unresolved + // -ref behavior) and will fail later with a missing-table error if they + // actually query it. + if isDisabled(name) { + continue + } + + // Filter dependencies to only include known models/seeds. + // Disabled models are silently dropped so the DAG stays consistent. + deps := make([]string, 0) + for _, dep := range model.DependsOn { + if _, ok := project.Models[dep]; ok { + if isDisabled(dep) { + continue + } + deps = append(deps, dep) + } else if _, ok := project.Seeds[dep]; ok { + deps = append(deps, dep) + } + // Unknown deps (from src() or external tables) are silently skipped + } + + dag.Nodes[name] = &DAGNode{ + Name: name, + Model: model, + Dependencies: deps, + } + } + + // Build reverse edges (dependents) + for name, node := range dag.Nodes { + for _, dep := range node.Dependencies { + if depNode, ok := dag.Nodes[dep]; ok { + depNode.Dependents = append(depNode.Dependents, name) + } + } + } + + // Topological sort + order, err := dag.TopologicalSort() + if err != nil { + return nil, err + } + dag.Order = order + + // Compute depths + dag.computeDepths() + + return dag, nil +} + +// TopologicalSort performs Kahn's algorithm to produce a topological ordering. +// Returns an error if a cycle is detected. +func (dag *DAG) TopologicalSort() ([]string, error) { + // Count incoming edges for each node + inDegree := make(map[string]int) + for name := range dag.Nodes { + inDegree[name] = len(dag.Nodes[name].Dependencies) + } + + // Find all nodes with in-degree 0 + queue := make([]string, 0) + for name, degree := range inDegree { + if degree == 0 { + queue = append(queue, name) + } + } + // Sort for deterministic output + sort.Strings(queue) + + var order []string + for len(queue) > 0 { + // Pop from queue + name := queue[0] + queue = queue[1:] + order = append(order, name) + + // Reduce in-degree of dependents + node := dag.Nodes[name] + nextBatch := make([]string, 0) + for _, dependent := range node.Dependents { + inDegree[dependent]-- + if inDegree[dependent] == 0 { + nextBatch = append(nextBatch, dependent) + } + } + // Sort for deterministic ordering within same level + sort.Strings(nextBatch) + queue = append(queue, nextBatch...) + } + + if len(order) != len(dag.Nodes) { + // Cycle detected + cycles := dag.DetectCycles() + cycleStrs := make([]string, 0, len(cycles)) + for _, cycle := range cycles { + cycleStrs = append(cycleStrs, strings.Join(cycle, " -> ")) + } + return nil, g.Error("cycle detected in model dependencies: %s", strings.Join(cycleStrs, "; ")) + } + + return order, nil +} + +// DetectCycles finds cycles in the graph using DFS. +func (dag *DAG) DetectCycles() [][]string { + var cycles [][]string + visited := make(map[string]int) // 0=unvisited, 1=in-progress, 2=done + path := make([]string, 0) + + var dfs func(name string) + dfs = func(name string) { + if visited[name] == 2 { + return + } + if visited[name] == 1 { + // Found a cycle — extract it from path + cycleStart := -1 + for i, p := range path { + if p == name { + cycleStart = i + break + } + } + if cycleStart >= 0 { + cycle := make([]string, 0) + cycle = append(cycle, path[cycleStart:]...) + cycle = append(cycle, name) // close the cycle + cycles = append(cycles, cycle) + } + return + } + + visited[name] = 1 + path = append(path, name) + + node := dag.Nodes[name] + if node != nil { + for _, dep := range node.Dependencies { + dfs(dep) + } + } + + path = path[:len(path)-1] + visited[name] = 2 + } + + // Sort names for deterministic output + names := make([]string, 0, len(dag.Nodes)) + for name := range dag.Nodes { + names = append(names, name) + } + sort.Strings(names) + + for _, name := range names { + dfs(name) + } + + return cycles +} + +// GetUpstream returns all transitive upstream dependencies of the given node. +func (dag *DAG) GetUpstream(name string) []string { + visited := make(map[string]bool) + var result []string + + var walk func(n string) + walk = func(n string) { + node, ok := dag.Nodes[n] + if !ok { + return + } + for _, dep := range node.Dependencies { + if !visited[dep] { + visited[dep] = true + result = append(result, dep) + walk(dep) + } + } + } + + walk(name) + + // Return in topological order + orderMap := make(map[string]int) + for i, n := range dag.Order { + orderMap[n] = i + } + sort.Slice(result, func(i, j int) bool { + return orderMap[result[i]] < orderMap[result[j]] + }) + + return result +} + +// GetUpstreamN returns upstream dependencies up to N degrees from the given node. +func (dag *DAG) GetUpstreamN(name string, n int) []string { + visited := make(map[string]bool) + var result []string + + current := []string{name} + for depth := 0; depth < n && len(current) > 0; depth++ { + var next []string + for _, nodeName := range current { + node, ok := dag.Nodes[nodeName] + if !ok { + continue + } + for _, dep := range node.Dependencies { + if !visited[dep] { + visited[dep] = true + result = append(result, dep) + next = append(next, dep) + } + } + } + current = next + } + + // Return in topological order + orderMap := make(map[string]int) + for i, n := range dag.Order { + orderMap[n] = i + } + sort.Slice(result, func(i, j int) bool { + return orderMap[result[i]] < orderMap[result[j]] + }) + + return result +} + +// GetDownstreamN returns downstream dependents up to N degrees from the given node. +func (dag *DAG) GetDownstreamN(name string, n int) []string { + visited := make(map[string]bool) + var result []string + + current := []string{name} + for depth := 0; depth < n && len(current) > 0; depth++ { + var next []string + for _, nodeName := range current { + node, ok := dag.Nodes[nodeName] + if !ok { + continue + } + for _, dep := range node.Dependents { + if !visited[dep] { + visited[dep] = true + result = append(result, dep) + next = append(next, dep) + } + } + } + current = next + } + return result +} + +// GetDownstream returns all transitive downstream dependents of the given node. +func (dag *DAG) GetDownstream(name string) []string { + visited := make(map[string]bool) + var result []string + + var walk func(n string) + walk = func(n string) { + node, ok := dag.Nodes[n] + if !ok { + return + } + for _, dep := range node.Dependents { + if !visited[dep] { + visited[dep] = true + result = append(result, dep) + walk(dep) + } + } + } + + walk(name) + return result +} + +// GetExecutionLevels groups nodes by depth for parallel execution. +// Each level contains nodes that can be executed concurrently. +func (dag *DAG) GetExecutionLevels() [][]string { + if len(dag.Order) == 0 { + return nil + } + + maxDepth := 0 + for _, node := range dag.Nodes { + if node.Depth > maxDepth { + maxDepth = node.Depth + } + } + + levels := make([][]string, maxDepth+1) + for _, name := range dag.Order { + node := dag.Nodes[name] + levels[node.Depth] = append(levels[node.Depth], name) + } + + return levels +} + +// computeDepths calculates the depth of each node based on dependencies. +func (dag *DAG) computeDepths() { + for _, name := range dag.Order { + node := dag.Nodes[name] + maxDepth := -1 + for _, dep := range node.Dependencies { + if depNode, ok := dag.Nodes[dep]; ok { + if depNode.Depth > maxDepth { + maxDepth = depNode.Depth + } + } + } + node.Depth = maxDepth + 1 + } +} + +// ============================================================================= +// Selector +// ============================================================================= + +// Selector filters DAG nodes based on include/exclude patterns. +type Selector struct { + Includes []string + Excludes []string +} + +// NewSelector creates a new selector from include and exclude patterns. +func NewSelector(includes, excludes []string) *Selector { + return &Selector{ + Includes: includes, + Excludes: excludes, + } +} + +// Apply filters DAG nodes based on the selector patterns, returning names in DAG order. +func (s *Selector) Apply(dag *DAG) ([]string, error) { + selected := make(map[string]bool) + + if len(s.Includes) == 0 { + // No includes = select all + for _, name := range dag.Order { + selected[name] = true + } + } else { + // Apply each include pattern + for _, pattern := range s.Includes { + matched, err := matchPattern(pattern, dag) + if err != nil { + return nil, err + } + for _, name := range matched { + selected[name] = true + } + } + } + + // Apply excludes + for _, pattern := range s.Excludes { + matched, err := matchPattern(pattern, dag) + if err != nil { + return nil, err + } + for _, name := range matched { + delete(selected, name) + } + } + + // Return in DAG order + result := make([]string, 0, len(selected)) + for _, name := range dag.Order { + if selected[name] { + result = append(result, name) + } + } + + return result, nil +} + +// matchPattern matches a single pattern against DAG nodes. +// Supported forms: +// - model_name exact/glob match +// - +model all upstream + model (inclusive) +// - model+ model + all downstream (inclusive) +// - +model+ all upstream + model + all downstream +// - N+model N degrees upstream + model +// - model+N model + N degrees downstream +// - modelA-modelB slice: all nodes between A and B (inclusive) +// - tag:xxx match by tag +// - path/pattern match by file path +func matchPattern(pattern string, dag *DAG) ([]string, error) { + // Tag selector: tag:xxx (check early, before + parsing) + if strings.HasPrefix(pattern, "tag:") { + tag := strings.TrimPrefix(pattern, "tag:") + return matchByTag(tag, dag), nil + } + + // Graph traversal selectors (contains "+") + if strings.Contains(pattern, "+") { + return matchGraphSelector(pattern, dag) + } + + // Slice selector: modelA-modelB (all nodes between A and B inclusive) + // Only match if both sides resolve to known nodes (avoid matching glob patterns or paths with hyphens) + if idx := strings.Index(pattern, "-"); idx > 0 && idx < len(pattern)-1 && !strings.Contains(pattern, "/") { + left := pattern[:idx] + right := pattern[idx+1:] + _, leftOk := dag.Nodes[left] + _, rightOk := dag.Nodes[right] + if leftOk && rightOk { + return matchSlice(left, right, dag) + } + } + + // Path selector: contains "/" -> match against relative file paths + if strings.Contains(pattern, "/") { + return matchByPath(pattern, dag) + } + + // Glob selector: match against node names + return matchByGlob(pattern, dag) +} + +// matchGraphSelector handles all "+" based selectors: +// +// +model, model+, +model+, N+model, model+N +// +// The model portion can be an exact name or a glob pattern (e.g. stg_*, *_orders). +func matchGraphSelector(pattern string, dag *DAG) ([]string, error) { + hasPrefix := strings.HasPrefix(pattern, "+") + hasSuffix := strings.HasSuffix(pattern, "+") + + // Pure prefix/suffix forms: +model, model+, +model+ + if hasPrefix || hasSuffix { + modelPattern := strings.Trim(pattern, "+") + + nodes, err := resolveNodes(modelPattern, dag, pattern) + if err != nil { + return nil, err + } + + selected := make(map[string]bool) + for _, modelName := range nodes { + selected[modelName] = true + if hasPrefix { + for _, name := range dag.GetUpstream(modelName) { + selected[name] = true + } + } + if hasSuffix { + for _, name := range dag.GetDownstream(modelName) { + selected[name] = true + } + } + } + + return dagOrder(selected, dag), nil + } + + // Interior "+" — could be: N+model, model+N, or modelA+modelB + parts := strings.SplitN(pattern, "+", 2) + left, right := parts[0], parts[1] + + // N+model (degree upstream) + if n, err := strconv.Atoi(left); err == nil && n >= 0 { + nodes, err := resolveNodes(right, dag, pattern) + if err != nil { + return nil, err + } + selected := make(map[string]bool) + for _, modelName := range nodes { + selected[modelName] = true + for _, name := range dag.GetUpstreamN(modelName, n) { + selected[name] = true + } + } + return dagOrder(selected, dag), nil + } + + // model+N (degree downstream) + if n, err := strconv.Atoi(right); err == nil && n >= 0 { + nodes, err := resolveNodes(left, dag, pattern) + if err != nil { + return nil, err + } + selected := make(map[string]bool) + for _, modelName := range nodes { + selected[modelName] = true + for _, name := range dag.GetDownstreamN(modelName, n) { + selected[name] = true + } + } + return dagOrder(selected, dag), nil + } + + // Unrecognized + pattern — left and right are not numbers, fall through to error + return nil, g.Error("selector '%s': invalid selector pattern", pattern) +} + +// matchSlice returns all nodes between modelA and modelB (inclusive). +// This is the intersection of downstream(A) and upstream(B), plus A and B. +func matchSlice(left, right string, dag *DAG) ([]string, error) { + // Slice = downstream of A ∩ upstream of B, plus A and B themselves + downA := dag.GetDownstream(left) + upB := dag.GetUpstream(right) + downSet := make(map[string]bool) + for _, name := range downA { + downSet[name] = true + } + downSet[left] = true + + selected := make(map[string]bool) + selected[left] = true + selected[right] = true + for _, name := range upB { + if downSet[name] { + selected[name] = true + } + } + + return dagOrder(selected, dag), nil +} + +// dagOrder returns the selected node names in DAG topological order. +func dagOrder(selected map[string]bool, dag *DAG) []string { + var result []string + for _, name := range dag.Order { + if selected[name] { + result = append(result, name) + } + } + return result +} + +// isGlob returns true if the string contains glob metacharacters. +func isGlob(s string) bool { + return strings.ContainsAny(s, "*?[{") +} + +// resolveNodes resolves a name-or-glob to matching DAG node names. +// For exact names, returns the single name or errors if not found. +// For glob patterns, returns all matches (empty slice if none match). +func resolveNodes(nameOrGlob string, dag *DAG, selectorForError string) ([]string, error) { + if !isGlob(nameOrGlob) { + if _, ok := dag.Nodes[nameOrGlob]; !ok { + return nil, g.Error("selector '%s': model '%s' not found", selectorForError, nameOrGlob) + } + return []string{nameOrGlob}, nil + } + return matchByGlob(nameOrGlob, dag) +} + +// matchByGlob matches node names using a glob pattern. +func matchByGlob(pattern string, dag *DAG) ([]string, error) { + g, err := glob.Compile(pattern) + if err != nil { + return nil, err + } + + var result []string + for _, name := range dag.Order { + if g.Match(name) { + result = append(result, name) + } + } + return result, nil +} + +// matchByTag matches nodes whose model Config.Tags contain the given tag. +func matchByTag(tag string, dag *DAG) []string { + var result []string + for _, name := range dag.Order { + node := dag.Nodes[name] + if node.Model != nil && lo.Contains(node.Model.Config.Tags, tag) { + result = append(result, name) + } + } + return result +} + +// matchByPath matches nodes whose relative file path matches a glob pattern. +func matchByPath(pattern string, dag *DAG) ([]string, error) { + g, err := glob.Compile(pattern) + if err != nil { + return nil, err + } + + var result []string + for _, name := range dag.Order { + node := dag.Nodes[name] + var relPath string + if node.Model != nil { + relPath = filepath.ToSlash(node.Model.RelPath) + } else if node.Seed != nil { + relPath = filepath.ToSlash(node.Seed.RelPath) + } + if relPath != "" && g.Match(relPath) { + result = append(result, name) + } + } + return result, nil +} diff --git a/core/sling/build/selector_test.go b/core/sling/build/selector_test.go new file mode 100644 index 000000000..07bb989fd --- /dev/null +++ b/core/sling/build/selector_test.go @@ -0,0 +1,861 @@ +package build + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func newTestDAGProject() *BuildProject { + return &BuildProject{ + Models: map[string]*Model{ + "stg_orders": { + Name: "stg_orders", + FullTableName: "staging.stg_orders", + RelPath: "staging/stg_orders.sql", + Config: ModelConfig{Tags: []string{"daily", "staging"}}, + }, + "stg_customers": { + Name: "stg_customers", + FullTableName: "staging.stg_customers", + RelPath: "staging/stg_customers.sql", + Config: ModelConfig{Tags: []string{"daily", "staging"}}, + }, + "dim_customers": { + Name: "dim_customers", + FullTableName: "marts.core_dim_customers", + RelPath: "marts/core/dim_customers.sql", + DependsOn: []string{"stg_customers"}, + Config: ModelConfig{Tags: []string{"weekly"}}, + }, + "fct_orders": { + Name: "fct_orders", + FullTableName: "marts.core_fct_orders", + RelPath: "marts/core/fct_orders.sql", + DependsOn: []string{"stg_orders"}, + Config: ModelConfig{Tags: []string{"daily"}}, + }, + "revenue": { + Name: "revenue", + FullTableName: "marts.finance_revenue", + RelPath: "marts/finance/revenue.sql", + DependsOn: []string{"fct_orders"}, + }, + }, + Seeds: map[string]*Seed{ + "country_codes": { + Name: "country_codes", + FullTableName: "staging.country_codes", + RelPath: "staging/country_codes.csv", + }, + }, + } +} + +func TestSelectorGlob(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + sel := NewSelector([]string{"stg_*"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 2) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "stg_customers") +} + +func TestSelectorTag(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + sel := NewSelector([]string{"tag:daily"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 3) // stg_orders, stg_customers, fct_orders + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "stg_customers") + assert.Contains(t, result, "fct_orders") + assert.NotContains(t, result, "dim_customers") // weekly tag + assert.NotContains(t, result, "country_codes") // seed, no tags +} + +func TestSelectorUpstream(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + sel := NewSelector([]string{"+fct_orders"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + // fct_orders + stg_orders (upstream) + assert.Len(t, result, 2) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "fct_orders") + + // Should be in topological order (stg_orders before fct_orders) + stgIdx, fctIdx := -1, -1 + for i, name := range result { + if name == "stg_orders" { + stgIdx = i + } + if name == "fct_orders" { + fctIdx = i + } + } + assert.Less(t, stgIdx, fctIdx) +} + +func TestSelectorUpstreamDeep(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + sel := NewSelector([]string{"+revenue"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + // revenue + fct_orders + stg_orders (transitive upstream) + assert.Len(t, result, 3) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "fct_orders") + assert.Contains(t, result, "revenue") +} + +func TestSelectorPath(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + sel := NewSelector([]string{"staging/*"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + // Should match staging/stg_orders.sql, staging/stg_customers.sql, staging/country_codes.csv + assert.Len(t, result, 3) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "stg_customers") + assert.Contains(t, result, "country_codes") +} + +func TestSelectorPathNested(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + sel := NewSelector([]string{"marts/core/*"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 2) + assert.Contains(t, result, "dim_customers") + assert.Contains(t, result, "fct_orders") +} + +func TestSelectorExclude(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // Select all, then exclude stg_* + sel := NewSelector(nil, []string{"stg_*"}) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.NotContains(t, result, "stg_orders") + assert.NotContains(t, result, "stg_customers") + assert.Contains(t, result, "dim_customers") + assert.Contains(t, result, "fct_orders") + assert.Contains(t, result, "revenue") + assert.Contains(t, result, "country_codes") +} + +func TestSelectorNoFilter(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + sel := NewSelector(nil, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + // All nodes in DAG order + assert.Len(t, result, 6) + assert.Equal(t, dag.Order, result) +} + +func TestSelectorCombined(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // Select tag:daily, exclude stg_* + sel := NewSelector([]string{"tag:daily"}, []string{"stg_*"}) + result, err := sel.Apply(dag) + require.NoError(t, err) + + // Only fct_orders should remain (stg_ models are daily but excluded) + assert.Len(t, result, 1) + assert.Contains(t, result, "fct_orders") +} + +func TestSelectorDownstream(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + sel := NewSelector([]string{"stg_orders+"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + // stg_orders + fct_orders + revenue (downstream) + assert.Len(t, result, 3) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "fct_orders") + assert.Contains(t, result, "revenue") + + // Should be in topological order + stgIdx, fctIdx, revIdx := -1, -1, -1 + for i, name := range result { + switch name { + case "stg_orders": + stgIdx = i + case "fct_orders": + fctIdx = i + case "revenue": + revIdx = i + } + } + assert.Less(t, stgIdx, fctIdx) + assert.Less(t, fctIdx, revIdx) +} + +func TestSelectorDownstreamLeaf(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // revenue has no downstream, so result is just revenue itself + sel := NewSelector([]string{"revenue+"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 1) + assert.Contains(t, result, "revenue") +} + +func TestSelectorUpstreamAndDownstream(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // +fct_orders+ = upstream (stg_orders) + fct_orders + downstream (revenue) + sel := NewSelector([]string{"+fct_orders+"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 3) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "fct_orders") + assert.Contains(t, result, "revenue") +} + +func TestSelectorDownstreamDegree(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // stg_orders+1 = stg_orders + 1 step downstream (fct_orders) + sel := NewSelector([]string{"stg_orders+1"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 2) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "fct_orders") + assert.NotContains(t, result, "revenue") // 2 steps away +} + +func TestSelectorDownstreamDegreeTwo(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // stg_orders+2 = stg_orders + fct_orders + revenue + sel := NewSelector([]string{"stg_orders+2"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 3) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "fct_orders") + assert.Contains(t, result, "revenue") +} + +func TestSelectorUpstreamDegree(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // 1+revenue = revenue + 1 step upstream (fct_orders) + sel := NewSelector([]string{"1+revenue"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 2) + assert.Contains(t, result, "fct_orders") + assert.Contains(t, result, "revenue") + assert.NotContains(t, result, "stg_orders") // 2 steps away +} + +func TestSelectorUpstreamDegreeTwo(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // 2+revenue = revenue + fct_orders + stg_orders + sel := NewSelector([]string{"2+revenue"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 3) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "fct_orders") + assert.Contains(t, result, "revenue") +} + +func TestSelectorDegreeZero(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // 0+revenue = just revenue itself + sel := NewSelector([]string{"0+revenue"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 1) + assert.Contains(t, result, "revenue") + + // revenue+0 = just revenue itself + sel = NewSelector([]string{"revenue+0"}, nil) + result, err = sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 1) + assert.Contains(t, result, "revenue") +} + +func TestSelectorSlice(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // stg_orders-revenue = stg_orders, fct_orders, revenue + sel := NewSelector([]string{"stg_orders-revenue"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 3) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "fct_orders") + assert.Contains(t, result, "revenue") + + // Should be in topological order + stgIdx, fctIdx, revIdx := -1, -1, -1 + for i, name := range result { + switch name { + case "stg_orders": + stgIdx = i + case "fct_orders": + fctIdx = i + case "revenue": + revIdx = i + } + } + assert.Less(t, stgIdx, fctIdx) + assert.Less(t, fctIdx, revIdx) +} + +func TestSelectorSliceAdjacent(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // stg_orders-fct_orders = just those two + sel := NewSelector([]string{"stg_orders-fct_orders"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 2) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "fct_orders") +} + +func TestSelectorSliceDisjoint(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // stg_customers-revenue: no path between them, just the two endpoints + sel := NewSelector([]string{"stg_customers-revenue"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 2) + assert.Contains(t, result, "stg_customers") + assert.Contains(t, result, "revenue") +} + +func TestSelectorDownstreamNotFound(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + sel := NewSelector([]string{"nonexistent+"}, nil) + _, err = sel.Apply(dag) + assert.Error(t, err) + assert.Contains(t, err.Error(), "nonexistent") +} + +func TestSelectorUpstreamNotFound(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + sel := NewSelector([]string{"+nonexistent"}, nil) + _, err = sel.Apply(dag) + assert.Error(t, err) + assert.Contains(t, err.Error(), "nonexistent") +} + +func TestSelectorGlobNoMatch(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + sel := NewSelector([]string{"zzz_*"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Empty(t, result) +} + +func TestSelectorMultipleIncludes(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + sel := NewSelector([]string{"stg_orders", "dim_customers"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 2) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "dim_customers") +} + +func TestSelectorGlobUpstream(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // +stg_* = all stg_ models + their upstream (stg_ models have no upstream) + sel := NewSelector([]string{"+stg_*"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 2) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "stg_customers") +} + +func TestSelectorGlobDownstream(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // stg_*+ = all stg_ models + all their downstream + sel := NewSelector([]string{"stg_*+"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 5) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "stg_customers") + assert.Contains(t, result, "fct_orders") + assert.Contains(t, result, "dim_customers") + assert.Contains(t, result, "revenue") +} + +func TestSelectorGlobBoth(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // +fct_*+ = fct_orders + upstream (stg_orders) + downstream (revenue) + sel := NewSelector([]string{"+fct_*+"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 3) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "fct_orders") + assert.Contains(t, result, "revenue") +} + +func TestSelectorGlobUpstreamDegree(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // 1+*_orders = stg_orders and fct_orders, plus 1-upstream of each + // 1-upstream of stg_orders = nothing; 1-upstream of fct_orders = stg_orders + sel := NewSelector([]string{"1+*_orders"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 2) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "fct_orders") +} + +func TestSelectorGlobDownstreamDegree(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // stg_*+1 = stg_orders and stg_customers, plus 1-downstream of each + // 1-downstream of stg_orders = fct_orders; 1-downstream of stg_customers = dim_customers + sel := NewSelector([]string{"stg_*+1"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Len(t, result, 4) + assert.Contains(t, result, "stg_orders") + assert.Contains(t, result, "stg_customers") + assert.Contains(t, result, "fct_orders") + assert.Contains(t, result, "dim_customers") +} + +func TestSelectorGlobGraphNoMatch(t *testing.T) { + project := newTestDAGProject() + dag, err := BuildDAG(project) + require.NoError(t, err) + + // +zzz_* = glob matches nothing, should return empty (no error) + sel := NewSelector([]string{"+zzz_*"}, nil) + result, err := sel.Apply(dag) + require.NoError(t, err) + + assert.Empty(t, result) + + // zzz_*+ = same behavior + sel = NewSelector([]string{"zzz_*+"}, nil) + result, err = sel.Apply(dag) + require.NoError(t, err) + + assert.Empty(t, result) + + // 1+zzz_* = same behavior + sel = NewSelector([]string{"1+zzz_*"}, nil) + result, err = sel.Apply(dag) + require.NoError(t, err) + + assert.Empty(t, result) + + // zzz_*+1 = same behavior + sel = NewSelector([]string{"zzz_*+1"}, nil) + result, err = sel.Apply(dag) + require.NoError(t, err) + + assert.Empty(t, result) +} + +// ============================================================================= +// DAG tests +// ============================================================================= + +func TestDAGLinearChain(t *testing.T) { + // A -> B -> C + project := &BuildProject{ + Models: map[string]*Model{ + "a": {Name: "a", FullTableName: "public.a"}, + "b": {Name: "b", FullTableName: "public.b", DependsOn: []string{"a"}}, + "c": {Name: "c", FullTableName: "public.c", DependsOn: []string{"b"}}, + }, + Seeds: map[string]*Seed{}, + } + + dag, err := BuildDAG(project) + require.NoError(t, err) + + assert.Equal(t, []string{"a", "b", "c"}, dag.Order) + + // Check depths + assert.Equal(t, 0, dag.Nodes["a"].Depth) + assert.Equal(t, 1, dag.Nodes["b"].Depth) + assert.Equal(t, 2, dag.Nodes["c"].Depth) +} + +func TestDAGDiamond(t *testing.T) { + // A -> B, A -> C, B -> D, C -> D + project := &BuildProject{ + Models: map[string]*Model{ + "a": {Name: "a", FullTableName: "public.a"}, + "b": {Name: "b", FullTableName: "public.b", DependsOn: []string{"a"}}, + "c": {Name: "c", FullTableName: "public.c", DependsOn: []string{"a"}}, + "d": {Name: "d", FullTableName: "public.d", DependsOn: []string{"b", "c"}}, + }, + Seeds: map[string]*Seed{}, + } + + dag, err := BuildDAG(project) + require.NoError(t, err) + + // A must come first, D must come last + assert.Equal(t, "a", dag.Order[0]) + assert.Equal(t, "d", dag.Order[3]) + + // B and C can be in either order but both between A and D + middle := dag.Order[1:3] + assert.Contains(t, middle, "b") + assert.Contains(t, middle, "c") + + // Check depths + assert.Equal(t, 0, dag.Nodes["a"].Depth) + assert.Equal(t, 1, dag.Nodes["b"].Depth) + assert.Equal(t, 1, dag.Nodes["c"].Depth) + assert.Equal(t, 2, dag.Nodes["d"].Depth) +} + +func TestDAGCycleDetection(t *testing.T) { + // A -> B -> C -> A (cycle) + project := &BuildProject{ + Models: map[string]*Model{ + "a": {Name: "a", FullTableName: "public.a", DependsOn: []string{"c"}}, + "b": {Name: "b", FullTableName: "public.b", DependsOn: []string{"a"}}, + "c": {Name: "c", FullTableName: "public.c", DependsOn: []string{"b"}}, + }, + Seeds: map[string]*Seed{}, + } + + _, err := BuildDAG(project) + assert.Error(t, err) + assert.Contains(t, err.Error(), "cycle") +} + +func TestDAGDisconnected(t *testing.T) { + // Two independent chains: A -> B, C -> D + project := &BuildProject{ + Models: map[string]*Model{ + "a": {Name: "a", FullTableName: "public.a"}, + "b": {Name: "b", FullTableName: "public.b", DependsOn: []string{"a"}}, + "c": {Name: "c", FullTableName: "public.c"}, + "d": {Name: "d", FullTableName: "public.d", DependsOn: []string{"c"}}, + }, + Seeds: map[string]*Seed{}, + } + + dag, err := BuildDAG(project) + require.NoError(t, err) + + assert.Len(t, dag.Order, 4) + + // A before B, C before D + aIdx, bIdx, cIdx, dIdx := -1, -1, -1, -1 + for i, name := range dag.Order { + switch name { + case "a": + aIdx = i + case "b": + bIdx = i + case "c": + cIdx = i + case "d": + dIdx = i + } + } + assert.Less(t, aIdx, bIdx) + assert.Less(t, cIdx, dIdx) +} + +func TestDAGSeedsFirst(t *testing.T) { + // Seeds should be at depth 0 + project := &BuildProject{ + Models: map[string]*Model{ + "model_a": {Name: "model_a", FullTableName: "public.model_a", DependsOn: []string{"seed_x"}}, + }, + Seeds: map[string]*Seed{ + "seed_x": {Name: "seed_x", FullTableName: "public.seed_x"}, + }, + } + + dag, err := BuildDAG(project) + require.NoError(t, err) + + assert.Equal(t, 0, dag.Nodes["seed_x"].Depth) + assert.Equal(t, 1, dag.Nodes["model_a"].Depth) + + // Seed should come first in order + assert.Equal(t, "seed_x", dag.Order[0]) + assert.Equal(t, "model_a", dag.Order[1]) +} + +func TestDAGGetUpstream(t *testing.T) { + // seed -> A -> B -> C + project := &BuildProject{ + Models: map[string]*Model{ + "a": {Name: "a", FullTableName: "public.a", DependsOn: []string{"seed"}}, + "b": {Name: "b", FullTableName: "public.b", DependsOn: []string{"a"}}, + "c": {Name: "c", FullTableName: "public.c", DependsOn: []string{"b"}}, + }, + Seeds: map[string]*Seed{ + "seed": {Name: "seed", FullTableName: "public.seed"}, + }, + } + + dag, err := BuildDAG(project) + require.NoError(t, err) + + // Upstream of C should be seed, A, B (in topological order) + upstream := dag.GetUpstream("c") + assert.Equal(t, []string{"seed", "a", "b"}, upstream) + + // Upstream of A should be just seed + upstream = dag.GetUpstream("a") + assert.Equal(t, []string{"seed"}, upstream) + + // Upstream of seed should be empty + upstream = dag.GetUpstream("seed") + assert.Len(t, upstream, 0) +} + +func TestDAGGetDownstream(t *testing.T) { + // seed -> A -> B -> C + project := &BuildProject{ + Models: map[string]*Model{ + "a": {Name: "a", FullTableName: "public.a", DependsOn: []string{"seed"}}, + "b": {Name: "b", FullTableName: "public.b", DependsOn: []string{"a"}}, + "c": {Name: "c", FullTableName: "public.c", DependsOn: []string{"b"}}, + }, + Seeds: map[string]*Seed{ + "seed": {Name: "seed", FullTableName: "public.seed"}, + }, + } + + dag, err := BuildDAG(project) + require.NoError(t, err) + + downstream := dag.GetDownstream("seed") + assert.Len(t, downstream, 3) + assert.Contains(t, downstream, "a") + assert.Contains(t, downstream, "b") + assert.Contains(t, downstream, "c") +} + +func TestDAGExecutionLevels(t *testing.T) { + // seed1, seed2 -> A, B -> C + project := &BuildProject{ + Models: map[string]*Model{ + "a": {Name: "a", FullTableName: "public.a", DependsOn: []string{"seed1"}}, + "b": {Name: "b", FullTableName: "public.b", DependsOn: []string{"seed2"}}, + "c": {Name: "c", FullTableName: "public.c", DependsOn: []string{"a", "b"}}, + }, + Seeds: map[string]*Seed{ + "seed1": {Name: "seed1", FullTableName: "public.seed1"}, + "seed2": {Name: "seed2", FullTableName: "public.seed2"}, + }, + } + + dag, err := BuildDAG(project) + require.NoError(t, err) + + levels := dag.GetExecutionLevels() + require.Len(t, levels, 3) + + // Level 0: seeds + assert.Len(t, levels[0], 2) + assert.Contains(t, levels[0], "seed1") + assert.Contains(t, levels[0], "seed2") + + // Level 1: models depending on seeds + assert.Len(t, levels[1], 2) + assert.Contains(t, levels[1], "a") + assert.Contains(t, levels[1], "b") + + // Level 2: model depending on level 1 + assert.Len(t, levels[2], 1) + assert.Contains(t, levels[2], "c") +} + +func TestDAGEmptyProject(t *testing.T) { + project := &BuildProject{ + Models: map[string]*Model{}, + Seeds: map[string]*Seed{}, + } + + dag, err := BuildDAG(project) + require.NoError(t, err) + assert.Empty(t, dag.Order) + assert.Empty(t, dag.Nodes) + + levels := dag.GetExecutionLevels() + assert.Nil(t, levels) +} + +func TestDAGSingleNode(t *testing.T) { + project := &BuildProject{ + Models: map[string]*Model{ + "only_model": {Name: "only_model", FullTableName: "public.only_model"}, + }, + Seeds: map[string]*Seed{}, + } + + dag, err := BuildDAG(project) + require.NoError(t, err) + + assert.Equal(t, []string{"only_model"}, dag.Order) + assert.Equal(t, 0, dag.Nodes["only_model"].Depth) +} + +func TestDAGExternalDependenciesIgnored(t *testing.T) { + // Model depends on something not in the project (external source) + project := &BuildProject{ + Models: map[string]*Model{ + "a": {Name: "a", FullTableName: "public.a", DependsOn: []string{"external_table"}}, + }, + Seeds: map[string]*Seed{}, + } + + dag, err := BuildDAG(project) + require.NoError(t, err) + + assert.Equal(t, []string{"a"}, dag.Order) + assert.Empty(t, dag.Nodes["a"].Dependencies) // external deps filtered out +} + +func TestDAGDependentsPopulated(t *testing.T) { + project := &BuildProject{ + Models: map[string]*Model{ + "a": {Name: "a", FullTableName: "public.a"}, + "b": {Name: "b", FullTableName: "public.b", DependsOn: []string{"a"}}, + "c": {Name: "c", FullTableName: "public.c", DependsOn: []string{"a"}}, + }, + Seeds: map[string]*Seed{}, + } + + dag, err := BuildDAG(project) + require.NoError(t, err) + + // A should have B and C as dependents + dependents := dag.Nodes["a"].Dependents + assert.Len(t, dependents, 2) + assert.Contains(t, dependents, "b") + assert.Contains(t, dependents, "c") +} diff --git a/core/sling/build/template.go b/core/sling/build/template.go new file mode 100644 index 000000000..02ed1dbff --- /dev/null +++ b/core/sling/build/template.go @@ -0,0 +1,896 @@ +package build + +import ( + "os" + "path/filepath" + "regexp" + "sort" + "strings" + + "github.com/flarco/g" + "github.com/nikolalohinski/gonja/v2/builtins" + "github.com/nikolalohinski/gonja/v2/config" + "github.com/nikolalohinski/gonja/v2/exec" + "github.com/nikolalohinski/gonja/v2/loaders" + "github.com/spf13/cast" +) + +// ============================================================================= +// Macros +// ============================================================================= + +// MacroFile represents a .macros.sql file containing Jinja macro definitions. +type MacroFile struct { + FilePath string // absolute path + Dir string // directory relative to project root ("" for root, "staging", "marts/core") + RawSQL string // raw file content with {% macro %} definitions +} + +// macroNameRegex extracts macro names from {% macro name(...) %} blocks. +var macroNameRegex = regexp.MustCompile(`\{%-?\s*macro\s+(\w+)\s*\(`) + +// collectMacro reads a .macros.sql file and adds it to the project. +func collectMacro(project *BuildProject, absPath string, relDir string) error { + rawSQL, err := os.ReadFile(absPath) + if err != nil { + return g.Error(err, "could not read macro file: %s", absPath) + } + + // Normalize relDir: use forward slashes, "" for root + relDir = filepath.ToSlash(relDir) + if relDir == "." { + relDir = "" + } + + project.Macros = append(project.Macros, &MacroFile{ + FilePath: absPath, + Dir: relDir, + RawSQL: string(rawSQL), + }) + + return nil +} + +// GetMacrosForModel returns the concatenated macro SQL applicable to a model. +// Macros are ordered root-first (outermost to innermost scope). +func GetMacrosForModel(project *BuildProject, model *Model) string { + if len(project.Macros) == 0 { + return "" + } + + // Get model's relative directory + modelDir := filepath.ToSlash(filepath.Dir(model.RelPath)) + if modelDir == "." { + modelDir = "" + } + + // Filter applicable macros + var applicable []*MacroFile + for _, m := range project.Macros { + if macroApplies(m.Dir, modelDir) { + applicable = append(applicable, m) + } + } + + if len(applicable) == 0 { + return "" + } + + // Sort: by dir depth ascending (root first), then dir name, then filename + sort.Slice(applicable, func(i, j int) bool { + di := dirDepth(applicable[i].Dir) + dj := dirDepth(applicable[j].Dir) + if di != dj { + return di < dj + } + if applicable[i].Dir != applicable[j].Dir { + return applicable[i].Dir < applicable[j].Dir + } + return filepath.Base(applicable[i].FilePath) < filepath.Base(applicable[j].FilePath) + }) + + // Concatenate macro content + var parts []string + for _, m := range applicable { + parts = append(parts, m.RawSQL) + } + + return strings.Join(parts, "\n") +} + +// macroApplies returns true if a macro defined in macroDir is accessible from modelDir. +func macroApplies(macroDir, modelDir string) bool { + // Root-level macros are global + if macroDir == "" { + return true + } + // Same directory + if macroDir == modelDir { + return true + } + // Model is in a child directory of the macro + return strings.HasPrefix(modelDir, macroDir+"/") +} + +// dirDepth returns the number of path segments (0 for root). +func dirDepth(dir string) int { + if dir == "" { + return 0 + } + return len(strings.Split(dir, "/")) +} + +// warnMacroShadows warns when the same macro name is defined at multiple scope levels. +// Always emits (not debug-only) — macro shadowing is a correctness smell. +func warnMacroShadows(project *BuildProject) { + if len(project.Macros) == 0 { + return + } + + // Extract macro names from each file with their directory scope + type macroLoc struct { + name string + dir string + filePath string + } + + var locs []macroLoc + for _, m := range project.Macros { + matches := macroNameRegex.FindAllStringSubmatch(m.RawSQL, -1) + for _, match := range matches { + locs = append(locs, macroLoc{ + name: match[1], + dir: m.Dir, + filePath: m.FilePath, + }) + } + } + + // Group by name first (O(n) groups, then O(k²) within each name) + byName := make(map[string][]macroLoc) + for _, loc := range locs { + byName[loc.name] = append(byName[loc.name], loc) + } + + for name, group := range byName { + if len(group) < 2 { + continue + } + for i, a := range group { + for _, b := range group[i+1:] { + if macroApplies(a.dir, b.dir) && a.dir != b.dir { + g.Warn("macro '%s' in %s shadows definition in %s", name, b.filePath, a.filePath) + } else if macroApplies(b.dir, a.dir) && a.dir != b.dir { + g.Warn("macro '%s' in %s shadows definition in %s", name, a.filePath, b.filePath) + } + } + } + } +} + +// ============================================================================= +// SQL Parser +// ============================================================================= + +// tableRefRegex matches FROM/JOIN followed by a table reference. +// Captures: optional schema (word + dot) + table name. +// Handles optional quoting with double quotes or backticks. +// Ignores subqueries (parentheses after FROM/JOIN). +var tableRefRegex = regexp.MustCompile( + `(?i)(?:FROM|JOIN)\s+` + // FROM or JOIN keyword + `(?:(?:LATERAL|NATURAL|LEFT|RIGHT|INNER|OUTER|CROSS|FULL)\s+)*` + // optional join qualifiers + `(?:(?:OUTER|INNER)\s+)?` + // another optional qualifier + `(?:JOIN\s+)?` + // optional repeated JOIN after qualifiers like LEFT OUTER JOIN + "(" + // start capture group + `(?:` + + `(?:` + quotePattern(`"`) + `|` + quotePattern("`") + `|\w+)` + // db/schema/table name (quoted or unquoted) + `\.` + // dot separator + `){0,2}` + // 0-2 prefix parts (database.schema.) + `(?:` + quotePattern(`"`) + `|` + quotePattern("`") + `|\w+)` + // final table name + ")" + // end capture group + `(?:\s|$|,|\))`, // followed by whitespace, end, comma, or close paren +) + +// quotePattern returns a regex pattern matching a quoted identifier with the given quote char. +func quotePattern(q string) string { + return q + `[^` + q + `]+` + q +} + +// cteNameRegex matches WITH ... AS or , name AS patterns to identify CTE names. +var cteNameRegex = regexp.MustCompile(`(?i)(?:WITH|,)\s+(\w+)\s+AS\s*\(`) + +// ExtractTableReferences extracts table references from FROM/JOIN clauses in SQL. +// It returns unique table references (schema.table or just table) found in the SQL, +// excluding CTEs and Jinja template expressions. +func ExtractTableReferences(sql string) []string { + // First, collect CTE names so we can exclude them + cteNames := make(map[string]bool) + for _, match := range cteNameRegex.FindAllStringSubmatch(sql, -1) { + if len(match) > 1 { + cteNames[strings.ToLower(match[1])] = true + } + } + + // Strip Jinja expressions to avoid matching template variables + cleaned := stripJinjaExpressions(sql) + + // Find all table references + seen := make(map[string]bool) + var results []string + + for _, match := range tableRefRegex.FindAllStringSubmatch(cleaned, -1) { + if len(match) < 2 { + continue + } + + ref := strings.TrimSpace(match[1]) + if ref == "" { + continue + } + + // Skip if it looks like a subquery or keyword + refLower := strings.ToLower(ref) + if isReservedWord(refLower) { + continue + } + + // Skip Jinja placeholder + if strings.Contains(ref, "__JINJA__") { + continue + } + + // Skip CTE references + namePart := refLower + if idx := strings.LastIndex(namePart, "."); idx >= 0 { + namePart = namePart[idx+1:] + } + // Unquote for comparison + namePart = unquoteIdentifier(namePart) + if cteNames[namePart] { + continue + } + + // Normalize: unquote identifiers + ref = normalizeTableRef(ref) + if ref == "" { + continue + } + + if !seen[ref] { + seen[ref] = true + results = append(results, ref) + } + } + + return results +} + +// stripJinjaExpressions removes {{ ... }} and {% ... %} blocks from SQL +// so they don't interfere with table reference detection. +func stripJinjaExpressions(sql string) string { + // Remove {{ ... }} + result := regexp.MustCompile(`\{\{.*?\}\}`).ReplaceAllString(sql, "__JINJA__") + // Remove {% ... %} + result = regexp.MustCompile(`\{%.*?%\}`).ReplaceAllString(result, "") + return result +} + +// normalizeTableRef removes quotes from a table reference like "schema"."table" -> schema.table. +func normalizeTableRef(ref string) string { + parts := strings.Split(ref, ".") + normalized := make([]string, 0, len(parts)) + for _, part := range parts { + part = unquoteIdentifier(part) + if part == "" { + return "" + } + normalized = append(normalized, part) + } + return strings.Join(normalized, ".") +} + +// detectQuoteStyle returns the quote character used in a table reference (", `, or empty). +func detectQuoteStyle(ref string) string { + for _, c := range ref { + if c == '"' { + return `"` + } + if c == '`' { + return "`" + } + if c == '.' { + continue + } + } + return "" +} + +// requoteTableRef applies the quoting style from the original reference to a new table name. +// E.g., if original was `"staging"."stg_orders"` and replacement is `dev_fritz.staging_stg_orders`, +// returns `"dev_fritz"."staging_stg_orders"`. +func requoteTableRef(original, replacement string) string { + q := detectQuoteStyle(original) + if q == "" { + return replacement + } + parts := strings.Split(replacement, ".") + for i, part := range parts { + parts[i] = q + part + q + } + return strings.Join(parts, ".") +} + +// unquoteIdentifier removes surrounding double quotes or backticks. +func unquoteIdentifier(s string) string { + if len(s) >= 2 { + if (s[0] == '"' && s[len(s)-1] == '"') || (s[0] == '`' && s[len(s)-1] == '`') { + return s[1 : len(s)-1] + } + } + return s +} + +// isReservedWord checks if the string is a SQL keyword that shouldn't be treated as a table. +func isReservedWord(s string) bool { + reserved := map[string]bool{ + "select": true, "from": true, "where": true, "and": true, "or": true, + "not": true, "in": true, "exists": true, "between": true, "like": true, + "is": true, "null": true, "true": true, "false": true, "as": true, + "on": true, "using": true, "case": true, "when": true, "then": true, + "else": true, "end": true, "group": true, "order": true, "by": true, + "having": true, "limit": true, "offset": true, "union": true, "all": true, + "intersect": true, "except": true, "values": true, "set": true, + "insert": true, "update": true, "delete": true, "into": true, + "lateral": true, "unnest": true, "generate_series": true, + } + return reserved[s] +} + +// ============================================================================= +// SQL Table Reference Rewriting +// ============================================================================= + +// protectLiterals replaces string literals, comments, and Jinja expressions with numbered +// placeholders so they are not affected by table reference rewriting. +func protectLiterals(sql string) (string, []string) { + var placeholders []string + var result strings.Builder + i := 0 + n := len(sql) + + for i < n { + // Jinja {{ ... }} + if i+1 < n && sql[i] == '{' && sql[i+1] == '{' { + end := strings.Index(sql[i+2:], "}}") + if end >= 0 { + end += i + 2 + 2 + placeholders = append(placeholders, sql[i:end]) + result.WriteString(g.F("__PROTECTED_%d__", len(placeholders)-1)) + i = end + continue + } + } + + // Jinja {% ... %} + if i+1 < n && sql[i] == '{' && sql[i+1] == '%' { + end := strings.Index(sql[i+2:], "%}") + if end >= 0 { + end += i + 2 + 2 + placeholders = append(placeholders, sql[i:end]) + result.WriteString(g.F("__PROTECTED_%d__", len(placeholders)-1)) + i = end + continue + } + } + + // Single-line comment -- + if i+1 < n && sql[i] == '-' && sql[i+1] == '-' { + end := strings.IndexByte(sql[i:], '\n') + if end < 0 { + end = n - i + } + placeholders = append(placeholders, sql[i:i+end]) + result.WriteString(g.F("__PROTECTED_%d__", len(placeholders)-1)) + i += end + continue + } + + // Block comment /* ... */ + if i+1 < n && sql[i] == '/' && sql[i+1] == '*' { + end := strings.Index(sql[i+2:], "*/") + if end >= 0 { + end += i + 2 + 2 + placeholders = append(placeholders, sql[i:end]) + result.WriteString(g.F("__PROTECTED_%d__", len(placeholders)-1)) + i = end + continue + } + } + + // Single-quoted string literal '...' (with '' escape) + if sql[i] == '\'' { + j := i + 1 + for j < n { + if sql[j] == '\'' { + if j+1 < n && sql[j+1] == '\'' { + j += 2 // escaped quote + continue + } + j++ // closing quote + break + } + j++ + } + placeholders = append(placeholders, sql[i:j]) + result.WriteString(g.F("__PROTECTED_%d__", len(placeholders)-1)) + i = j + continue + } + + result.WriteByte(sql[i]) + i++ + } + + return result.String(), placeholders +} + +// restoreLiterals replaces placeholders back with their original content. +func restoreLiterals(sql string, placeholders []string) string { + for i, p := range placeholders { + sql = strings.Replace(sql, g.F("__PROTECTED_%d__", i), p, 1) + } + return sql +} + +// RewriteTableReferences scans compiled SQL for table references matching prod-mode names +// of known models/seeds and rewrites them to current-mode FullTableNames. +// Returns the rewritten SQL and matched model/seed names (for DependsOn). +func RewriteTableReferences(sql string, project *BuildProject, selfName string) (string, []string) { + // Protect literals so we don't rewrite inside strings/comments + protected, placeholders := protectLiterals(sql) + + // Collect CTE names to exclude + cteNames := make(map[string]bool) + for _, match := range cteNameRegex.FindAllStringSubmatch(protected, -1) { + if len(match) > 1 { + cteNames[strings.ToLower(match[1])] = true + } + } + + // Build lookup index + index := project.BuildProdNameIndex() + + // Find all table reference positions + matches := tableRefRegex.FindAllStringSubmatchIndex(protected, -1) + + var deps []string + var result strings.Builder + lastEnd := 0 + + for _, loc := range matches { + if len(loc) < 4 { + continue + } + // loc[2]:loc[3] is capture group 1 (the table reference) + refStart, refEnd := loc[2], loc[3] + ref := protected[refStart:refEnd] + + // Normalize for lookup + normalized := normalizeTableRef(ref) + if normalized == "" { + continue + } + normalizedLower := strings.ToLower(normalized) + + // Skip reserved words + if isReservedWord(normalizedLower) { + continue + } + + // Skip CTE names + namePart := normalizedLower + if idx := strings.LastIndex(namePart, "."); idx >= 0 { + namePart = namePart[idx+1:] + } + if cteNames[namePart] { + continue + } + + // Look up in prod-name index + entry, ok := index[normalizedLower] + if !ok || entry.Name == selfName { + continue + } + + // Replace the table ref with current-mode FullTableName, + // preserving the original quoting style (double quotes, backticks, or none) + result.WriteString(protected[lastEnd:refStart]) + result.WriteString(requoteTableRef(ref, entry.FullTableName)) + lastEnd = refEnd + + if !containsStr(deps, entry.Name) { + deps = append(deps, entry.Name) + } + } + + result.WriteString(protected[lastEnd:]) + + // Restore protected regions + final := restoreLiterals(result.String(), placeholders) + return final, deps +} + +// ============================================================================= +// @model_name Preprocessing +// ============================================================================= + +// atRefRegex matches @identifier but not @@identifier (MySQL system variables). +// Captures the identifier after @. +var atRefRegex = regexp.MustCompile(`(?:^|[^@])@([a-zA-Z_]\w*)`) + +// preprocessAtRefs replaces @model_name references with the current-mode FullTableName. +// Must be called before Jinja rendering. Populates model.DependsOn for each resolved reference. +func (te *TemplateEngine) preprocessAtRefs(sql string, model *Model) string { + return atRefRegex.ReplaceAllStringFunc(sql, func(match string) string { + // Find where @ starts in the match (the prefix char may be included) + atIdx := strings.Index(match, "@") + prefix := match[:atIdx] + name := match[atIdx+1:] + + // Look up in models and seeds + if m, ok := te.project.Models[name]; ok && name != model.Name { + if !containsStr(model.DependsOn, name) { + model.DependsOn = append(model.DependsOn, name) + } + return prefix + m.FullTableName + } + if s, ok := te.project.Seeds[name]; ok { + if !containsStr(model.DependsOn, name) { + model.DependsOn = append(model.DependsOn, name) + } + return prefix + s.FullTableName + } + + // Not a known model/seed — leave as-is (could be a SQL Server @variable) + return match + }) +} + +// ============================================================================= +// Incremental style detection +// ============================================================================= + +// Style identifies which incremental pattern a model uses. +type Style int + +const ( + // StyleDbt is the dbt-compatible pattern: models use is_incremental() and {{ this }} + // to write their own WHERE clauses. This is the zero value and the harmless default + // for non-incremental models. + StyleDbt Style = iota + // StyleSling is the sling-native pattern: models use {incremental_where_cond} and/or + // {incremental_value} placeholders, and sling owns the WHERE clause / watermark. + StyleSling +) + +// detectModelStyle scans raw model SQL (before Jinja rendering) to determine which +// incremental pattern the model uses. Returns an error if both patterns appear in +// the same file — the user must pick one. +func detectModelStyle(rawSQL string) (Style, error) { + hasSling := strings.Contains(rawSQL, "{incremental_where_cond}") || + strings.Contains(rawSQL, "{incremental_value}") + hasDbt := strings.Contains(rawSQL, "is_incremental(") + + if hasSling && hasDbt { + return StyleDbt, g.Error("cannot mix is_incremental() and {incremental_where_cond} in the same model. Choose one pattern: either dbt-compatible (is_incremental() + {{ this }}) or sling-native ({incremental_where_cond})") + } + if hasSling { + return StyleSling, nil + } + // Only dbt pattern, or neither pattern, → StyleDbt (harmless default; + // is_incremental() is registered but never called for models that don't use it) + return StyleDbt, nil +} + +// ============================================================================= +// Incremental Context +// ============================================================================= + +// IncrementalContext carries values used by CompileModel to resolve incremental +// placeholders and flags. One type serves both styles: +// +// - Style A (dbt): CompileModel reads IsIncremental only and passes it to +// the is_incremental() Jinja function. WhereCond/Value +// are ignored because the SQL does not contain the +// {incremental_where_cond}/{incremental_value} placeholders. +// +// - Style B (sling): CompileModel reads WhereCond/Value and substitutes +// them into the rendered SQL via g.R(). is_incremental() +// is still registered but returns IsIncremental (which +// callers typically leave false for sling-style models). +// +// A nil *IncrementalContext is equivalent to DefaultIncrementalContext(): +// first-run semantics (WhereCond=1=1, Value=null, IsIncremental=false). +type IncrementalContext struct { + WhereCond string // e.g., `"created_at" > '2024-01-01'` or "1=1" + Value string // e.g., `'2024-01-01'` or "null" + IsIncremental bool // drives is_incremental() for dbt-style models +} + +// DefaultIncrementalContext returns a first-run context: placeholders resolve +// to "1=1"/"null" and is_incremental() returns false. +func DefaultIncrementalContext() *IncrementalContext { + return &IncrementalContext{ + WhereCond: "1=1", + Value: "null", + IsIncremental: false, + } +} + +// ============================================================================= +// Template Engine +// ============================================================================= + +// configBlockRegex matches {%- config(...) -%} or {% config(...) %} blocks +// and rewrites them to {{ config(...) }} so gonja treats them as expressions. +var configBlockRegex = regexp.MustCompile(`\{%-?\s*config\(([^)]*)\)\s*-?%\}`) + +// TemplateEngine compiles SQL model templates using Jinja-like syntax. +type TemplateEngine struct { + project *BuildProject + vars map[string]any +} + +// NewTemplateEngine creates a new template engine for the given project. +func NewTemplateEngine(project *BuildProject, vars map[string]any) *TemplateEngine { + if vars == nil { + vars = make(map[string]any) + } + return &TemplateEngine{ + project: project, + vars: vars, + } +} + +// CompileModel compiles a model's SQL template, extracting config and resolving references. +// The incCtx parameter carries incremental-pattern values: +// - Style A (dbt) models read incCtx.IsIncremental via the is_incremental() Jinja function. +// - Style B (sling) models read incCtx.WhereCond / incCtx.Value via {incremental_where_cond} +// and {incremental_value} placeholders, substituted after Jinja rendering. +// +// A nil incCtx is equivalent to DefaultIncrementalContext() — first-run semantics. +func (te *TemplateEngine) CompileModel(model *Model, incCtx *IncrementalContext) (string, error) { + if incCtx == nil { + incCtx = DefaultIncrementalContext() + } + + // Build the gonja environment + envCtx := builtins.GlobalFunctions.Inherit().Update(builtins.GlobalVariables) + + // Register config() function — captures kwargs into model.Config, returns empty string. + // If YAML frontmatter was used, config() is a no-op (frontmatter is canonical). + envCtx.Set("config", func(_ *exec.Evaluator, params *exec.VarArgs) (string, error) { + if !model.HasFrontmatter { + if err := te.applyConfig(model, params); err != nil { + return "", err + } + } + return "", nil + }) + + // Register ref() function — resolves model/seed name to full table name + envCtx.Set("ref", func(_ *exec.Evaluator, params *exec.VarArgs) (string, error) { + if len(params.Args) == 0 { + return "", g.Error("ref() requires a model name argument") + } + name := params.Args[0].String() + + fullName, ok := te.project.LookupFullTableName(name) + if !ok { + return "", g.Error("ref('%s'): model or seed not found in project", name) + } + + // Record dependency + if !containsStr(model.Refs, name) { + model.Refs = append(model.Refs, name) + } + if !containsStr(model.DependsOn, name) { + model.DependsOn = append(model.DependsOn, name) + } + + return fullName, nil + }) + + // Register src()/source() functions — passthrough, records as source + // Accepts src('schema.table') or source('schema', 'table') + srcFunc := func(_ *exec.Evaluator, params *exec.VarArgs) (string, error) { + if len(params.Args) == 0 { + return "", g.Error("src()/source() requires a table name argument") + } + var tableName string + if len(params.Args) >= 2 { + tableName = params.Args[0].String() + "." + params.Args[1].String() + } else { + tableName = params.Args[0].String() + } + + if !containsStr(model.Sources, tableName) { + model.Sources = append(model.Sources, tableName) + } + + return tableName, nil + } + envCtx.Set("src", srcFunc) + envCtx.Set("source", srcFunc) + + // Register this — resolves to model's own full table name + envCtx.Set("this", model.FullTableName) + + // Register is_incremental() function — drives dbt-style {% if is_incremental() %} blocks + envCtx.Set("is_incremental", func(params *exec.VarArgs) bool { + return incCtx.IsIncremental + }) + + // Register user vars + for k, v := range te.vars { + envCtx.Set(k, v) + } + + env := &exec.Environment{ + Context: envCtx, + Filters: builtins.Filters, + Tests: builtins.Tests, + ControlStructures: builtins.ControlStructures, + Methods: builtins.Methods, + } + + // Pre-process: convert {%- config(...) -%} to {{ config(...) }} + processedSQL := preprocessSQL(model.RawSQL) + + // Prepend applicable macros + macroSQL := GetMacrosForModel(te.project, model) + if macroSQL != "" { + processedSQL = macroSQL + "\n" + processedSQL + } + + // Resolve @model_name references before Jinja rendering + processedSQL = te.preprocessAtRefs(processedSQL, model) + + // Create template from model SQL + templateID := "/" + model.Name + loader := loaders.MustNewMemoryLoader(map[string]string{ + templateID: processedSQL, + }) + + tpl, err := exec.NewTemplate(templateID, config.New(), loader, env) + if err != nil { + return "", g.Error(err, "could not parse template for model '%s'", model.Name) + } + + // Execute template + result, err := tpl.ExecuteToString(exec.EmptyContext()) + if err != nil { + return "", g.Error(err, "could not compile template for model '%s'", model.Name) + } + + // Trim whitespace + compiled := strings.TrimSpace(result) + + // Substitute sling-style placeholders. g.R uses {key} literal-bracket syntax; + // gonja preserves single-brace text verbatim, so this runs after Jinja rendering. + // g.R is a no-op for placeholders not present in the string, so dbt-style and + // view/full-refresh models are unaffected. + compiled = g.R(compiled, + "incremental_where_cond", incCtx.WhereCond, + "incremental_value", incCtx.Value, + ) + + model.CompiledSQL = compiled + + return compiled, nil +} + +// CompileAll compiles all models in the project using the provided incremental +// context. A nil incCtx is equivalent to DefaultIncrementalContext(). +func (te *TemplateEngine) CompileAll(incCtx *IncrementalContext) error { + for _, model := range te.project.Models { + if _, err := te.CompileModel(model, incCtx); err != nil { + return err + } + } + return nil +} + +// applyConfig extracts config kwargs into the model's ModelConfig. +func (te *TemplateEngine) applyConfig(model *Model, params *exec.VarArgs) error { + for key, val := range params.KwArgs { + strVal := val.String() + switch key { + case "mode": + canonical, warn := normalizeMode(strVal) + if warn != "" { + g.Warn("model '%s': %s", model.Name, warn) + } + if canonical == "ephemeral" { + return g.Error("model '%s': ephemeral models are not supported; use view or table", model.Name) + } + model.Config.Mode = canonical + case "materialized": + // dbt alias — only applied when mode is unset + mapped, err := mapMaterialized(strVal) + if err != nil { + return g.Error(err, "model '%s'", model.Name) + } + if model.Config.Mode == "" { + model.Config.Mode = mapped + } + case "unique_key": + // Can be a string or list + if val.IsList() { + model.Config.UniqueKey = toStringSlice(val) + } else { + model.Config.UniqueKey = strVal + } + case "merge_strategy": + model.Config.MergeStrategy = strVal + case "update_key": + model.Config.UpdateKey = strVal + case "tags": + model.Config.Tags = toStringSlice(val) + case "pre_hook", "post_hook": + return g.Error("pre_hook/post_hook are not supported in sling build config(). Use YAML frontmatter with hooks.start/hooks.end instead.\nSee https://docs.slingdata.io/concepts/sling-build for details") + case "schema": + model.Config.Schema = strVal + case "enabled": + enabled := cast.ToBool(strVal) + model.Config.Enabled = &enabled + case "engine": + model.Config.Engine = strVal + case "drop_cascade": + dc := cast.ToBool(strVal) + model.Config.DropCascade = &dc + case "rewrite": + rw := cast.ToBool(strVal) + model.Config.Rewrite = &rw + default: + g.Warn("model '%s': unrecognized config key '%s' (ignored)", model.Name, key) + } + } + return nil +} + +// toStringSlice converts a gonja Value to a string slice. +func toStringSlice(val *exec.Value) []string { + if val == nil { + return nil + } + iface := val.Interface() + switch v := iface.(type) { + case []string: + return v + case []interface{}: + result := make([]string, 0, len(v)) + for _, item := range v { + result = append(result, cast.ToString(item)) + } + return result + default: + return []string{cast.ToString(iface)} + } +} + +// containsStr checks if a string is in a slice. +func containsStr(slice []string, s string) bool { + for _, item := range slice { + if item == s { + return true + } + } + return false +} + +// preprocessSQL converts dbt-style {%- config(...) -%} blocks to {{ config(...) }} +// so gonja treats them as expressions rather than control structures. +func preprocessSQL(sql string) string { + return configBlockRegex.ReplaceAllString(sql, "{{ config($1) }}") +} diff --git a/core/sling/build/template_test.go b/core/sling/build/template_test.go new file mode 100644 index 000000000..0212d9077 --- /dev/null +++ b/core/sling/build/template_test.go @@ -0,0 +1,1598 @@ +package build + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// ============================================================================= +// Macro Tests +// ============================================================================= + +func TestMacroApplies(t *testing.T) { + tests := []struct { + name string + macroDir string + modelDir string + want bool + }{ + {"root macro applies to root model", "", "", true}, + {"root macro applies to staging model", "", "staging", true}, + {"root macro applies to deeply nested model", "", "marts/core/sub", true}, + {"staging macro applies to staging model", "staging", "staging", true}, + {"staging macro applies to staging child", "staging", "staging/sub", true}, + {"staging macro does not apply to marts model", "staging", "marts", false}, + {"staging macro does not apply to root model", "staging", "", false}, + {"marts/core macro applies to marts/core model", "marts/core", "marts/core", true}, + {"marts/core macro applies to marts/core/sub", "marts/core", "marts/core/sub", true}, + {"marts/core macro does not apply to marts model", "marts/core", "marts", false}, + {"marts macro does not apply to marts_v2 model", "marts", "marts_v2", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := macroApplies(tt.macroDir, tt.modelDir) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestDirDepth(t *testing.T) { + assert.Equal(t, 0, dirDepth("")) + assert.Equal(t, 1, dirDepth("staging")) + assert.Equal(t, 2, dirDepth("marts/core")) + assert.Equal(t, 3, dirDepth("marts/core/sub")) +} + +func TestGetMacrosForModel(t *testing.T) { + project := &BuildProject{ + Dir: "/test", + Mode: "prod", + DefaultSchema: "public", + Models: make(map[string]*Model), + Seeds: make(map[string]*Seed), + Macros: []*MacroFile{ + { + FilePath: "/test/utils.macros.sql", + Dir: "", + RawSQL: "{% macro global_fn() %}GLOBAL{% endmacro %}", + }, + { + FilePath: "/test/staging/helpers.macros.sql", + Dir: "staging", + RawSQL: "{% macro staging_fn() %}STAGING{% endmacro %}", + }, + { + FilePath: "/test/marts/marts_utils.macros.sql", + Dir: "marts", + RawSQL: "{% macro marts_fn() %}MARTS{% endmacro %}", + }, + }, + } + + // Staging model should get global + staging macros + stagingModel := &Model{ + Name: "stg_orders", + RelPath: "staging/stg_orders.sql", + } + result := GetMacrosForModel(project, stagingModel) + assert.Contains(t, result, "global_fn") + assert.Contains(t, result, "staging_fn") + assert.NotContains(t, result, "marts_fn") + + // Marts model should get global + marts macros + martsModel := &Model{ + Name: "dim_customers", + RelPath: "marts/core/dim_customers.sql", + } + result = GetMacrosForModel(project, martsModel) + assert.Contains(t, result, "global_fn") + assert.Contains(t, result, "marts_fn") + assert.NotContains(t, result, "staging_fn") + + // Root model should get only global macros + rootModel := &Model{ + Name: "raw", + RelPath: "raw.sql", + } + result = GetMacrosForModel(project, rootModel) + assert.Contains(t, result, "global_fn") + assert.NotContains(t, result, "staging_fn") + assert.NotContains(t, result, "marts_fn") +} + +func TestGetMacrosForModelOrdering(t *testing.T) { + project := &BuildProject{ + Dir: "/test", + Mode: "prod", + DefaultSchema: "public", + Models: make(map[string]*Model), + Seeds: make(map[string]*Seed), + Macros: []*MacroFile{ + { + FilePath: "/test/staging/b_helpers.macros.sql", + Dir: "staging", + RawSQL: "STAGING_B", + }, + { + FilePath: "/test/utils.macros.sql", + Dir: "", + RawSQL: "GLOBAL", + }, + { + FilePath: "/test/staging/a_helpers.macros.sql", + Dir: "staging", + RawSQL: "STAGING_A", + }, + }, + } + + model := &Model{ + Name: "stg_orders", + RelPath: "staging/stg_orders.sql", + } + + result := GetMacrosForModel(project, model) + + // Root macros should come before staging macros + globalIdx := strings.Index(result, "GLOBAL") + stagingAIdx := strings.Index(result, "STAGING_A") + stagingBIdx := strings.Index(result, "STAGING_B") + + assert.True(t, globalIdx < stagingAIdx, "global macros should come before staging macros") + assert.True(t, stagingAIdx < stagingBIdx, "same-dir macros should be sorted by filename") +} + +func TestGetMacrosForModelEmpty(t *testing.T) { + project := &BuildProject{ + Dir: "/test", + Mode: "prod", + DefaultSchema: "public", + Models: make(map[string]*Model), + Seeds: make(map[string]*Seed), + } + + model := &Model{ + Name: "test", + RelPath: "staging/test.sql", + } + + result := GetMacrosForModel(project, model) + assert.Empty(t, result) +} + +func TestCollectMacro(t *testing.T) { + // Create a temp macro file + dir := t.TempDir() + macroPath := filepath.Join(dir, "test.macros.sql") + content := "{% macro test_fn() %}TEST{% endmacro %}" + require.NoError(t, os.WriteFile(macroPath, []byte(content), 0644)) + + project := &BuildProject{ + Dir: dir, + Models: make(map[string]*Model), + Seeds: make(map[string]*Seed), + DefaultSchema: "public", + } + + err := collectMacro(project, macroPath, "") + require.NoError(t, err) + + assert.Len(t, project.Macros, 1) + assert.Equal(t, macroPath, project.Macros[0].FilePath) + assert.Equal(t, "", project.Macros[0].Dir) + assert.Equal(t, content, project.Macros[0].RawSQL) +} + +func TestMacroDiscoveryFlatMode(t *testing.T) { + dir := getTestFixturePath("sample_project") + + project, err := LoadProject(dir) + require.NoError(t, err) + + // Should discover macro files + assert.Len(t, project.Macros, 2, "should discover utils.macros.sql and staging_helpers.macros.sql") + + // Model and seed counts should be unchanged + assert.Len(t, project.Models, 6) + assert.Len(t, project.Seeds, 2) + + // Verify macro dirs + macroDirs := make(map[string]bool) + for _, m := range project.Macros { + macroDirs[m.Dir] = true + } + assert.True(t, macroDirs[""], "should have root-level macro") + assert.True(t, macroDirs["staging"], "should have staging-level macro") +} + +func TestMacroCompilationEndToEnd(t *testing.T) { + project := &BuildProject{ + Dir: "/test", + Mode: "prod", + DefaultSchema: "public", + Models: make(map[string]*Model), + Seeds: make(map[string]*Seed), + Macros: []*MacroFile{ + { + FilePath: "/test/utils.macros.sql", + Dir: "", + RawSQL: `{% macro cents_to_dollars(column_name) %} + ({{ column_name }} / 100.0) +{% endmacro %}`, + }, + }, + } + + model := &Model{ + Name: "test_model", + Schema: "staging", + FullTableName: "staging.test_model", + RelPath: "staging/test_model.sql", + RawSQL: "SELECT id, {{ cents_to_dollars('amount_cents') }} as amount_dollars FROM orders", + } + project.Models["test_model"] = model + + te := NewTemplateEngine(project, nil) + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Contains(t, result, "(amount_cents / 100.0)") + assert.Contains(t, result, "as amount_dollars") +} + +func TestMacroScopeIsolation(t *testing.T) { + project := &BuildProject{ + Dir: "/test", + Mode: "prod", + DefaultSchema: "public", + Models: make(map[string]*Model), + Seeds: make(map[string]*Seed), + Macros: []*MacroFile{ + { + FilePath: "/test/staging/helpers.macros.sql", + Dir: "staging", + RawSQL: "{% macro staging_only() %}STAGING_RESULT{% endmacro %}", + }, + }, + } + + // Staging model can use the staging-scoped macro + stagingModel := &Model{ + Name: "stg_test", + Schema: "staging", + FullTableName: "staging.stg_test", + RelPath: "staging/stg_test.sql", + RawSQL: "SELECT {{ staging_only() }} as val", + } + project.Models["stg_test"] = stagingModel + + te := NewTemplateEngine(project, nil) + result, err := te.CompileModel(stagingModel, nil) + require.NoError(t, err) + assert.Contains(t, result, "STAGING_RESULT") + + // Marts model cannot use the staging-scoped macro + martsModel := &Model{ + Name: "dim_test", + Schema: "marts", + FullTableName: "marts.dim_test", + RelPath: "marts/dim_test.sql", + RawSQL: "SELECT {{ staging_only() }} as val", + } + project.Models["dim_test"] = martsModel + + _, err = te.CompileModel(martsModel, nil) + assert.Error(t, err, "marts model should not have access to staging-scoped macro") +} + +func TestMacroWithMultipleArgs(t *testing.T) { + project := &BuildProject{ + Dir: "/test", + Mode: "prod", + DefaultSchema: "public", + Models: make(map[string]*Model), + Seeds: make(map[string]*Seed), + Macros: []*MacroFile{ + { + FilePath: "/test/utils.macros.sql", + Dir: "", + RawSQL: `{% macro safe_divide(numerator, denominator) %} + CASE WHEN {{ denominator }} = 0 THEN NULL ELSE {{ numerator }}::float / {{ denominator }} END +{% endmacro %}`, + }, + }, + } + + model := &Model{ + Name: "test_model", + Schema: "staging", + FullTableName: "staging.test_model", + RelPath: "staging/test_model.sql", + RawSQL: "SELECT {{ safe_divide('revenue', 'num_orders') }} as avg_order_value", + } + project.Models["test_model"] = model + + te := NewTemplateEngine(project, nil) + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Contains(t, result, "CASE WHEN num_orders = 0 THEN NULL") + assert.Contains(t, result, "revenue::float / num_orders") +} + +func TestMacroWithExistingModelRefs(t *testing.T) { + project := &BuildProject{ + Dir: "/test", + Mode: "prod", + DefaultSchema: "public", + Models: make(map[string]*Model), + Seeds: make(map[string]*Seed), + Macros: []*MacroFile{ + { + FilePath: "/test/utils.macros.sql", + Dir: "", + RawSQL: "{% macro add_prefix(val) %}prefix_{{ val }}{% endmacro %}", + }, + }, + } + + project.Models["stg_orders"] = &Model{ + Name: "stg_orders", + Schema: "staging", + FullTableName: "staging.stg_orders", + RelPath: "staging/stg_orders.sql", + RawSQL: "SELECT 1 as id", + } + + // Model that uses both macros and refs + model := &Model{ + Name: "test_model", + Schema: "staging", + FullTableName: "staging.test_model", + RelPath: "staging/test_model.sql", + RawSQL: "SELECT {{ add_prefix('hello') }} FROM {{ ref('stg_orders') }}", + } + project.Models["test_model"] = model + + te := NewTemplateEngine(project, nil) + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Contains(t, result, "prefix_hello") + assert.Contains(t, result, "FROM staging.stg_orders") + assert.Contains(t, model.Refs, "stg_orders") +} + +func TestMacroSampleProjectCompiles(t *testing.T) { + dir := getTestFixturePath("sample_project") + + b, err := NewBuild(dir, BuildOptions{Target: "POSTGRES"}) + require.NoError(t, err) + + err = b.Compile() + require.NoError(t, err) + + // Macros are present but not called by existing models — should compile fine + assert.NotNil(t, b.DAG) + assert.Len(t, b.Selected, 8) // 6 models + 2 seeds, unchanged +} + +// ============================================================================= +// SQL Parser Tests +// ============================================================================= + +func TestSQLParserSimpleFrom(t *testing.T) { + sql := `SELECT * FROM staging.orders` + refs := ExtractTableReferences(sql) + assert.Contains(t, refs, "staging.orders") +} + +func TestSQLParserMultipleFromJoin(t *testing.T) { + sql := `SELECT o.*, c.name +FROM staging.orders o +JOIN staging.customers c ON o.customer_id = c.id +LEFT JOIN public.products p ON o.product_id = p.id` + + refs := ExtractTableReferences(sql) + assert.Contains(t, refs, "staging.orders") + assert.Contains(t, refs, "staging.customers") + assert.Contains(t, refs, "public.products") + assert.Len(t, refs, 3) +} + +func TestSQLParserUnqualifiedTable(t *testing.T) { + sql := `SELECT * FROM orders` + refs := ExtractTableReferences(sql) + assert.Contains(t, refs, "orders") +} + +func TestSQLParserQuotedIdentifiers(t *testing.T) { + sql := `SELECT * FROM "my schema"."my table"` + refs := ExtractTableReferences(sql) + assert.Contains(t, refs, "my schema.my table") +} + +func TestSQLParserBacktickQuotes(t *testing.T) { + sql := "SELECT * FROM `staging`.`orders`" + refs := ExtractTableReferences(sql) + assert.Contains(t, refs, "staging.orders") +} + +func TestSQLParserSkipsCTEs(t *testing.T) { + sql := `WITH cte_orders AS ( + SELECT * FROM staging.raw_orders +), +cte_customers AS ( + SELECT * FROM staging.raw_customers +) +SELECT o.*, c.name +FROM cte_orders o +JOIN cte_customers c ON o.customer_id = c.id` + + refs := ExtractTableReferences(sql) + // Should contain the real tables, not the CTEs + assert.Contains(t, refs, "staging.raw_orders") + assert.Contains(t, refs, "staging.raw_customers") + assert.NotContains(t, refs, "cte_orders") + assert.NotContains(t, refs, "cte_customers") +} + +func TestSQLParserSkipsJinja(t *testing.T) { + sql := `SELECT * FROM {{ ref('stg_orders') }} +JOIN {{ src('raw_db.accounts') }} ON 1=1` + + refs := ExtractTableReferences(sql) + // Jinja expressions should be stripped, not matched as tables + assert.NotContains(t, refs, "ref") + assert.NotContains(t, refs, "src") + // __JINJA__ placeholder might be captured but should be filtered + for _, ref := range refs { + assert.NotContains(t, ref, "JINJA") + } +} + +func TestSQLParserDeduplicates(t *testing.T) { + sql := `SELECT * FROM staging.orders +UNION ALL +SELECT * FROM staging.orders` + + refs := ExtractTableReferences(sql) + // Count occurrences of staging.orders + count := 0 + for _, ref := range refs { + if ref == "staging.orders" { + count++ + } + } + assert.Equal(t, 1, count, "should deduplicate") +} + +func TestSQLParserThreePartName(t *testing.T) { + sql := `SELECT * FROM mydb.staging.orders` + refs := ExtractTableReferences(sql) + assert.Contains(t, refs, "mydb.staging.orders") +} + +func TestSQLParserSubqueryNotMatched(t *testing.T) { + sql := `SELECT * FROM staging.orders WHERE id IN (SELECT order_id FROM staging.line_items)` + refs := ExtractTableReferences(sql) + assert.Contains(t, refs, "staging.orders") + assert.Contains(t, refs, "staging.line_items") +} + +func TestSQLParserLeftOuterJoin(t *testing.T) { + sql := `SELECT * FROM staging.orders o LEFT OUTER JOIN staging.customers c ON o.id = c.order_id` + refs := ExtractTableReferences(sql) + assert.Contains(t, refs, "staging.orders") + assert.Contains(t, refs, "staging.customers") +} + +func TestSQLParserCrossJoin(t *testing.T) { + sql := `SELECT * FROM staging.orders CROSS JOIN staging.dates` + refs := ExtractTableReferences(sql) + assert.Contains(t, refs, "staging.orders") + assert.Contains(t, refs, "staging.dates") +} + +func TestSQLParserEmptySQL(t *testing.T) { + refs := ExtractTableReferences("") + assert.Empty(t, refs) +} + +func TestSQLParserNoFrom(t *testing.T) { + sql := `SELECT 1 as id, 'test' as name` + refs := ExtractTableReferences(sql) + assert.Empty(t, refs) +} + +func TestSQLParserSkipsReservedWords(t *testing.T) { + // Ensure we don't match keywords that might appear after FROM-like constructs + sql := `SELECT * FROM staging.orders WHERE EXISTS (SELECT 1 FROM staging.items)` + refs := ExtractTableReferences(sql) + assert.Contains(t, refs, "staging.orders") + assert.Contains(t, refs, "staging.items") + assert.NotContains(t, refs, "exists") +} + +func TestSQLParserMultilineSQL(t *testing.T) { + sql := ` +SELECT + o.id, + o.name +FROM + staging.orders o +INNER JOIN + staging.customers c + ON o.customer_id = c.id +WHERE + o.status = 'active' +` + refs := ExtractTableReferences(sql) + assert.Contains(t, refs, "staging.orders") + assert.Contains(t, refs, "staging.customers") +} + +func TestSQLParserMixedRefsAndAutoDetect(t *testing.T) { + // SQL that has both Jinja refs (stripped) and real table references + sql := `SELECT * FROM {{ ref('stg_orders') }} +JOIN raw_db.accounts ON 1=1` + + refs := ExtractTableReferences(sql) + assert.Contains(t, refs, "raw_db.accounts") +} + +// ============================================================================= +// Template Engine Tests +// ============================================================================= + +func newTestProject() *BuildProject { + return &BuildProject{ + Dir: "/test", + Mode: "prod", + DefaultSchema: "public", + Models: map[string]*Model{ + "stg_orders": { + Name: "stg_orders", + Schema: "staging", + FullTableName: "staging.stg_orders", + ProdFullTableName: "staging.stg_orders", + RawSQL: "SELECT 1 as id, 'order_1' as name", + }, + "stg_customers": { + Name: "stg_customers", + Schema: "staging", + FullTableName: "staging.stg_customers", + ProdFullTableName: "staging.stg_customers", + RawSQL: "SELECT 1 as id, 'customer_1' as name", + }, + }, + Seeds: map[string]*Seed{ + "country_codes": { + Name: "country_codes", + Schema: "staging", + FullTableName: "staging.country_codes", + ProdFullTableName: "staging.country_codes", + Format: "csv", + }, + }, + } +} + +func newTestProjectDev() *BuildProject { + return &BuildProject{ + Dir: "/test", + Mode: "dev", + SchemaOverride: "dev_fritz", + DefaultSchema: "public", + Models: map[string]*Model{ + "stg_orders": { + Name: "stg_orders", + Schema: "dev_fritz", + Prefix: "staging", + FullTableName: "dev_fritz.staging_stg_orders", + ProdFullTableName: "staging.stg_orders", + RawSQL: "SELECT 1 as id, 'order_1' as name", + }, + "stg_customers": { + Name: "stg_customers", + Schema: "dev_fritz", + Prefix: "staging", + FullTableName: "dev_fritz.staging_stg_customers", + ProdFullTableName: "staging.stg_customers", + RawSQL: "SELECT 1 as id, 'customer_1' as name", + }, + }, + Seeds: map[string]*Seed{ + "country_codes": { + Name: "country_codes", + Schema: "dev_fritz", + Prefix: "staging", + FullTableName: "dev_fritz.staging_country_codes", + ProdFullTableName: "staging.country_codes", + Format: "csv", + }, + }, + } +} + +func TestCompileSimpleModel(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := project.Models["stg_orders"] + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Equal(t, "SELECT 1 as id, 'order_1' as name", result) + assert.Empty(t, model.Refs) + assert.Empty(t, model.Sources) + assert.Empty(t, model.DependsOn) +} + +func TestCompileWithRef(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "dim_customers", + Schema: "marts", + FullTableName: "marts.dim_customers", + RawSQL: "SELECT * FROM {{ ref('stg_customers') }}", + } + project.Models["dim_customers"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Equal(t, "SELECT * FROM staging.stg_customers", result) + assert.Contains(t, model.Refs, "stg_customers") + assert.Contains(t, model.DependsOn, "stg_customers") +} + +func TestCompileWithSeedRef(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "dim_countries", + Schema: "marts", + FullTableName: "marts.dim_countries", + RawSQL: "SELECT * FROM {{ ref('country_codes') }}", + } + project.Models["dim_countries"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Equal(t, "SELECT * FROM staging.country_codes", result) + assert.Contains(t, model.Refs, "country_codes") +} + +func TestCompileWithSrc(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "test_model", + Schema: "staging", + FullTableName: "staging.test_model", + RawSQL: "SELECT * FROM {{ src('raw_db.accounts') }}", + } + project.Models["test_model"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Equal(t, "SELECT * FROM raw_db.accounts", result) + assert.Contains(t, model.Sources, "raw_db.accounts") + assert.Empty(t, model.DependsOn) // src() does not create DAG edges +} + +func TestCompileWithSrcTwoArgs(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "test_model", + Schema: "staging", + FullTableName: "staging.test_model", + RawSQL: "SELECT * FROM {{ src('raw_db', 'accounts') }}", + } + project.Models["test_model"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Equal(t, "SELECT * FROM raw_db.accounts", result) + assert.Contains(t, model.Sources, "raw_db.accounts") + assert.Empty(t, model.DependsOn) +} + +func TestCompileWithSource(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "test_model", + Schema: "staging", + FullTableName: "staging.test_model", + RawSQL: "SELECT * FROM {{ source('raw_db', 'accounts') }}", + } + project.Models["test_model"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Equal(t, "SELECT * FROM raw_db.accounts", result) + assert.Contains(t, model.Sources, "raw_db.accounts") + assert.Empty(t, model.DependsOn) +} + +func TestCompileWithSourceOneArg(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "test_model", + Schema: "staging", + FullTableName: "staging.test_model", + RawSQL: "SELECT * FROM {{ source('raw_db.accounts') }}", + } + project.Models["test_model"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Equal(t, "SELECT * FROM raw_db.accounts", result) + assert.Contains(t, model.Sources, "raw_db.accounts") + assert.Empty(t, model.DependsOn) +} + +func TestCompileWithThis(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "fct_orders", + Schema: "marts", + FullTableName: "marts.fct_orders", + RawSQL: "SELECT MAX(updated_at) FROM {{ this }}", + } + project.Models["fct_orders"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Equal(t, "SELECT MAX(updated_at) FROM marts.fct_orders", result) +} + +func TestCompileWithIsIncremental(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "fct_orders", + Schema: "marts", + FullTableName: "marts.fct_orders", + RawSQL: `SELECT id FROM {{ ref('stg_orders') }} +{% if is_incremental() %} +WHERE updated_at > (SELECT MAX(updated_at) FROM {{ this }}) +{% endif %}`, + } + project.Models["fct_orders"] = model + + // Not incremental + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + assert.NotContains(t, result, "WHERE updated_at") + assert.Contains(t, result, "SELECT id FROM staging.stg_orders") + + // Reset refs for re-compile + model.Refs = nil + model.DependsOn = nil + + // Incremental + result, err = te.CompileModel(model, &IncrementalContext{IsIncremental: true}) + require.NoError(t, err) + assert.Contains(t, result, "WHERE updated_at > (SELECT MAX(updated_at) FROM marts.fct_orders)") +} + +func TestCompileWithConfig(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "fct_orders", + Schema: "marts", + FullTableName: "marts.fct_orders", + RawSQL: `{%- config(mode='incremental', unique_key='id', merge_strategy='delete+insert', update_key='updated_at') -%} +SELECT id, name FROM {{ ref('stg_orders') }}`, + } + project.Models["fct_orders"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + // Config block should produce no output + assert.Equal(t, "SELECT id, name FROM staging.stg_orders", result) + + // Config should be extracted + assert.Equal(t, "incremental", model.Config.Mode) + assert.Equal(t, "id", model.Config.UniqueKey) + assert.Equal(t, "delete+insert", model.Config.MergeStrategy) + assert.Equal(t, "updated_at", model.Config.UpdateKey) +} + +func TestCompileWithVars(t *testing.T) { + project := newTestProject() + vars := map[string]any{ + "start_date": "2024-01-01", + "environment": "dev", + } + te := NewTemplateEngine(project, vars) + + model := &Model{ + Name: "test_model", + Schema: "staging", + FullTableName: "staging.test_model", + RawSQL: "SELECT * FROM orders WHERE created_at >= '{{ start_date }}' AND env = '{{ environment }}'", + } + project.Models["test_model"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Contains(t, result, "2024-01-01") + assert.Contains(t, result, "dev") +} + +func TestCompileWithAtRef(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "dim_customers", + Schema: "marts", + FullTableName: "marts.dim_customers", + RawSQL: "SELECT * FROM @stg_customers", + } + project.Models["dim_customers"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + // @model_name resolves to the model's full table name + assert.Equal(t, "SELECT * FROM staging.stg_customers", result) + assert.Contains(t, model.DependsOn, "stg_customers") +} + +func TestCompileWithAtRefSeed(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "dim_countries", + Schema: "marts", + FullTableName: "marts.dim_countries", + RawSQL: "SELECT * FROM @country_codes", + } + project.Models["dim_countries"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Equal(t, "SELECT * FROM staging.country_codes", result) + assert.Contains(t, model.DependsOn, "country_codes") +} + +func TestCompileWithAtRefNoMatch(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "test_model", + Schema: "staging", + FullTableName: "staging.test_model", + RawSQL: "SELECT @rowcount, @unknown_var FROM @stg_orders", + } + project.Models["test_model"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + // @rowcount and @unknown_var are not models, left as-is + // @stg_orders is a model, gets resolved + assert.Contains(t, result, "@rowcount") + assert.Contains(t, result, "@unknown_var") + assert.Contains(t, result, "staging.stg_orders") +} + +func TestCompileWithDoubleAtSkipped(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "test_model", + Schema: "staging", + FullTableName: "staging.test_model", + RawSQL: "SELECT @@version, * FROM @stg_orders", + } + project.Models["test_model"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + // @@version should not be touched + assert.Contains(t, result, "@@version") + assert.Contains(t, result, "staging.stg_orders") +} + +func TestCompileWithBareRefRemoved(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "dim_customers", + Schema: "marts", + FullTableName: "marts.dim_customers", + RawSQL: "SELECT * FROM {{ stg_customers }}", + } + project.Models["dim_customers"] = model + + // Bare Jinja variable {{ stg_customers }} no longer resolves to a full table name. + // gonja renders undefined variables as empty string, so the SQL will be broken. + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + // The variable is no longer registered, so it renders as empty + assert.Equal(t, "SELECT * FROM", result) +} + +func TestCompileRefNotFound(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "bad_model", + Schema: "staging", + FullTableName: "staging.bad_model", + RawSQL: "SELECT * FROM {{ ref('nonexistent_model') }}", + } + project.Models["bad_model"] = model + + _, err := te.CompileModel(model, nil) + assert.Error(t, err) + assert.Contains(t, err.Error(), "nonexistent_model") +} + +func TestCompileWithConfigView(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "dim_view", + Schema: "marts", + FullTableName: "marts.dim_view", + RawSQL: `{%- config(mode='view') -%} +SELECT * FROM {{ ref('stg_customers') }}`, + } + project.Models["dim_view"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Equal(t, "SELECT * FROM staging.stg_customers", result) + assert.Equal(t, "view", model.Config.Mode) +} + +func TestCompileWithConfigEnabled(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "disabled_model", + Schema: "staging", + FullTableName: "staging.disabled_model", + RawSQL: `{%- config(enabled=false) -%} +SELECT 1`, + } + project.Models["disabled_model"] = model + + _, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.NotNil(t, model.Config.Enabled) + assert.False(t, *model.Config.Enabled) +} + +func TestCompileMultipleRefs(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "fct_combined", + Schema: "marts", + FullTableName: "marts.fct_combined", + RawSQL: `SELECT o.*, c.name as customer_name +FROM {{ ref('stg_orders') }} o +JOIN {{ ref('stg_customers') }} c ON o.id = c.id`, + } + project.Models["fct_combined"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Contains(t, result, "FROM staging.stg_orders o") + assert.Contains(t, result, "JOIN staging.stg_customers c") + assert.Len(t, model.Refs, 2) + assert.Contains(t, model.Refs, "stg_orders") + assert.Contains(t, model.Refs, "stg_customers") + assert.Len(t, model.DependsOn, 2) +} + +func TestCompileWithJinjaControlFlow(t *testing.T) { + project := newTestProject() + vars := map[string]any{ + "include_archived": true, + } + te := NewTemplateEngine(project, vars) + + model := &Model{ + Name: "test_model", + Schema: "staging", + FullTableName: "staging.test_model", + RawSQL: `SELECT * FROM {{ ref('stg_orders') }} +{% if include_archived %} +WHERE status IN ('active', 'archived') +{% else %} +WHERE status = 'active' +{% endif %}`, + } + project.Models["test_model"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Contains(t, result, "WHERE status IN ('active', 'archived')") + assert.NotContains(t, result, "WHERE status = 'active'") +} + +func TestCompileAllModels(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + err := te.CompileAll(nil) + require.NoError(t, err) + + for _, model := range project.Models { + assert.NotEmpty(t, model.CompiledSQL, "model %s should have compiled SQL", model.Name) + } +} + +func TestCompileDuplicateRef(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "test_model", + Schema: "staging", + FullTableName: "staging.test_model", + RawSQL: `SELECT * FROM {{ ref('stg_orders') }} +UNION ALL +SELECT * FROM {{ ref('stg_orders') }}`, + } + project.Models["test_model"] = model + + _, err := te.CompileModel(model, nil) + require.NoError(t, err) + + // Duplicate refs should only appear once + assert.Len(t, model.Refs, 1) + assert.Len(t, model.DependsOn, 1) +} + +// ============================================================================= +// YAML Frontmatter Compile Tests +// ============================================================================= + +func TestCompileWithYAMLFrontmatter(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "fct_orders", + Schema: "marts", + FullTableName: "marts.fct_orders", + HasFrontmatter: true, + Config: ModelConfig{ + Mode: "incremental", + UniqueKey: "id", + MergeStrategy: "delete+insert", + UpdateKey: "updated_at", + }, + RawSQL: "SELECT id, name FROM {{ ref('stg_orders') }}", + } + project.Models["fct_orders"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Equal(t, "SELECT id, name FROM staging.stg_orders", result) + assert.Equal(t, "incremental", model.Config.Mode) + assert.Equal(t, "id", model.Config.UniqueKey) + assert.Equal(t, "delete+insert", model.Config.MergeStrategy) + assert.Equal(t, "updated_at", model.Config.UpdateKey) +} + +func TestCompileFrontmatterOverridesJinjaConfig(t *testing.T) { + // When YAML frontmatter is present, jinja config() should be a no-op + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "fct_orders", + Schema: "marts", + FullTableName: "marts.fct_orders", + HasFrontmatter: true, + Config: ModelConfig{ + Mode: "incremental", + UniqueKey: "id", + }, + // This jinja config should be ignored because frontmatter takes precedence + RawSQL: `{%- config(mode='full-refresh', unique_key='other_id') -%} +SELECT id FROM {{ ref('stg_orders') }}`, + } + project.Models["fct_orders"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + // SQL should be rendered (config block produces empty string) + assert.Equal(t, "SELECT id FROM staging.stg_orders", result) + + // Config should retain frontmatter values, NOT jinja overrides + assert.Equal(t, "incremental", model.Config.Mode) + assert.Equal(t, "id", model.Config.UniqueKey) +} + +func TestCompileJinjaConfigFallback(t *testing.T) { + // When no frontmatter, jinja config() works as before (dbt compat) + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "fct_orders", + Schema: "marts", + FullTableName: "marts.fct_orders", + HasFrontmatter: false, + RawSQL: `{%- config(mode='incremental', unique_key='id') -%} +SELECT id FROM {{ ref('stg_orders') }}`, + } + project.Models["fct_orders"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + + assert.Equal(t, "SELECT id FROM staging.stg_orders", result) + assert.Equal(t, "incremental", model.Config.Mode) + assert.Equal(t, "id", model.Config.UniqueKey) +} + +func TestContainsSemicolon(t *testing.T) { + tests := []struct { + sql string + expected bool + }{ + {"SELECT 1", false}, + {"SELECT 1;", true}, + {"SELECT 'a;b'", false}, + {"SELECT 1 -- comment; here\nFROM t", false}, + {"SELECT 1; SELECT 2", true}, + {"SELECT /* ; */ 1", false}, + {"SELECT 1; -- trailing", true}, + } + + for _, tt := range tests { + got := containsSemicolon(tt.sql) + if got != tt.expected { + t.Errorf("containsSemicolon(%q) = %v, want %v", tt.sql, got, tt.expected) + } + } +} + +// ============================================================================= +// RewriteTableReferences Tests +// ============================================================================= + +func TestRewriteTableReferences_ProdMode(t *testing.T) { + project := newTestProject() + + // In prod mode, prod name == current name, so rewriting is a no-op + sql := "SELECT * FROM staging.stg_orders" + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + assert.Equal(t, "SELECT * FROM staging.stg_orders", rewritten) + assert.Contains(t, deps, "stg_orders") +} + +func TestRewriteTableReferences_DevMode(t *testing.T) { + project := newTestProjectDev() + + sql := "SELECT * FROM staging.stg_orders" + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + assert.Equal(t, "SELECT * FROM dev_fritz.staging_stg_orders", rewritten) + assert.Contains(t, deps, "stg_orders") +} + +func TestRewriteTableReferences_DevModeMultiple(t *testing.T) { + project := newTestProjectDev() + + sql := `SELECT o.*, c.name +FROM staging.stg_orders o +JOIN staging.stg_customers c ON o.id = c.id` + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + assert.Contains(t, rewritten, "FROM dev_fritz.staging_stg_orders o") + assert.Contains(t, rewritten, "JOIN dev_fritz.staging_stg_customers c") + assert.Contains(t, deps, "stg_orders") + assert.Contains(t, deps, "stg_customers") +} + +func TestRewriteTableReferences_NoMatch(t *testing.T) { + project := newTestProjectDev() + + // External table not in project — should not be rewritten + sql := "SELECT * FROM external_db.raw_events" + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + assert.Equal(t, "SELECT * FROM external_db.raw_events", rewritten) + assert.Empty(t, deps) +} + +func TestRewriteTableReferences_StringsNotRewritten(t *testing.T) { + project := newTestProjectDev() + + // Table name inside string literal should NOT be rewritten + sql := "SELECT 'staging.stg_orders' as table_name FROM staging.stg_orders" + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + assert.Contains(t, rewritten, "'staging.stg_orders'") // string preserved + assert.Contains(t, rewritten, "FROM dev_fritz.staging_stg_orders") // real ref rewritten + assert.Contains(t, deps, "stg_orders") +} + +func TestRewriteTableReferences_CommentsNotRewritten(t *testing.T) { + project := newTestProjectDev() + + sql := `-- FROM staging.stg_orders +SELECT * FROM staging.stg_customers` + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + assert.Contains(t, rewritten, "-- FROM staging.stg_orders") // comment preserved + assert.Contains(t, rewritten, "FROM dev_fritz.staging_stg_customers") + assert.Contains(t, deps, "stg_customers") +} + +func TestRewriteTableReferences_CTENotRewritten(t *testing.T) { + project := newTestProjectDev() + + // CTE name matching a model should not be rewritten when referenced + sql := `WITH stg_orders AS ( + SELECT 1 as id +) +SELECT * FROM stg_orders` + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + // The CTE reference should not be rewritten + assert.Contains(t, rewritten, "FROM stg_orders") + assert.NotContains(t, rewritten, "dev_fritz") + assert.Empty(t, deps) +} + +func TestRewriteTableReferences_UnqualifiedName(t *testing.T) { + project := newTestProjectDev() + + // Bare table name without schema should match by model name + sql := "SELECT * FROM stg_orders" + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + assert.Equal(t, "SELECT * FROM dev_fritz.staging_stg_orders", rewritten) + assert.Contains(t, deps, "stg_orders") +} + +func TestRewriteTableReferences_SelfNotRewritten(t *testing.T) { + project := newTestProjectDev() + + // Self-references should not be rewritten + sql := "SELECT * FROM staging.stg_orders" + rewritten, deps := RewriteTableReferences(sql, project, "stg_orders") + + assert.Equal(t, "SELECT * FROM staging.stg_orders", rewritten) + assert.Empty(t, deps) +} + +func TestRewriteTableReferences_DoubleQuoted(t *testing.T) { + project := newTestProjectDev() + + sql := `SELECT * FROM "staging"."stg_orders"` + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + // Quotes should be preserved in the replacement + assert.Equal(t, `SELECT * FROM "dev_fritz"."staging_stg_orders"`, rewritten) + assert.Contains(t, deps, "stg_orders") +} + +func TestRewriteTableReferences_Backticked(t *testing.T) { + project := newTestProjectDev() + + sql := "SELECT * FROM `staging`.`stg_orders`" + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + // Backticks should be preserved in the replacement + assert.Equal(t, "SELECT * FROM `dev_fritz`.`staging_stg_orders`", rewritten) + assert.Contains(t, deps, "stg_orders") +} + +func TestRewriteTableReferences_UnquotedPreserved(t *testing.T) { + project := newTestProjectDev() + + sql := "SELECT * FROM staging.stg_orders" + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + // No quotes in original, no quotes in replacement + assert.Equal(t, "SELECT * FROM dev_fritz.staging_stg_orders", rewritten) + assert.Contains(t, deps, "stg_orders") +} + +func TestRewriteTableReferences_MixedQuotesAndPlain(t *testing.T) { + project := newTestProjectDev() + + sql := `SELECT * FROM "staging"."stg_orders" o JOIN staging.stg_customers c ON o.id = c.id` + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + assert.Contains(t, rewritten, `"dev_fritz"."staging_stg_orders"`) + assert.Contains(t, rewritten, "dev_fritz.staging_stg_customers") + assert.Contains(t, deps, "stg_orders") + assert.Contains(t, deps, "stg_customers") +} + +func TestRewriteTableReferences_QuotedProdMode(t *testing.T) { + project := newTestProject() + + // In prod mode, quoted identifiers should still be preserved (identity rewrite) + sql := `SELECT * FROM "staging"."stg_orders"` + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + assert.Equal(t, `SELECT * FROM "staging"."stg_orders"`, rewritten) + assert.Contains(t, deps, "stg_orders") +} + +func TestRewriteTableReferences_SeedRewritten(t *testing.T) { + project := newTestProjectDev() + + sql := "SELECT * FROM staging.country_codes" + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + assert.Equal(t, "SELECT * FROM dev_fritz.staging_country_codes", rewritten) + assert.Contains(t, deps, "country_codes") +} + +func TestRewriteTableReferences_RefOutputNotDoubleRewritten(t *testing.T) { + project := newTestProjectDev() + + // In dev mode, ref() already returns the dev name. + // The rewriter should NOT match it again (dev name is not in the prod-name index). + sql := "SELECT * FROM dev_fritz.staging_stg_orders" + rewritten, deps := RewriteTableReferences(sql, project, "some_model") + + assert.Equal(t, "SELECT * FROM dev_fritz.staging_stg_orders", rewritten) + assert.Empty(t, deps) // no match since dev name != prod name +} + +// ============================================================================= +// protectLiterals Tests +// ============================================================================= + +func TestProtectLiterals(t *testing.T) { + tests := []struct { + name string + sql string + wantSafe bool // whether protected string contains no raw literals + }{ + {"single-quoted string", "SELECT 'hello' FROM t", true}, + {"line comment", "SELECT 1 -- comment\nFROM t", true}, + {"block comment", "SELECT /* block */ 1 FROM t", true}, + {"jinja expression", "SELECT {{ var }} FROM t", true}, + {"jinja block", "{% if true %}SELECT 1{% endif %}", true}, + {"escaped quote", "SELECT 'it''s' FROM t", true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + protected, placeholders := protectLiterals(tt.sql) + assert.True(t, len(placeholders) > 0) + restored := restoreLiterals(protected, placeholders) + assert.Equal(t, tt.sql, restored) // round-trip must be lossless + }) + } +} + +// ============================================================================= +// preprocessAtRefs Tests +// ============================================================================= + +func TestPreprocessAtRefs(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "test_model", + Schema: "staging", + FullTableName: "staging.test_model", + } + + result := te.preprocessAtRefs("SELECT * FROM @stg_orders o JOIN @stg_customers c ON o.id = c.id", model) + + assert.Contains(t, result, "staging.stg_orders") + assert.Contains(t, result, "staging.stg_customers") + assert.Contains(t, model.DependsOn, "stg_orders") + assert.Contains(t, model.DependsOn, "stg_customers") +} + +func TestPreprocessAtRefsDevMode(t *testing.T) { + project := newTestProjectDev() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "test_model", + Schema: "dev_fritz", + FullTableName: "dev_fritz.test_model", + } + + result := te.preprocessAtRefs("SELECT * FROM @stg_orders", model) + + // Should resolve to dev-mode FullTableName + assert.Contains(t, result, "dev_fritz.staging_stg_orders") + assert.Contains(t, model.DependsOn, "stg_orders") +} + +// ============================================================================= +// IncrementalContext placeholder tests (Phase 3.5) +// ============================================================================= + +func TestCompileModel_SlingStyle_Placeholder_Default(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "stg_orders", + Schema: "staging", + FullTableName: "staging.stg_orders", + RawSQL: "SELECT * FROM raw WHERE {incremental_where_cond}", + } + project.Models["stg_orders"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + assert.Contains(t, result, "WHERE 1=1") + assert.NotContains(t, result, "{incremental_where_cond}") +} + +func TestCompileModel_SlingStyle_Placeholder_Custom(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "stg_orders", + Schema: "staging", + FullTableName: "staging.stg_orders", + RawSQL: "SELECT * FROM raw WHERE {incremental_where_cond}", + } + project.Models["stg_orders"] = model + + ctx := &IncrementalContext{WhereCond: `"created_at" > '2024-01-01'`, IsIncremental: true} + result, err := te.CompileModel(model, ctx) + require.NoError(t, err) + assert.Contains(t, result, `"created_at" > '2024-01-01'`) + assert.NotContains(t, result, "{incremental_where_cond}") +} + +func TestCompileModel_SlingStyle_IncrementalValue(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "stg_orders", + Schema: "staging", + FullTableName: "staging.stg_orders", + RawSQL: "SELECT {incremental_value} AS watermark FROM raw", + } + project.Models["stg_orders"] = model + + ctx := &IncrementalContext{Value: "'2024-01-01'", IsIncremental: true} + result, err := te.CompileModel(model, ctx) + require.NoError(t, err) + assert.Contains(t, result, "'2024-01-01' AS watermark") + assert.NotContains(t, result, "{incremental_value}") +} + +func TestCompileModel_DbtStyle_IsIncrementalTrue(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "fct_orders", + Schema: "marts", + FullTableName: "marts.fct_orders", + RawSQL: `SELECT id FROM raw +{% if is_incremental() %} +WHERE updated_at > (SELECT MAX(updated_at) FROM {{ this }}) +{% endif %}`, + } + project.Models["fct_orders"] = model + + result, err := te.CompileModel(model, &IncrementalContext{IsIncremental: true}) + require.NoError(t, err) + assert.Contains(t, result, "WHERE updated_at > (SELECT MAX(updated_at) FROM marts.fct_orders)") +} + +func TestCompileModel_DbtStyle_IsIncrementalFalse(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "fct_orders", + Schema: "marts", + FullTableName: "marts.fct_orders", + RawSQL: `SELECT id FROM raw +{% if is_incremental() %} +WHERE updated_at > (SELECT MAX(updated_at) FROM {{ this }}) +{% endif %}`, + } + project.Models["fct_orders"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + assert.NotContains(t, result, "WHERE updated_at") +} + +func TestCompileModel_NilContext_UsesDefault(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "stg_orders", + Schema: "staging", + FullTableName: "staging.stg_orders", + RawSQL: "SELECT * FROM raw WHERE {incremental_where_cond}", + } + project.Models["stg_orders"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + // nil context → DefaultIncrementalContext → WhereCond="1=1" + assert.Contains(t, result, "WHERE 1=1") +} + +func TestCompileModel_ViewModel_NoPlaceholders_NoOp(t *testing.T) { + project := newTestProject() + te := NewTemplateEngine(project, nil) + + model := &Model{ + Name: "stg_orders", + Schema: "staging", + FullTableName: "staging.stg_orders", + RawSQL: "SELECT * FROM raw_orders", + } + project.Models["stg_orders"] = model + + result, err := te.CompileModel(model, nil) + require.NoError(t, err) + assert.Equal(t, "SELECT * FROM raw_orders", result) +} diff --git a/core/sling/build/tests.go b/core/sling/build/tests.go new file mode 100644 index 000000000..cf3947fde --- /dev/null +++ b/core/sling/build/tests.go @@ -0,0 +1,177 @@ +package build + +import ( + "fmt" + "strings" + + "github.com/flarco/g" + "github.com/spf13/cast" +) + +// executeModelTests runs declarative frontmatter data tests against the +// materialized table. Each test compiles to a SELECT count(*) that must be 0. +func (e *Executor) executeModelTests(model *Model) error { + if len(model.Config.Tests) == 0 { + g.Debug("model '%s': no tests defined", model.Name) + return nil + } + + quoted, err := e.quoteFullTableName(model.FullTableName) + if err != nil { + return err + } + + for i, raw := range model.Config.Tests { + testSQL, label, err := compileDataTest(raw, quoted, e.DbConn.Quote) + if err != nil { + return g.Error(err, "model '%s' test %d", model.Name, i+1) + } + g.Debug("running test %q on %s: %s", label, model.Name, testSQL) + + data, err := e.DbConn.Query(testSQL) + if err != nil { + return g.Error(err, "model '%s' test %q failed to execute", model.Name, label) + } + if len(data.Rows) == 0 || len(data.Rows[0]) == 0 { + continue + } + cnt := cast.ToInt64(data.Rows[0][0]) + if cnt > 0 { + return g.Error("model '%s' test %q failed: %d violating row(s)", model.Name, label, cnt) + } + } + return nil +} + +// compileDataTest turns a frontmatter test entry into a failure-count SQL query. +// Supported forms (YAML): +// +// - not_null: [id, customer_id] +// - unique: [id] +// - unique: id +// - accepted_values: {column: status, values: [a, b]} +// - expr: sum(amount) >= 0 +// - {not_null: id} (single column string) +func compileDataTest(raw any, quotedTable string, quote func(string) string) (sql, label string, err error) { + m, ok := raw.(map[string]any) + if !ok { + // YAML may produce map[any]any + if m2, ok2 := raw.(map[any]any); ok2 { + m = make(map[string]any, len(m2)) + for k, v := range m2 { + m[cast.ToString(k)] = v + } + } else { + return "", "", g.Error("test entry must be a mapping, got %T", raw) + } + } + + if v, ok := m["not_null"]; ok { + cols := toStringList(v) + if len(cols) == 0 { + return "", "", g.Error("not_null requires one or more columns") + } + parts := make([]string, len(cols)) + for i, c := range cols { + parts[i] = quote(c) + " IS NULL" + } + label = "not_null(" + strings.Join(cols, ", ") + ")" + sql = fmt.Sprintf("SELECT count(*) FROM %s WHERE %s", quotedTable, strings.Join(parts, " OR ")) + return sql, label, nil + } + + if v, ok := m["unique"]; ok { + cols := toStringList(v) + if len(cols) == 0 { + return "", "", g.Error("unique requires one or more columns") + } + qcols := make([]string, len(cols)) + for i, c := range cols { + qcols[i] = quote(c) + } + colList := strings.Join(qcols, ", ") + label = "unique(" + strings.Join(cols, ", ") + ")" + sql = fmt.Sprintf( + "SELECT count(*) FROM (SELECT %s FROM %s GROUP BY %s HAVING count(*) > 1) _sling_uniq", + colList, quotedTable, colList, + ) + return sql, label, nil + } + + if v, ok := m["accepted_values"]; ok { + av, ok := v.(map[string]any) + if !ok { + if m2, ok2 := v.(map[any]any); ok2 { + av = make(map[string]any, len(m2)) + for k, val := range m2 { + av[cast.ToString(k)] = val + } + } else { + return "", "", g.Error("accepted_values must be a mapping with column and values") + } + } + col := cast.ToString(av["column"]) + vals := toStringList(av["values"]) + if col == "" || len(vals) == 0 { + return "", "", g.Error("accepted_values requires column and values") + } + quotedVals := make([]string, len(vals)) + for i, val := range vals { + quotedVals[i] = "'" + strings.ReplaceAll(val, "'", "''") + "'" + } + label = "accepted_values(" + col + ")" + sql = fmt.Sprintf( + "SELECT count(*) FROM %s WHERE %s IS NOT NULL AND %s NOT IN (%s)", + quotedTable, quote(col), quote(col), strings.Join(quotedVals, ", "), + ) + return sql, label, nil + } + + if v, ok := m["expr"]; ok { + expr := strings.TrimSpace(cast.ToString(v)) + if expr == "" { + return "", "", g.Error("expr test requires a non-empty expression") + } + label = "expr(" + expr + ")" + // Fail count = 1 when expression is false (or null) + sql = fmt.Sprintf( + "SELECT CASE WHEN (%s) THEN 0 ELSE 1 END FROM %s", + expr, quotedTable, + ) + return sql, label, nil + } + + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + return "", "", g.Error("unknown test type in %v; expected not_null, unique, accepted_values, or expr", keys) +} + +// toStringList coerces a string or list of strings. +func toStringList(v any) []string { + if v == nil { + return nil + } + switch t := v.(type) { + case string: + if t == "" { + return nil + } + return []string{t} + case []string: + return t + case []any: + out := make([]string, 0, len(t)) + for _, item := range t { + out = append(out, cast.ToString(item)) + } + return out + default: + s := cast.ToString(v) + if s == "" { + return nil + } + return []string{s} + } +} diff --git a/tests/build/clickhouse_project/marts/core/fct_orders.sql b/tests/build/clickhouse_project/marts/core/fct_orders.sql new file mode 100644 index 000000000..acacdd209 --- /dev/null +++ b/tests/build/clickhouse_project/marts/core/fct_orders.sql @@ -0,0 +1,10 @@ +{%- config(mode='incremental', unique_key='id', merge_strategy='delete+insert', update_key='created_at') -%} + +SELECT + id, + name, + created_at +FROM {{ ref('stg_orders') }} +{% if is_incremental() %} +WHERE created_at > (SELECT max(created_at) FROM {{ this }}) +{% endif %} diff --git a/tests/build/clickhouse_project/sling_build.yml b/tests/build/clickhouse_project/sling_build.yml new file mode 100644 index 000000000..d4a1722bd --- /dev/null +++ b/tests/build/clickhouse_project/sling_build.yml @@ -0,0 +1,4 @@ +target: CLICKHOUSE + +defaults: + mode: full-refresh diff --git a/tests/build/clickhouse_project/staging/stg_orders.sql b/tests/build/clickhouse_project/staging/stg_orders.sql new file mode 100644 index 000000000..1cadb00a1 --- /dev/null +++ b/tests/build/clickhouse_project/staging/stg_orders.sql @@ -0,0 +1,3 @@ +SELECT 1 as id, 'order_1' as name, toDate('2024-01-01') as created_at +UNION ALL +SELECT 2, 'order_2', toDate('2024-01-02') diff --git a/tests/build/cycle_project/sling_build.yml b/tests/build/cycle_project/sling_build.yml new file mode 100644 index 000000000..8bee91340 --- /dev/null +++ b/tests/build/cycle_project/sling_build.yml @@ -0,0 +1 @@ +target: POSTGRES diff --git a/tests/build/cycle_project/staging/model_a.sql b/tests/build/cycle_project/staging/model_a.sql new file mode 100644 index 000000000..db1147ab7 --- /dev/null +++ b/tests/build/cycle_project/staging/model_a.sql @@ -0,0 +1 @@ +SELECT * FROM {{ ref('model_b') }} diff --git a/tests/build/cycle_project/staging/model_b.sql b/tests/build/cycle_project/staging/model_b.sql new file mode 100644 index 000000000..ff0c2626f --- /dev/null +++ b/tests/build/cycle_project/staging/model_b.sql @@ -0,0 +1 @@ +SELECT * FROM {{ ref('model_a') }} diff --git a/tests/build/dbt_compat_project/models/staging/stg_orders.sql b/tests/build/dbt_compat_project/models/staging/stg_orders.sql new file mode 100644 index 000000000..1d40fcf4c --- /dev/null +++ b/tests/build/dbt_compat_project/models/staging/stg_orders.sql @@ -0,0 +1 @@ +SELECT 1 as id, 'order_1' as name diff --git a/tests/build/dbt_compat_project/seeds/staging/country_codes.csv b/tests/build/dbt_compat_project/seeds/staging/country_codes.csv new file mode 100644 index 000000000..f68a70ac1 --- /dev/null +++ b/tests/build/dbt_compat_project/seeds/staging/country_codes.csv @@ -0,0 +1,3 @@ +id,code,name +1,US,United States +2,CA,Canada diff --git a/tests/build/dbt_compat_project/sling_build.yml b/tests/build/dbt_compat_project/sling_build.yml new file mode 100644 index 000000000..1a8e90ee0 --- /dev/null +++ b/tests/build/dbt_compat_project/sling_build.yml @@ -0,0 +1,2 @@ +target: POSTGRES +dbt_project: true diff --git a/tests/build/defaults_expanded_project/marts/fct_orders.sql b/tests/build/defaults_expanded_project/marts/fct_orders.sql new file mode 100644 index 000000000..f99a9d34b --- /dev/null +++ b/tests/build/defaults_expanded_project/marts/fct_orders.sql @@ -0,0 +1,12 @@ +/** +mode: full-refresh +tags: [marts] +hooks: + start: + - type: log + message: "fct_orders frontmatter start hook" +**/ +SELECT + id, + name +FROM {{ ref('stg_orders') }} diff --git a/tests/build/defaults_expanded_project/sling_build.yml b/tests/build/defaults_expanded_project/sling_build.yml new file mode 100644 index 000000000..4da67ec00 --- /dev/null +++ b/tests/build/defaults_expanded_project/sling_build.yml @@ -0,0 +1,9 @@ +target: POSTGRES + +defaults: + schema: raw + tags: [core] + hooks: + start: + - type: log + message: "default start hook" diff --git a/tests/build/defaults_expanded_project/staging/sling_build.yml b/tests/build/defaults_expanded_project/staging/sling_build.yml new file mode 100644 index 000000000..e8d5c91d3 --- /dev/null +++ b/tests/build/defaults_expanded_project/staging/sling_build.yml @@ -0,0 +1,6 @@ +defaults: + mode: incremental + unique_key: id + update_key: updated_at + merge_strategy: delete+insert + tags: [staging] diff --git a/tests/build/defaults_expanded_project/staging/stg_disabled.sql b/tests/build/defaults_expanded_project/staging/stg_disabled.sql new file mode 100644 index 000000000..42c1b67bb --- /dev/null +++ b/tests/build/defaults_expanded_project/staging/stg_disabled.sql @@ -0,0 +1,4 @@ +/** +enabled: false +**/ +SELECT 1 AS id diff --git a/tests/build/defaults_expanded_project/staging/stg_orders.sql b/tests/build/defaults_expanded_project/staging/stg_orders.sql new file mode 100644 index 000000000..a8b08cfac --- /dev/null +++ b/tests/build/defaults_expanded_project/staging/stg_orders.sql @@ -0,0 +1,5 @@ +SELECT + 1 AS id, + 'order_a' AS name, + '2024-01-01'::timestamp AS updated_at +WHERE {incremental_where_cond} diff --git a/tests/build/duckdb_parallel_project/seeds/customers.csv b/tests/build/duckdb_parallel_project/seeds/customers.csv new file mode 100644 index 000000000..10d11582c --- /dev/null +++ b/tests/build/duckdb_parallel_project/seeds/customers.csv @@ -0,0 +1,3 @@ +id,name +1,alice +2,bob diff --git a/tests/build/duckdb_parallel_project/seeds/orders.csv b/tests/build/duckdb_parallel_project/seeds/orders.csv new file mode 100644 index 000000000..81fefbb0d --- /dev/null +++ b/tests/build/duckdb_parallel_project/seeds/orders.csv @@ -0,0 +1,3 @@ +id,name +1,order_a +2,order_b diff --git a/tests/build/duckdb_parallel_project/sling_build.yml b/tests/build/duckdb_parallel_project/sling_build.yml new file mode 100644 index 000000000..586822b79 --- /dev/null +++ b/tests/build/duckdb_parallel_project/sling_build.yml @@ -0,0 +1,4 @@ +target: DUCK_PARALLEL + +defaults: + mode: full-refresh diff --git a/tests/build/duckdb_parallel_project/staging/stg_ok.sql b/tests/build/duckdb_parallel_project/staging/stg_ok.sql new file mode 100644 index 000000000..f39920c68 --- /dev/null +++ b/tests/build/duckdb_parallel_project/staging/stg_ok.sql @@ -0,0 +1,4 @@ +/** +mode: full-refresh +**/ +SELECT 1 AS id, 'ok' AS label diff --git a/tests/build/duplicate_names_project/archive/stg_orders.sql b/tests/build/duplicate_names_project/archive/stg_orders.sql new file mode 100644 index 000000000..61cab9c38 --- /dev/null +++ b/tests/build/duplicate_names_project/archive/stg_orders.sql @@ -0,0 +1 @@ +SELECT 1 as id, 'old_order' as name diff --git a/tests/build/duplicate_names_project/sling_build.yml b/tests/build/duplicate_names_project/sling_build.yml new file mode 100644 index 000000000..8bee91340 --- /dev/null +++ b/tests/build/duplicate_names_project/sling_build.yml @@ -0,0 +1 @@ +target: POSTGRES diff --git a/tests/build/duplicate_names_project/staging/stg_orders.sql b/tests/build/duplicate_names_project/staging/stg_orders.sql new file mode 100644 index 000000000..1d40fcf4c --- /dev/null +++ b/tests/build/duplicate_names_project/staging/stg_orders.sql @@ -0,0 +1 @@ +SELECT 1 as id, 'order_1' as name diff --git a/tests/build/hooks_project/marts/fct_orders.sql b/tests/build/hooks_project/marts/fct_orders.sql new file mode 100644 index 000000000..1c1e44d0a --- /dev/null +++ b/tests/build/hooks_project/marts/fct_orders.sql @@ -0,0 +1,17 @@ +/** +mode: full-refresh +hooks: + start: + - type: log + message: "building fct_orders from stg_orders" + - type: query + connection: POSTGRES + query: "SELECT 1" + end: + - type: log + message: "fct_orders build complete" +**/ +SELECT + id, + name +FROM {{ ref('stg_orders') }} diff --git a/tests/build/hooks_project/sling_build.yml b/tests/build/hooks_project/sling_build.yml new file mode 100644 index 000000000..c3fbf9e06 --- /dev/null +++ b/tests/build/hooks_project/sling_build.yml @@ -0,0 +1,4 @@ +target: POSTGRES + +defaults: + mode: full-refresh diff --git a/tests/build/hooks_project/staging/stg_orders.sql b/tests/build/hooks_project/staging/stg_orders.sql new file mode 100644 index 000000000..a379e550c --- /dev/null +++ b/tests/build/hooks_project/staging/stg_orders.sql @@ -0,0 +1,14 @@ +/** +hooks: + start: + - type: log + message: "building stg_orders model" + level: warn + end: + - type: log + message: "finished stg_orders model" + level: warn +**/ +SELECT 1 as id, 'order_1' as name +UNION ALL +SELECT 2, 'order_2' diff --git a/tests/build/macro_project/marts/product_margins.sql b/tests/build/macro_project/marts/product_margins.sql new file mode 100644 index 000000000..220502a28 --- /dev/null +++ b/tests/build/macro_project/marts/product_margins.sql @@ -0,0 +1,9 @@ +/** +mode: full-refresh +**/ +SELECT + id, + name, + {{ cents_to_dollars('price_cents') }} as price_dollars, + {{ safe_divide('price_cents', '100') }} as price_ratio +FROM {{ ref('stg_products') }} diff --git a/tests/build/macro_project/sling_build.yml b/tests/build/macro_project/sling_build.yml new file mode 100644 index 000000000..c3fbf9e06 --- /dev/null +++ b/tests/build/macro_project/sling_build.yml @@ -0,0 +1,4 @@ +target: POSTGRES + +defaults: + mode: full-refresh diff --git a/tests/build/macro_project/staging/staging_helpers.macros.sql b/tests/build/macro_project/staging/staging_helpers.macros.sql new file mode 100644 index 000000000..ead534b87 --- /dev/null +++ b/tests/build/macro_project/staging/staging_helpers.macros.sql @@ -0,0 +1,7 @@ +{% macro clean_string(column_name) %} + TRIM(LOWER({{ column_name }})) +{% endmacro %} + +{% macro null_if_empty(column_name) %} + CASE WHEN {{ column_name }} = '' THEN NULL ELSE {{ column_name }} END +{% endmacro %} diff --git a/tests/build/macro_project/staging/stg_products.sql b/tests/build/macro_project/staging/stg_products.sql new file mode 100644 index 000000000..b4e46076e --- /dev/null +++ b/tests/build/macro_project/staging/stg_products.sql @@ -0,0 +1,11 @@ +SELECT + 1 as id, + {{ clean_string("' Widget A '") }} as name, + {{ null_if_empty("'active'") }} as status, + 1500 as price_cents +UNION ALL +SELECT + 2, + {{ clean_string("' Widget B '") }}, + {{ null_if_empty("''") }}, + 2500 diff --git a/tests/build/macro_project/utils.macros.sql b/tests/build/macro_project/utils.macros.sql new file mode 100644 index 000000000..78ffd711d --- /dev/null +++ b/tests/build/macro_project/utils.macros.sql @@ -0,0 +1,7 @@ +{% macro cents_to_dollars(column_name) %} + ({{ column_name }} / 100.0) +{% endmacro %} + +{% macro safe_divide(numerator, denominator) %} + CASE WHEN {{ denominator }} = 0 THEN NULL ELSE {{ numerator }}::float / {{ denominator }} END +{% endmacro %} diff --git a/tests/build/multi_statement_project/sling_build.yml b/tests/build/multi_statement_project/sling_build.yml new file mode 100644 index 000000000..c3fbf9e06 --- /dev/null +++ b/tests/build/multi_statement_project/sling_build.yml @@ -0,0 +1,4 @@ +target: POSTGRES + +defaults: + mode: full-refresh diff --git a/tests/build/multi_statement_project/staging/stg_multi.sql b/tests/build/multi_statement_project/staging/stg_multi.sql new file mode 100644 index 000000000..c2f69d12a --- /dev/null +++ b/tests/build/multi_statement_project/staging/stg_multi.sql @@ -0,0 +1,8 @@ +-- Pre-statement: create a temp table for staging +CREATE TEMP TABLE tmp_raw_data AS SELECT 1 AS id, 'alice' AS name; + +-- This is the model query +SELECT id, name FROM tmp_raw_data; + +-- Post-statement: clean up +DROP TABLE IF EXISTS tmp_raw_data; diff --git a/tests/build/multi_target_project/warehouse_a/sling_build.yml b/tests/build/multi_target_project/warehouse_a/sling_build.yml new file mode 100644 index 000000000..8bee91340 --- /dev/null +++ b/tests/build/multi_target_project/warehouse_a/sling_build.yml @@ -0,0 +1 @@ +target: POSTGRES diff --git a/tests/build/multi_target_project/warehouse_a/staging/stg_orders.sql b/tests/build/multi_target_project/warehouse_a/staging/stg_orders.sql new file mode 100644 index 000000000..7a3248ddb --- /dev/null +++ b/tests/build/multi_target_project/warehouse_a/staging/stg_orders.sql @@ -0,0 +1,6 @@ +SELECT + g.id, + 'order_' || g.id::text as name, + now() - (random() * interval '365 days') as created_at, + round((random() * 1000)::numeric, 2) as amount +FROM generate_series(1, 500000) as g(id) diff --git a/tests/build/multi_target_project/warehouse_b/sling_build.yml b/tests/build/multi_target_project/warehouse_b/sling_build.yml new file mode 100644 index 000000000..103729b29 --- /dev/null +++ b/tests/build/multi_target_project/warehouse_b/sling_build.yml @@ -0,0 +1 @@ +target: CLICKHOUSE diff --git a/tests/build/multi_target_project/warehouse_b/staging/stg_events.sql b/tests/build/multi_target_project/warehouse_b/staging/stg_events.sql new file mode 100644 index 000000000..c584a7c5c --- /dev/null +++ b/tests/build/multi_target_project/warehouse_b/staging/stg_events.sql @@ -0,0 +1 @@ +SELECT 1 as id, 'event_1' as name diff --git a/tests/build/nested_yml_project/marts/core/dim_customers.sql b/tests/build/nested_yml_project/marts/core/dim_customers.sql new file mode 100644 index 000000000..dbc57d4cd --- /dev/null +++ b/tests/build/nested_yml_project/marts/core/dim_customers.sql @@ -0,0 +1 @@ +SELECT 1 as id, 'customer_1' as name diff --git a/tests/build/nested_yml_project/sling_build.yml b/tests/build/nested_yml_project/sling_build.yml new file mode 100644 index 000000000..c3fbf9e06 --- /dev/null +++ b/tests/build/nested_yml_project/sling_build.yml @@ -0,0 +1,4 @@ +target: POSTGRES + +defaults: + mode: full-refresh diff --git a/tests/build/nested_yml_project/staging/sling_build.yml b/tests/build/nested_yml_project/staging/sling_build.yml new file mode 100644 index 000000000..cabb33bd9 --- /dev/null +++ b/tests/build/nested_yml_project/staging/sling_build.yml @@ -0,0 +1,2 @@ +defaults: + mode: truncate diff --git a/tests/build/nested_yml_project/staging/stg_orders.sql b/tests/build/nested_yml_project/staging/stg_orders.sql new file mode 100644 index 000000000..1d40fcf4c --- /dev/null +++ b/tests/build/nested_yml_project/staging/stg_orders.sql @@ -0,0 +1 @@ +SELECT 1 as id, 'order_1' as name diff --git a/tests/build/pipeline_step_project/marts/fct_orders.sql b/tests/build/pipeline_step_project/marts/fct_orders.sql new file mode 100644 index 000000000..184f90a79 --- /dev/null +++ b/tests/build/pipeline_step_project/marts/fct_orders.sql @@ -0,0 +1,5 @@ +/** +mode: full-refresh +**/ +SELECT id, name, run_label +FROM {{ ref('stg_orders') }} diff --git a/tests/build/pipeline_step_project/sling_build.yml b/tests/build/pipeline_step_project/sling_build.yml new file mode 100644 index 000000000..39a536eed --- /dev/null +++ b/tests/build/pipeline_step_project/sling_build.yml @@ -0,0 +1,7 @@ +target: POSTGRES + +defaults: + mode: full-refresh + +vars: + run_label: default diff --git a/tests/build/pipeline_step_project/staging/stg_orders.sql b/tests/build/pipeline_step_project/staging/stg_orders.sql new file mode 100644 index 000000000..b0f5d6ec9 --- /dev/null +++ b/tests/build/pipeline_step_project/staging/stg_orders.sql @@ -0,0 +1,6 @@ +/** +mode: full-refresh +**/ +SELECT 1 AS id, 'order_1' AS name, '{{ run_label }}' AS run_label +UNION ALL +SELECT 2, 'order_2', '{{ run_label }}' diff --git a/tests/build/range_bad_dbt_with_range/models/bad_model.sql b/tests/build/range_bad_dbt_with_range/models/bad_model.sql new file mode 100644 index 000000000..d4ebdf7ee --- /dev/null +++ b/tests/build/range_bad_dbt_with_range/models/bad_model.sql @@ -0,0 +1,11 @@ +/** +mode: incremental +unique_key: id +update_key: created_at +range: + advance: 7d +**/ +SELECT 1 AS id, '2024-01-01'::date AS created_at +{% if is_incremental() %} +WHERE created_at > (SELECT MAX(created_at) FROM {{ this }}) +{% endif %} diff --git a/tests/build/range_bad_dbt_with_range/sling_build.yml b/tests/build/range_bad_dbt_with_range/sling_build.yml new file mode 100644 index 000000000..776e6aa65 --- /dev/null +++ b/tests/build/range_bad_dbt_with_range/sling_build.yml @@ -0,0 +1,5 @@ +target: POSTGRES +dbt_project: true + +defaults: + mode: full-refresh diff --git a/tests/build/range_bad_mixed_style/sling_build.yml b/tests/build/range_bad_mixed_style/sling_build.yml new file mode 100644 index 000000000..c3fbf9e06 --- /dev/null +++ b/tests/build/range_bad_mixed_style/sling_build.yml @@ -0,0 +1,4 @@ +target: POSTGRES + +defaults: + mode: full-refresh diff --git a/tests/build/range_bad_mixed_style/staging/bad_model.sql b/tests/build/range_bad_mixed_style/staging/bad_model.sql new file mode 100644 index 000000000..c8f8280c7 --- /dev/null +++ b/tests/build/range_bad_mixed_style/staging/bad_model.sql @@ -0,0 +1,10 @@ +/** +mode: incremental +unique_key: id +update_key: created_at +**/ +SELECT 1 AS id, '2024-01-01'::date AS created_at +{% if is_incremental() %} +WHERE created_at > (SELECT MAX(created_at) FROM {{ this }}) +{% endif %} +AND {incremental_where_cond} diff --git a/tests/build/range_bad_start_no_advance/sling_build.yml b/tests/build/range_bad_start_no_advance/sling_build.yml new file mode 100644 index 000000000..c3fbf9e06 --- /dev/null +++ b/tests/build/range_bad_start_no_advance/sling_build.yml @@ -0,0 +1,4 @@ +target: POSTGRES + +defaults: + mode: full-refresh diff --git a/tests/build/range_bad_start_no_advance/staging/bad_model.sql b/tests/build/range_bad_start_no_advance/staging/bad_model.sql new file mode 100644 index 000000000..efad98abe --- /dev/null +++ b/tests/build/range_bad_start_no_advance/staging/bad_model.sql @@ -0,0 +1,9 @@ +/** +mode: incremental +unique_key: id +update_key: created_at +range: + start: '2024-01-01' +**/ +SELECT 1 AS id, '2024-01-01'::date AS created_at +WHERE {incremental_where_cond} diff --git a/tests/build/range_test_project/.gitignore b/tests/build/range_test_project/.gitignore new file mode 100644 index 000000000..c5077a371 --- /dev/null +++ b/tests/build/range_test_project/.gitignore @@ -0,0 +1 @@ +.sling_state/ diff --git a/tests/build/range_test_project/range_test/fact_orders.sql b/tests/build/range_test_project/range_test/fact_orders.sql new file mode 100644 index 000000000..acb02171f --- /dev/null +++ b/tests/build/range_test_project/range_test/fact_orders.sql @@ -0,0 +1,9 @@ +/** +mode: incremental +unique_key: id +update_key: created_at +merge_strategy: delete+insert +**/ +SELECT id, name, created_at::date AS created_at +FROM {{ ref('stg_orders') }} +WHERE {incremental_where_cond} diff --git a/tests/build/range_test_project/range_test/fact_orders_lookback.sql b/tests/build/range_test_project/range_test/fact_orders_lookback.sql new file mode 100644 index 000000000..bf8562b65 --- /dev/null +++ b/tests/build/range_test_project/range_test/fact_orders_lookback.sql @@ -0,0 +1,11 @@ +/** +mode: incremental +unique_key: id +update_key: created_at +merge_strategy: delete+insert +range: + lookback: 2d +**/ +SELECT id, name, created_at::date AS created_at +FROM {{ ref('stg_orders') }} +WHERE {incremental_where_cond} diff --git a/tests/build/range_test_project/range_test/fact_orders_paged.sql b/tests/build/range_test_project/range_test/fact_orders_paged.sql new file mode 100644 index 000000000..05ee2d5d7 --- /dev/null +++ b/tests/build/range_test_project/range_test/fact_orders_paged.sql @@ -0,0 +1,11 @@ +/** +mode: incremental +unique_key: id +update_key: created_at +merge_strategy: delete+insert +range: + advance: 7d +**/ +SELECT id, name, created_at::date AS created_at +FROM {{ ref('stg_orders') }} +WHERE {incremental_where_cond} diff --git a/tests/build/range_test_project/range_test/fact_orders_paged_start.sql b/tests/build/range_test_project/range_test/fact_orders_paged_start.sql new file mode 100644 index 000000000..2018e253a --- /dev/null +++ b/tests/build/range_test_project/range_test/fact_orders_paged_start.sql @@ -0,0 +1,12 @@ +/** +mode: incremental +unique_key: id +update_key: created_at +merge_strategy: delete+insert +range: + start: '2024-01-01' + advance: 7d +**/ +SELECT id, name, created_at::date AS created_at +FROM {{ ref('stg_orders') }} +WHERE {incremental_where_cond} diff --git a/tests/build/range_test_project/range_test/stg_orders.csv b/tests/build/range_test_project/range_test/stg_orders.csv new file mode 100644 index 000000000..255dea380 --- /dev/null +++ b/tests/build/range_test_project/range_test/stg_orders.csv @@ -0,0 +1,31 @@ +id,name,created_at +1,order_01,2024-01-02 +2,order_02,2024-01-05 +3,order_03,2024-01-09 +4,order_04,2024-01-12 +5,order_05,2024-01-16 +6,order_06,2024-01-20 +7,order_07,2024-01-23 +8,order_08,2024-01-27 +9,order_09,2024-01-30 +10,order_10,2024-02-02 +11,order_11,2024-02-05 +12,order_12,2024-02-09 +13,order_13,2024-02-12 +14,order_14,2024-02-16 +15,order_15,2024-02-19 +16,order_16,2024-02-23 +17,order_17,2024-02-26 +18,order_18,2024-02-29 +19,order_19,2024-03-03 +20,order_20,2024-03-06 +21,order_21,2024-03-10 +22,order_22,2024-03-13 +23,order_23,2024-03-16 +24,order_24,2024-03-19 +25,order_25,2024-03-22 +26,order_26,2024-03-25 +27,order_27,2024-03-27 +28,order_28,2024-03-29 +29,order_29,2024-03-30 +30,order_30,2024-03-31 diff --git a/tests/build/range_test_project/sling_build.yml b/tests/build/range_test_project/sling_build.yml new file mode 100644 index 000000000..c3fbf9e06 --- /dev/null +++ b/tests/build/range_test_project/sling_build.yml @@ -0,0 +1,4 @@ +target: POSTGRES + +defaults: + mode: full-refresh diff --git a/tests/build/sample_project/marts/core/dim_customers.sql b/tests/build/sample_project/marts/core/dim_customers.sql new file mode 100644 index 000000000..3b0ae42d8 --- /dev/null +++ b/tests/build/sample_project/marts/core/dim_customers.sql @@ -0,0 +1,7 @@ +{%- config(mode='view') -%} + +SELECT + id, + name, + status +FROM {{ ref('stg_customers') }} diff --git a/tests/build/sample_project/marts/core/fct_orders.sql b/tests/build/sample_project/marts/core/fct_orders.sql new file mode 100644 index 000000000..c1b78cfc9 --- /dev/null +++ b/tests/build/sample_project/marts/core/fct_orders.sql @@ -0,0 +1,10 @@ +{%- config(mode='incremental', unique_key='id', merge_strategy='delete+insert', update_key='created_at') -%} + +SELECT + id, + name, + created_at +FROM {{ ref('stg_orders') }} +{% if is_incremental() %} +WHERE created_at > (SELECT MAX(created_at) FROM {{ this }}) +{% endif %} diff --git a/tests/build/sample_project/marts/finance/revenue.sql b/tests/build/sample_project/marts/finance/revenue.sql new file mode 100644 index 000000000..e88f017bd --- /dev/null +++ b/tests/build/sample_project/marts/finance/revenue.sql @@ -0,0 +1,8 @@ +/** +mode: view +tags: + - finance +**/ +SELECT + count(*) as total_orders +FROM {{ ref('fct_orders') }} diff --git a/tests/build/sample_project/raw.sql b/tests/build/sample_project/raw.sql new file mode 100644 index 000000000..9177d006a --- /dev/null +++ b/tests/build/sample_project/raw.sql @@ -0,0 +1 @@ +SELECT 1 as id, 'raw_data' as value diff --git a/tests/build/sample_project/seeds/status_map.json b/tests/build/sample_project/seeds/status_map.json new file mode 100644 index 000000000..1d041f66a --- /dev/null +++ b/tests/build/sample_project/seeds/status_map.json @@ -0,0 +1,4 @@ +[ + {"id": 1, "status": "active", "label": "Active"}, + {"id": 2, "status": "inactive", "label": "Inactive"} +] diff --git a/tests/build/sample_project/sling_build.yml b/tests/build/sample_project/sling_build.yml new file mode 100644 index 000000000..8330e0f8b --- /dev/null +++ b/tests/build/sample_project/sling_build.yml @@ -0,0 +1,5 @@ +target: POSTGRES + +defaults: + mode: full-refresh + drop_cascade: true # needed so re-runs can drop staging under mart views diff --git a/tests/build/sample_project/staging/country_codes.csv b/tests/build/sample_project/staging/country_codes.csv new file mode 100644 index 000000000..80867a22a --- /dev/null +++ b/tests/build/sample_project/staging/country_codes.csv @@ -0,0 +1,4 @@ +id,code,name +1,US,United States +2,CA,Canada +3,GB,United Kingdom diff --git a/tests/build/sample_project/staging/staging_helpers.macros.sql b/tests/build/sample_project/staging/staging_helpers.macros.sql new file mode 100644 index 000000000..a2459e88b --- /dev/null +++ b/tests/build/sample_project/staging/staging_helpers.macros.sql @@ -0,0 +1,3 @@ +{% macro clean_string(column_name) %} + TRIM(LOWER({{ column_name }})) +{% endmacro %} diff --git a/tests/build/sample_project/staging/stg_customers.sql b/tests/build/sample_project/staging/stg_customers.sql new file mode 100644 index 000000000..201b203d5 --- /dev/null +++ b/tests/build/sample_project/staging/stg_customers.sql @@ -0,0 +1,3 @@ +SELECT 1 as id, 'customer_1' as name, 'active' as status +UNION ALL +SELECT 2, 'customer_2', 'inactive' diff --git a/tests/build/sample_project/staging/stg_orders.sql b/tests/build/sample_project/staging/stg_orders.sql new file mode 100644 index 000000000..87d634201 --- /dev/null +++ b/tests/build/sample_project/staging/stg_orders.sql @@ -0,0 +1,3 @@ +SELECT 1 as id, 'order_1' as name, '2024-01-01'::date as created_at +UNION ALL +SELECT 2, 'order_2', '2024-01-02'::date diff --git a/tests/build/sample_project/utils.macros.sql b/tests/build/sample_project/utils.macros.sql new file mode 100644 index 000000000..78ffd711d --- /dev/null +++ b/tests/build/sample_project/utils.macros.sql @@ -0,0 +1,7 @@ +{% macro cents_to_dollars(column_name) %} + ({{ column_name }} / 100.0) +{% endmacro %} + +{% macro safe_divide(numerator, denominator) %} + CASE WHEN {{ denominator }} = 0 THEN NULL ELSE {{ numerator }}::float / {{ denominator }} END +{% endmacro %} diff --git a/tests/pipelines/p.45.build_step.yaml b/tests/pipelines/p.45.build_step.yaml new file mode 100644 index 000000000..5e6f81c50 --- /dev/null +++ b/tests/pipelines/p.45.build_step.yaml @@ -0,0 +1,81 @@ +# Pipeline with type: build — load-adjacent transform in one YAML (review §5.3). +steps: + - type: log + message: 'starting pipeline build step test' + + # Full form: type + build path + options + - type: build + id: build + build: tests/build/pipeline_step_project + target: POSTGRES + full_refresh: true + fail_fast: true + threads: 2 + select: + - stg_orders + - fct_orders + vars: + run_label: from_pipeline + + - type: log + message: 'build total={state.build.total} ok={state.build.ok} failed={state.build.failed} skipped={state.build.skipped}' + + - type: check + check: state.build.total >= 2 + failure_message: 'expected at least 2 build results, got total={state.build.total}' + + - type: check + check: state.build.failed == 0 + failure_message: 'build reported failures: {state.build.failed}' + + - type: check + check: state.build.ok >= 2 + failure_message: 'expected ok >= 2, got {state.build.ok}' + + - type: query + connection: POSTGRES + query: | + SELECT count(*) AS cnt, + max(run_label) AS run_label + FROM marts.fct_orders + into: fct + + - type: check + check: store.fct[0].cnt == 2 + failure_message: 'expected 2 rows in marts.fct_orders' + + - type: check + check: store.fct[0].run_label == "from_pipeline" + failure_message: 'vars not passed into model (run_label={store.fct[0].run_label})' + + # Shortcut form: type inferred from `build:` key (like `replication:`) + - build: tests/build/pipeline_step_project + id: build_shortcut + target: POSTGRES + full_refresh: true + select: stg_orders + vars: + run_label: from_shortcut + + - type: check + check: state.build_shortcut.ok >= 1 + failure_message: 'shortcut build: form did not run' + + - type: query + connection: POSTGRES + query: SELECT max(run_label) AS run_label FROM staging.stg_orders + into: stg_shortcut + + - type: check + check: store.stg_shortcut[0].run_label == "from_shortcut" + failure_message: 'shortcut build vars not applied' + + - type: log + message: 'SUCCESS: pipeline type:build step completed (rows={store.fct[0].cnt}, label={store.fct[0].run_label})' + + # Cleanup + - type: query + connection: POSTGRES + query: | + DROP TABLE IF EXISTS marts.fct_orders; + DROP TABLE IF EXISTS staging.stg_orders; diff --git a/tests/pipelines/p.46.build_step_replication_hook.yaml b/tests/pipelines/p.46.build_step_replication_hook.yaml new file mode 100644 index 000000000..ea6c46e44 --- /dev/null +++ b/tests/pipelines/p.46.build_step_replication_hook.yaml @@ -0,0 +1,56 @@ +# Replication end-hook runs type: build after the load (review §5.3). +source: local +target: POSTGRES + +defaults: + mode: full-refresh + +hooks: + start: + - type: query + connection: '{target.name}' + query: | + CREATE SCHEMA IF NOT EXISTS build_hook; + DROP TABLE IF EXISTS build_hook.raw_src; + CREATE TABLE build_hook.raw_src (id int, name text); + + end: + - type: check + check: execution.status.error == 0 + on_failure: break + + - build: tests/build/pipeline_step_project + id: transform + target: POSTGRES + full_refresh: true + select: stg_orders + vars: + run_label: from_repl_hook + + - type: query + connection: '{target.name}' + query: SELECT count(*) AS cnt, max(run_label) AS run_label FROM staging.stg_orders + into: stg + + - type: check + check: store.stg[0].cnt == 2 + failure_message: 'build end-hook did not materialize stg_orders' + + - type: check + check: state.transform.ok >= 1 + failure_message: 'transform step state missing ok count' + + - type: log + message: 'SUCCESS: replication end-hook type:build ok (rows={store.stg[0].cnt}, label={store.stg[0].run_label})' + + - type: query + connection: '{target.name}' + query: | + DROP TABLE IF EXISTS staging.stg_orders; + DROP TABLE IF EXISTS marts.fct_orders; + DROP TABLE IF EXISTS build_hook.raw_src; + +streams: + tests/files/test1.csv: + object: build_hook.raw_src + mode: full-refresh diff --git a/tests/suite.cli.build.yaml b/tests/suite.cli.build.yaml new file mode 100644 index 000000000..9ce4e90a7 --- /dev/null +++ b/tests/suite.cli.build.yaml @@ -0,0 +1,528 @@ + + +- id: 400 + name: 'sling build execute nested_yml_project with config inheritance' + run: 'sling build tests/build/nested_yml_project --target POSTGRES --full-refresh' + group: build + output_contains: + - 'OK' + - 'Build Completed' + + +- id: 401 + name: 'sling build execute dbt_compat_project' + run: 'sling build tests/build/dbt_compat_project --target POSTGRES --full-refresh' + group: build + after: [400] + output_contains: + - 'seed' + - 'OK' + - 'Build Completed' + + +- id: 402 + name: 'sling build execute -R multi_target_project' + run: 'sling build tests/build/multi_target_project -R --full-refresh' + group: build + after: [401] + output_contains: + - 'stg_orders' + - 'stg_events' + - 'OK' + - 'Build Completed' + +# Multi-statement SQL models (pre/post statements) + +- id: 403 + name: 'sling build --compile multi_statement_project shows pre/post' + run: 'sling build tests/build/multi_statement_project --compile --target POSTGRES' + output_contains: + - 'stg_multi' + - 'mode: full-refresh' + - 'pre_statements:' + - 'CREATE TEMPORARY TABLE' + - 'sql:' + - 'post_statements:' + - 'DROP TABLE IF EXISTS' + + +- id: 404 + name: 'sling build execute multi_statement_project' + run: 'sling build tests/build/multi_statement_project --target POSTGRES --full-refresh' + group: build + after: [402] + output_contains: + - 'stg_multi' + - 'OK' + - 'Build Completed' + +# Macro usage in models + +- id: 405 + name: 'sling build --compile macro_project shows expanded macros' + run: 'sling build tests/build/macro_project --compile --target POSTGRES' + output_contains: + - 'stg_products' + - 'product_margins' + - 'mode: full-refresh' + - 'dependencies: [stg_products]' + - 'TRIM(LOWER(' + - 'CASE WHEN' + - 'price_cents / 100.0' + - 'sql:' + + +- id: 406 + name: 'sling build execute macro_project' + run: 'sling build tests/build/macro_project --target POSTGRES --full-refresh' + output_contains: + - 'stg_products' + - 'product_margins' + - 'OK' + - 'Build Completed' + +# Model hooks (start/end) + +- id: 407 + name: 'sling build --compile hooks_project shows hooks' + run: 'sling build tests/build/hooks_project --compile --target POSTGRES' + output_contains: + - 'stg_orders' + - 'fct_orders' + - 'mode: full-refresh' + - 'start_hooks:' + - 'end_hooks:' + - 'building stg_orders model' + - 'building fct_orders from stg_orders' + - 'fct_orders build complete' + - 'dependencies: [stg_orders]' + + +- id: 408 + name: 'sling build execute hooks_project' + run: 'sling build tests/build/hooks_project --target POSTGRES --full-refresh' + group: build + output_contains: + - 'building stg_orders model' + - 'finished stg_orders model' + - 'building fct_orders from stg_orders' + - 'fct_orders build complete' + - 'OK' + - 'Build Completed' + +# sling build incremental+range (tiers A/B/C, lookback, paged, --range) + +- id: 409 + name: 'sling build --compile range_test_project validates all models' + run: 'sling build tests/build/range_test_project --compile --target POSTGRES' + group: build + output_contains: + - 'range_test.stg_orders' + - 'range_test.fact_orders' + - 'range_test.fact_orders_lookback' + - 'range_test.fact_orders_paged' + - 'range_test.fact_orders_paged_start' + - 'mode: incremental' + - 'dependencies: [stg_orders]' + + +- id: 410 + name: 'sling build plain incremental first run loads full data' + after: [409] + run: | + sling conns exec POSTGRES "DROP SCHEMA IF EXISTS range_test CASCADE; CREATE SCHEMA range_test" + rm -rf tests/build/range_test_project/.sling_state/261 + sling build tests/build/range_test_project --target POSTGRES --full-refresh + output_contains: + - 'range_test.stg_orders' + - 'range_test.fact_orders' + - 'OK' + - 'Build Completed' + + +- id: 411 + name: 'sling build plain incremental second run reads SLING_STATE (tier A)' + after: [410] + env: + SLING_STATE: LOCAL/tests/build/range_test_project/.sling_state/262 + run: | + rm -rf tests/build/range_test_project/.sling_state/262 + sling build tests/build/range_test_project --target POSTGRES --full-refresh -s fact_orders + sling build tests/build/range_test_project --target POSTGRES -s fact_orders --debug + output_contains: + - 'incremental' + - 'created_at" >' + - 'OK' + - 'Build Completed' + + +- id: 412 + name: 'sling build plain incremental without SLING_STATE queries target MAX (tier B)' + after: [411] + run: | + sling build tests/build/range_test_project --target POSTGRES --full-refresh -s fact_orders + sling build tests/build/range_test_project --target POSTGRES -s fact_orders --debug + output_contains: + - 'incremental' + - 'tier B' + - 'SELECT MAX(' + - 'Build Completed' + + +- id: 413 + name: 'sling build lookback-only uses inclusive bound with 2d offset' + after: [412] + run: | + sling build tests/build/range_test_project --target POSTGRES --full-refresh -s fact_orders_lookback + sling build tests/build/range_test_project --target POSTGRES -s fact_orders_lookback --debug + output_contains: + - 'fact_orders_lookback' + - 'created_at" >=' + - 'Build Completed' + + +- id: 414 + name: 'sling build paged first run with explicit start' + after: [413] + env: + SLING_STATE: LOCAL/tests/build/range_test_project/.sling_state/265 + run: | + rm -rf tests/build/range_test_project/.sling_state/265 + sling build tests/build/range_test_project --target POSTGRES --full-refresh -s fact_orders_paged_start + sling build tests/build/range_test_project --target POSTGRES -s fact_orders_paged_start --debug + output_contains: + - 'fact_orders_paged_start' + - '2024-01-01' + - '2024-01-08' + - 'Build Completed' + + +- id: 415 + name: 'sling build paged first run auto-detects origin from source MIN' + after: [414] + env: + SLING_STATE: LOCAL/tests/build/range_test_project/.sling_state/266 + run: | + rm -rf tests/build/range_test_project/.sling_state/266 + sling build tests/build/range_test_project --target POSTGRES --full-refresh -s fact_orders_paged + sling build tests/build/range_test_project --target POSTGRES -s fact_orders_paged --debug + output_contains: + - 'fact_orders_paged' + - 'SELECT MIN(' + - 'Build Completed' + + +- id: 416 + name: 'sling build paged subsequent run advances state by one step' + after: [415] + env: + SLING_STATE: LOCAL/tests/build/range_test_project/.sling_state/267 + run: | + rm -rf tests/build/range_test_project/.sling_state/267 + sling build tests/build/range_test_project --target POSTGRES --full-refresh -s fact_orders_paged_start + sling build tests/build/range_test_project --target POSTGRES -s fact_orders_paged_start + sling build tests/build/range_test_project --target POSTGRES -s fact_orders_paged_start --debug + output_contains: + - 'fact_orders_paged_start' + - 'Build Completed' + + +- id: 417 + name: 'sling build advance range without SLING_STATE errors' + after: [416] + run: | + unset SLING_STATE + sling build tests/build/range_test_project --target POSTGRES -s fact_orders_paged_start + err: true + output_contains: + - 'range.advance requires SLING_STATE' + + +- id: 418 + name: 'sling build --range single chunk runs once' + after: [417] + run: | + sling build tests/build/range_test_project --target POSTGRES --full-refresh -s fact_orders + sling build tests/build/range_test_project --target POSTGRES \ + --range '2024-01-01,2024-02-01' -s fact_orders --debug + output_contains: + - 'fact_orders' + - '2024-01-01' + - '2024-02-01' + - 'Build Completed' + + +- id: 419 + name: 'sling build --range multi-chunk backfill shows per-chunk progress' + after: [418] + run: | + sling build tests/build/range_test_project --target POSTGRES --full-refresh -s fact_orders + sling build tests/build/range_test_project --target POSTGRES \ + --range '2024-01-01,2024-04-01,1mo' -s fact_orders --debug + output_contains: + - 'chunk 1/4' + - 'chunk 2/4' + - 'chunk 3/4' + - 'chunk 4/4' + - 'Build Completed' + + +- id: 420 + name: 'sling build --range does not advance SLING_STATE' + after: [419] + env: + SLING_STATE: LOCAL/tests/build/range_test_project/.sling_state/271 + run: | + rm -rf tests/build/range_test_project/.sling_state/271 + sling build tests/build/range_test_project --target POSTGRES --full-refresh -s fact_orders_paged_start + sling build tests/build/range_test_project --target POSTGRES -s fact_orders_paged_start + cp tests/build/range_test_project/.sling_state/271/*.json /tmp/sling_state_before_271.json + sling build tests/build/range_test_project --target POSTGRES \ + --range '2024-01-01,2024-01-08,7d' -s fact_orders_paged_start + diff /tmp/sling_state_before_271.json tests/build/range_test_project/.sling_state/271/*.json + output_contains: + - 'Build Completed' + + +- id: 421 + name: 'sling build range.start requires range.advance validation error' + group: build + run: 'sling build tests/build/range_bad_start_no_advance --compile --target POSTGRES' + err: true + output_contains: + - 'range.start requires range.advance' + + +- id: 422 + name: 'sling build dbt-style + range.* validation error' + group: build + run: 'sling build tests/build/range_bad_dbt_with_range --compile --target POSTGRES' + err: true + output_contains: + - 'range.* requires {incremental_where_cond}' + + +- id: 423 + name: 'sling build mixed is_incremental + where_cond validation error' + group: build + run: 'sling build tests/build/range_bad_mixed_style --compile --target POSTGRES' + err: true + output_contains: + - 'cannot mix is_incremental() and {incremental_where_cond}' + +# expanded BuildDefaults — layered schema, additive tags/hooks, +# enabled:false skip, defaults unique_key/update_key/merge_strategy. + +- id: 424 + name: 'sling build --compile defaults_expanded_project layers defaults' + group: build + run: 'sling build tests/build/defaults_expanded_project --compile --target POSTGRES --recursive' + output_contains: + # stg_orders inherits root defaults.schema=raw (overrides folder schema) + - 'table: raw.stg_orders' + # child defaults.mode + unique_key + update_key + merge_strategy apply + - 'mode: incremental' + - 'unique_key: id' + - 'update_key: updated_at' + - 'merge_strategy: delete+insert' + # root tag + child tag — union+dedupe + - 'tags: [core, staging]' + # root defaults hook inherited into stg_orders + - '"message":"default start hook"' + # fct_orders: schema override propagates, frontmatter merges with root defaults + - 'table: raw.fct_orders' + - 'tags: [core, marts]' + - '"message":"fct_orders frontmatter start hook"' + # ref() rewrite uses overridden schema + - 'FROM raw.stg_orders' + - 'dependencies: [stg_orders]' + output_does_not_contain: + # enabled:false model must be absent from compile output + - 'stg_disabled' + +# Sling Build - CLI registration, compile mode, seed config + +- id: 425 + name: 'sling build --help shows command description' + run: 'sling build --help' + output_contains: + - 'Build and execute SQL models' + - '--compile' + - '--target' + - '--select' + - '--full-refresh' + - '--schema' + output_does_not_contain: + - '--init' + +- id: 426 + name: 'sling build --compile sample_project with target' + run: 'sling build tests/build/sample_project --compile --target POSTGRES' + group: build + output_contains: + - 'DAG Execution Order:' + - 'country_codes (seed)' + - 'stg_orders (full-refresh)' + - 'stg_customers (full-refresh)' + - 'dim_customers (view)' + - 'fct_orders (incremental)' + - 'revenue (view)' + - 'raw (full-refresh)' + - 'sql:' + - 'dependencies: [stg_orders]' + - 'dependencies: [stg_customers]' + - 'dependencies: [fct_orders]' + + +- id: 427 + name: 'sling build --compile with selector stg_*' + run: 'sling build tests/build/sample_project --compile --target POSTGRES -s "stg_*"' + group: build + output_contains: + - 'stg_customers (full-refresh)' + - 'stg_orders (full-refresh)' + output_does_not_contain: + - 'dim_customers' + - 'fct_orders' + - 'revenue' + - 'country_codes' + + +- id: 428 + name: 'sling build --compile dev mode with --schema override' + run: 'sling build tests/build/sample_project --compile --target POSTGRES --schema dev_test' + group: build + output_contains: + - 'dev_test.staging_stg_orders' + - 'dev_test.staging_stg_customers' + - 'dev_test.marts_core_dim_customers' + - 'dev_test.marts_core_fct_orders' + - 'dev_test.marts_finance_revenue' + - 'dev_test.raw' + - 'dev_test.staging_country_codes' + - 'dev_test.seeds_status_map' + + +- id: 429 + name: 'sling build --compile with upstream selector +revenue' + run: 'sling build tests/build/sample_project --compile --target POSTGRES -s "+revenue"' + group: build + output_contains: + - 'stg_orders' + - 'fct_orders' + - 'revenue' + output_does_not_contain: + - 'stg_customers' + - 'dim_customers' + - 'country_codes' + + +- id: 430 + name: 'sling build --compile dbt_compat_project' + run: 'sling build tests/build/dbt_compat_project --compile --target POSTGRES' + group: build + output_contains: + - 'stg_orders' + - 'country_codes (seed)' + + +- id: 431 + name: 'sling build --compile nested_yml_project inherits config' + run: 'sling build tests/build/nested_yml_project --compile --target POSTGRES --recursive' + group: build + output_contains: + - 'stg_orders (truncate)' + - 'dim_customers (full-refresh)' + + +- id: 432 + name: 'sling build with no config and no target shows help' + run: 'sling build /tmp --compile' + output_contains: + - 'Build and execute SQL models' + - '--recursive' + + +- id: 433 + name: 'sling build execute with selector' + run: 'sling build tests/build/sample_project --target POSTGRES --full-refresh -s "stg_*"' + group: build + output_contains: + - 'stg_orders' + - 'stg_customers' + - 'OK' + output_does_not_contain: + - 'dim_customers' + - 'revenue' + +- id: 434 + name: 'sling build execute sample_project full-refresh' + run: 'sling build tests/build/sample_project --target POSTGRES --full-refresh' + group: build + output_contains: + - 'OK' + - 'Build Completed' + +- id: 435 + name: 'sling build second run (view and incremental modes)' + run: 'sling build tests/build/sample_project --target POSTGRES' + group: build + after: [434] + output_contains: + - 'view' + - 'incremental' + - 'OK' + - 'Build Completed' + +- id: 436 + name: 'sling build --help shows range without short -r' + run: 'sling build --help' + output_contains: + - 'Build and execute SQL models' + - '--range' + - '--threads' + - '--test' + - '--json' + output_does_not_contain: + - '-r, --range' + +# §5.3 — type: build pipeline step / replication end-hook +- id: 437 + name: 'pipeline type:build step runs project and exposes state results' + run: 'sling run -d -p tests/pipelines/p.45.build_step.yaml' + group: build + output_contains: + - 'Build Completed' + - 'SUCCESS: pipeline type:build step completed' + - 'rows=2' + - 'label=from_pipeline' + - 'build total=' + +- id: 438 + name: 'replication end-hook type:build after load' + run: 'sling run -d -r tests/pipelines/p.46.build_step_replication_hook.yaml' + group: build + after: [437] + output_contains: + - 'Build Completed' + - 'SUCCESS: replication end-hook type:build ok' + - 'rows=2' + - 'label=from_repl_hook' + +- id: 439 + name: 'sling build serializes independent DuckDB roots' + env: + DUCK_PARALLEL: '{type: duckdb, instance: "temp/duckdb_parallel/test.duckdb"}' + run: | + mkdir -p temp/duckdb_parallel + rm -f temp/duckdb_parallel/test.duckdb + sling build tests/build/duckdb_parallel_project --target DUCK_PARALLEL --full-refresh --threads 4 + group: build + output_contains: + - 'OK' + - 'Build Completed' + output_does_not_contain: + - 'Conflicting lock' + - 'Could not set lock' + + From 67e5c468a7557cbeb239ca7ceacf6e354bedc049 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Thu, 27 Aug 2026 10:54:54 -0300 Subject: [PATCH 07/30] feat(cli): add sling validate command for file validation Add a new validate subcommand that detects the file kind (replication, pipeline, endpoint, connection, api spec) and checks its structure. Compilation is the default mode, resolving ${VAR} variables using available connections; --parse-only performs a syntax check without compiling. Supported options include --quiet (exit code only), --json, --ndjson, and --output json for machine-readable results, --detailed for per-kind tables, and --debug/--trace for logging levels. Invoking the command bare inside a project validates the project root. Includes unit tests for parsing, compilation, unknown connection warnings, and api spec linting, plus a CLI test suite covering json, ndjson, parse-only, and quiet output modes. --- cmd/sling/sling_validate.go | 131 +++++ core/sling/validate/dto.go | 337 +++++++++++ core/sling/validate/kind.go | 111 ++++ core/sling/validate/output.go | 404 +++++++++++++ core/sling/validate/redact.go | 107 ++++ core/sling/validate/util.go | 102 ++++ core/sling/validate/validate.go | 413 ++++++++++++++ core/sling/validate/validate_test.go | 564 +++++++++++++++++++ tests/suite.cli.validate.yaml | 354 ++++++++++++ tests/validate/env.yaml | 14 + tests/validate/pipeline_nested_bad_type.yaml | 12 + tests/validate/repl_missing_object.yaml | 5 + tests/validate/repl_unknown_conn.yaml | 7 + tests/validate/spec.yaml | 12 + tests/validate/spec_bad_depends.yaml | 6 + tests/validate/spec_missing_url.yaml | 5 + tests/validate/unknown.yaml | 2 + tests/validate/walk/docker-compose.yml | 5 + tests/validate/walk/notes.yaml | 3 + tests/validate/walk/replication_ok.yaml | 7 + 20 files changed, 2601 insertions(+) create mode 100644 cmd/sling/sling_validate.go create mode 100644 core/sling/validate/dto.go create mode 100644 core/sling/validate/kind.go create mode 100644 core/sling/validate/output.go create mode 100644 core/sling/validate/redact.go create mode 100644 core/sling/validate/util.go create mode 100644 core/sling/validate/validate.go create mode 100644 core/sling/validate/validate_test.go create mode 100644 tests/suite.cli.validate.yaml create mode 100644 tests/validate/env.yaml create mode 100644 tests/validate/pipeline_nested_bad_type.yaml create mode 100644 tests/validate/repl_missing_object.yaml create mode 100644 tests/validate/repl_unknown_conn.yaml create mode 100644 tests/validate/spec.yaml create mode 100644 tests/validate/spec_bad_depends.yaml create mode 100644 tests/validate/spec_missing_url.yaml create mode 100644 tests/validate/unknown.yaml create mode 100644 tests/validate/walk/docker-compose.yml create mode 100644 tests/validate/walk/notes.yaml create mode 100644 tests/validate/walk/replication_ok.yaml diff --git a/cmd/sling/sling_validate.go b/cmd/sling/sling_validate.go new file mode 100644 index 000000000..5f4a51c3b --- /dev/null +++ b/cmd/sling/sling_validate.go @@ -0,0 +1,131 @@ +package main + +import ( + "fmt" + "os" + "strings" + + "github.com/flarco/g" + "github.com/integrii/flaggy" + "github.com/slingdata-io/sling-cli/core/env" + "github.com/slingdata-io/sling-cli/core/sling/project" + "github.com/slingdata-io/sling-cli/core/sling/validate" + "github.com/spf13/cast" +) + +var cliValidate = &g.CliSC{ + Name: "validate", + Description: "Validate Sling files", + AdditionalHelpPrepend: "\nDetect the file kind and check the structure. Compile is the default. Compile needs connections and replaces ${VAR}. Use --parse-only for a syntax check.", + Flags: []g.Flag{ + {Name: "quiet", ShortName: "q", Type: "bool", Description: "Exit code only. No output."}, + {Name: "parse-only", Type: "bool", Description: "Skip compilation step, parse structure only. Do not replace ${VAR}."}, + {Name: "detailed", Type: "bool", Description: "Show one table per kind (streams, steps, endpoints, connections)."}, + {Name: "ndjson", Type: "bool", Description: "One JSON object per line."}, + {Name: "output", ShortName: "o", Type: "string", Description: "Output format: json. Human table is the TTY default."}, + {Name: "json", Type: "bool", Description: "Emit one JSON object keyed by path."}, + {Name: "debug", ShortName: "d", Type: "bool", Description: "Set logging level to DEBUG."}, + {Name: "trace", Type: "bool", Description: "Set logging level to TRACE."}, + }, + PosFlags: []g.Flag{ + { + Name: "paths...", + Type: "string", + Description: "Files or folders to validate.", + Required: false, + }, + }, + ExecProcess: processValidate, +} + +func init() { + cliValidate.Make().Add() +} + +func processValidate(c *g.CliSC) (ok bool, err error) { + ok = true + + if cast.ToBool(c.Vals["trace"]) { + os.Setenv("DEBUG", "TRACE") + env.InitLogger() + } else if cast.ToBool(c.Vals["debug"]) { + os.Setenv("DEBUG", "LOW") + env.InitLogger() + } + + paths := collectValidatePaths(c) + if len(paths) == 0 { + // Bare invocation inside a project validates the project root. + wd, wdErr := os.Getwd() + if wdErr == nil { + if root, findErr := project.FindRoot(wd); findErr == nil && root != "" { + g.Debug("validating project root %s", root) + paths = []string{root} + } + } + } + if len(paths) == 0 { + flaggy.ShowHelp("") + return ok, nil + } + + opts := validate.Options{ + Compile: !cast.ToBool(c.Vals["parse-only"]), + Quiet: cast.ToBool(c.Vals["quiet"]), + NDJSON: cast.ToBool(c.Vals["ndjson"]), + JSON: cast.ToBool(c.Vals["json"]), + Detailed: cast.ToBool(c.Vals["detailed"]), + } + + output := strings.ToLower(strings.TrimSpace(cast.ToString(c.Vals["output"]))) + switch output { + case "", "text": + case "json": + opts.JSON = true + default: + return ok, g.Error("invalid --output %q; expected json", output) + } + + results := validate.ParsePaths(paths, opts) + text, err := validate.GetOutput(results, opts) + if err != nil { + return ok, g.Error(err, "could not render validate output") + } + if text != "" { + fmt.Fprint(os.Stdout, text+"\n") + } + + if validate.AnyFailed(results) { + return ok, validateFailErr(results) + } + return ok, nil +} + +func collectValidatePaths(c *g.CliSC) []string { + paths := []string{} + if v := strings.TrimSpace(cast.ToString(c.Vals["paths..."])); v != "" { + paths = append(paths, v) + } + paths = append(paths, flaggy.TrailingArguments...) + return paths +} + +func validateFailErr(results []validate.FileResult) error { + n := 0 + var first validate.FileResult + for _, r := range results { + if !r.OK { + n++ + if first.Path == "" { + first = r + } + } + } + if n == 1 { + if first.Error != "" { + return g.Error("%s: %s", first.Path, first.Error) + } + return g.Error("%s: parse failed", first.Path) + } + return g.Error("%d files failed to parse", n) +} diff --git a/core/sling/validate/dto.go b/core/sling/validate/dto.go new file mode 100644 index 000000000..364e44e82 --- /dev/null +++ b/core/sling/validate/dto.go @@ -0,0 +1,337 @@ +package validate + +import ( + "fmt" + "os" + "path/filepath" + "strings" + + "github.com/slingdata-io/sling-cli/core/dbio/api" + "github.com/slingdata-io/sling-cli/core/dbio/connection" + "github.com/slingdata-io/sling-cli/core/sling" + "github.com/spf13/cast" + "gopkg.in/yaml.v3" +) + +// ReplicationDTO is a replication file without engine-only fields. +type ReplicationDTO struct { + Source any `json:"source" yaml:"source"` + Target any `json:"target" yaml:"target"` + Hooks any `json:"hooks,omitempty" yaml:"hooks,omitempty"` + Defaults any `json:"defaults,omitempty" yaml:"defaults,omitempty"` + Streams map[string]any `json:"streams" yaml:"streams"` + Env map[string]any `json:"env,omitempty" yaml:"env,omitempty"` +} + +// PipelineDTO is a pipeline file without engine-only fields. +type PipelineDTO struct { + Steps []any `json:"steps" yaml:"steps"` + Env map[string]any `json:"env,omitempty" yaml:"env,omitempty"` +} + +// EnvDTO is an env.yaml file. Values are not interpolated. +type EnvDTO struct { + Connections map[string]any `json:"connections,omitempty" yaml:"connections,omitempty"` + Env map[string]any `json:"env,omitempty" yaml:"env,omitempty"` + Variables map[string]any `json:"variables,omitempty" yaml:"variables,omitempty"` +} + +// APISpecDTO is an API spec without engine-only fields. +type APISpecDTO struct { + Name string `json:"name" yaml:"name"` + Description string `json:"description,omitempty" yaml:"description,omitempty"` + Authentication any `json:"authentication,omitempty" yaml:"authentication,omitempty"` + Defaults any `json:"defaults,omitempty" yaml:"defaults,omitempty"` + Endpoints map[string]any `json:"endpoints,omitempty" yaml:"endpoints,omitempty"` + DynamicEndpoints any `json:"dynamic_endpoints,omitempty" yaml:"dynamic_endpoints,omitempty"` + Queues any `json:"queues,omitempty" yaml:"queues,omitempty"` +} + +// BuildDTO is a sling_build.yml file. +type BuildDTO struct { + Target string `json:"target,omitempty" yaml:"target,omitempty"` + Dev any `json:"dev,omitempty" yaml:"dev,omitempty"` + DbtProject any `json:"dbt_project,omitempty" yaml:"dbt_project,omitempty"` + Vars map[string]any `json:"vars,omitempty" yaml:"vars,omitempty"` + Defaults any `json:"defaults,omitempty" yaml:"defaults,omitempty"` +} + +// ProjectDTO is a sling_project.yml manifest. +type ProjectDTO struct { + Name string `json:"name,omitempty" yaml:"name,omitempty"` + ProjectID string `json:"project_id,omitempty" yaml:"project_id,omitempty"` + Jobs map[string]any `json:"jobs,omitempty" yaml:"jobs,omitempty"` +} + +var knownStepTypes = map[string]bool{ + "query": true, + "http": true, + "check": true, + "copy": true, + "list": true, + "write": true, + "read": true, + "replication": true, + "command": true, + "group": true, + "delete": true, + "log": true, + "inspect": true, + "store": true, + "set": true, + "routine": true, + "read_cdc": true, + "build": true, +} + +func parseDTO(kind Kind, body []byte) (any, error) { + switch kind { + case KindReplication: + return parseReplicationDTO(body) + case KindPipeline: + return parsePipelineDTO(body) + case KindEnv: + return parseEnvDTO(body) + case KindAPISpec: + return parseAPISpecDTO(body) + case KindBuild: + return parseBuildDTO(body) + case KindMonitor, KindRoutine: + root := map[string]any{} + if err := yaml.Unmarshal(body, &root); err != nil { + return nil, fmt.Errorf("invalid yaml: %w", err) + } + return Redact(root), nil + default: + return nil, fmt.Errorf("unknown kind") + } +} + +// parseReplicationDTO parses a replication file and checks required keys. +func parseReplicationDTO(body []byte) (any, error) { + var dto ReplicationDTO + if err := yaml.Unmarshal(body, &dto); err != nil { + return nil, fmt.Errorf("invalid replication yaml: %w", err) + } + if strings.TrimSpace(cast.ToString(dto.Source)) == "" { + return nil, fmt.Errorf("replication is missing required key 'source'") + } + if strings.TrimSpace(cast.ToString(dto.Target)) == "" { + return nil, fmt.Errorf("replication is missing required key 'target'") + } + if dto.Streams == nil { + return nil, fmt.Errorf("replication is missing required key 'streams'") + } + if err := validateStreams(dto.Streams); err != nil { + return nil, err + } + if err := validateModeIn(dto.Defaults, "defaults"); err != nil { + return nil, err + } + return Redact(dtoToMap(dto)), nil +} + +// parsePipelineDTO parses a pipeline file and checks step mappings and types. +func parsePipelineDTO(body []byte) (any, error) { + var dto PipelineDTO + if err := yaml.Unmarshal(body, &dto); err != nil { + return nil, fmt.Errorf("invalid pipeline yaml: %w", err) + } + if dto.Steps == nil { + return nil, fmt.Errorf("pipeline is missing required key 'steps'") + } + if err := validatePipelineSteps(dto.Steps, ""); err != nil { + return nil, err + } + return Redact(dtoToMap(dto)), nil +} + +// validatePipelineSteps checks each step mapping and type, including +// nested steps under group and routine. +func validatePipelineSteps(steps []any, base string) error { + for i, step := range steps { + where := fmt.Sprintf("pipeline step %d", i) + if base != "" { + where = fmt.Sprintf("%s.%d", base, i) + } + m, ok := asMap(step) + if !ok { + return fmt.Errorf("%s must be a mapping", where) + } + typ := strings.TrimSpace(cast.ToString(m["type"])) + if typ == "" { + if inferred := stepType(m); inferred != "-" { + typ = inferred + } + } + if typ != "" && !knownStepTypes[typ] { + return fmt.Errorf("%s has unknown type %q", where, typ) + } + if typ != "group" && typ != "routine" { + continue + } + nested := asSlice(m["steps"]) + if nested == nil { + if inner, ok := asMap(m[typ]); ok { + nested = asSlice(inner["steps"]) + } + } + if err := validatePipelineSteps(nested, where+".steps"); err != nil { + return err + } + } + return nil +} + +// parseEnvDTO parses an env.yaml file. Values are not interpolated. +func parseEnvDTO(body []byte) (any, error) { + var dto EnvDTO + if err := yaml.Unmarshal(body, &dto); err != nil { + return nil, fmt.Errorf("invalid env yaml: %w", err) + } + return Redact(dtoToMap(dto)), nil +} + +// parseAPISpecDTO parses an API spec file and checks required keys. +func parseAPISpecDTO(body []byte) (any, error) { + var dto APISpecDTO + if err := yaml.Unmarshal(body, &dto); err != nil { + return nil, fmt.Errorf("invalid api spec yaml: %w", err) + } + if strings.TrimSpace(dto.Name) == "" { + return nil, fmt.Errorf("api spec is missing required key 'name'") + } + if dto.Endpoints == nil && dto.DynamicEndpoints == nil { + return nil, fmt.Errorf("api spec is missing 'endpoints' or 'dynamic_endpoints'") + } + for name, ep := range dto.Endpoints { + if ep == nil { + continue + } + if _, ok := ep.(map[string]any); !ok { + return nil, fmt.Errorf("api spec endpoint %q must be a mapping", name) + } + } + if _, err := api.LoadSpec(string(body)); err != nil { + return nil, fmt.Errorf("api spec is invalid: %s", err.Error()) + } + return Redact(dtoToMap(dto)), nil +} + +// parseBuildDTO parses a sling_build.yml file into its DTO. +func parseBuildDTO(body []byte) (any, error) { + var dto BuildDTO + if err := yaml.Unmarshal(body, &dto); err != nil { + return nil, fmt.Errorf("invalid build yaml: %w", err) + } + return Redact(dtoToMap(dto)), nil +} + +// unknownConnWarnings reports source/target names that look like connections +// but are not in the local connection list. Templates ({var}) and URLs are skipped. +func unknownConnWarnings(parsed any) []string { + m := asMapOrEmpty(parsed) + var out []string + for _, key := range []string{"source", "target"} { + name := strings.TrimSpace(cast.ToString(m[key])) + if name == "" || strings.Contains(name, "{") || strings.Contains(name, "://") { + continue + } + if connection.GetLocalConns().Get(name).Name == "" { + out = append(out, fmt.Sprintf("%s connection %q is not defined", key, name)) + } + } + return out +} + +// validateStreams checks that each stream is a mapping or null with a valid mode. +func validateStreams(streams map[string]any) error { + for name, val := range streams { + if val == nil { + continue + } + m, ok := asMap(val) + if !ok { + return fmt.Errorf("stream %q must be a mapping or null", name) + } + if err := validateModeIn(m, "stream "+name); err != nil { + return err + } + } + return nil +} + +// validateModeIn checks that the 'mode' key in v is a known mode, when set. +func validateModeIn(v any, where string) error { + m, ok := asMap(v) + if !ok { + return nil + } + mode := strings.TrimSpace(cast.ToString(m["mode"])) + if mode == "" { + return nil + } + if !isKnownMode(mode) { + return fmt.Errorf("%s has unknown mode %q", where, mode) + } + return nil +} + +// isKnownMode reports whether mode matches an entry in sling.AllMode. +func isKnownMode(mode string) bool { + for _, am := range sling.AllMode { + if string(am.Value) == mode { + return true + } + } + return false +} + +// pipelineStepPathWarnings reports replication/build step paths that do not +// resolve. Paths may resolve at run time from another root, so these are +// warnings, never errors. +func pipelineStepPathWarnings(absPipeline string, parsed any) (out []string) { + m, ok := asMap(parsed) + if !ok { + return nil + } + baseDir := filepath.Dir(absPipeline) + for i, step := range asSlice(m["steps"]) { + sm, ok := asMap(step) + if !ok { + continue + } + typ := stepType(sm) + var path string + switch typ { + case "replication": + path = firstNonEmpty(cast.ToString(sm["path"]), cast.ToString(sm["replication"])) + case "build": + path = firstNonEmpty(cast.ToString(sm["path"]), cast.ToString(sm["build"])) + default: + continue + } + path = strings.TrimSpace(path) + if path == "" || strings.Contains(path, "{") { + continue // empty, or resolved at run time from an expression + } + if stepPathExists(baseDir, path) { + continue + } + out = append(out, fmt.Sprintf("step %d (%s) path %s does not exist", i, typ, path)) + } + return out +} + +// stepPathExists looks relative to the pipeline file, then the working dir. +func stepPathExists(baseDir, path string) bool { + if filepath.IsAbs(path) { + _, err := os.Stat(path) + return err == nil + } + if _, err := os.Stat(filepath.Join(baseDir, path)); err == nil { + return true + } + _, err := os.Stat(path) + return err == nil +} diff --git a/core/sling/validate/kind.go b/core/sling/validate/kind.go new file mode 100644 index 000000000..4cd4b3400 --- /dev/null +++ b/core/sling/validate/kind.go @@ -0,0 +1,111 @@ +package validate + +import ( + "bytes" + "os" + "path/filepath" + "strings" + + "github.com/slingdata-io/sling-cli/core/sling/build" + "gopkg.in/yaml.v3" +) + +// Kind is a Sling YAML document kind. +type Kind string + +const ( + KindPipeline Kind = "pipeline" + KindReplication Kind = "replication" + KindAPISpec Kind = "api_spec" + KindMonitor Kind = "monitor" + KindRoutine Kind = "routine" + KindEnv Kind = "env" + KindBuild Kind = "build" + KindProject Kind = "project" + KindUnknown Kind = "unknown" +) + +// DetectFileKind classifies a YAML document from parsed root keys. +// Content rules match VS Code detectSchemaType order. Path rules run +// only when content matches nothing. sling_build.yml (or a directory +// that contains one) is the first path exception. env.yaml is the +// content-unknown fallback. +func DetectFileKind(body []byte, path string) Kind { + if kind := detectKindFromContent(body); kind != KindUnknown { + return kind + } + return detectKindFromPath(path) +} + +func detectKindFromContent(body []byte) Kind { + if len(bytes.TrimSpace(body)) == 0 { + return KindUnknown + } + + root := map[string]any{} + if err := yaml.Unmarshal(body, &root); err != nil || root == nil { + return KindUnknown + } + + if hasKey(root, "steps") { + return KindPipeline + } + if hasKey(root, "source") && hasKey(root, "target") && hasKey(root, "streams") { + return KindReplication + } + if hasKey(root, "name") && (hasKey(root, "endpoints") || hasKey(root, "dynamic_endpoints")) { + return KindAPISpec + } + if hasKey(root, "connection") && hasKey(root, "objects") { + return KindMonitor + } + if hasKey(root, "routines") { + return KindRoutine + } + if hasKey(root, "connections") { + return KindEnv + } + return KindUnknown +} + +// detectKindFromPath falls back to file/folder name conventions when content rules match nothing. +func detectKindFromPath(path string) Kind { + base := filepath.Base(path) + if isManifestName(base) { + return KindProject + } + if isBuildConfigName(base) { + return KindBuild + } + if info, err := os.Stat(path); err == nil && info.IsDir() { + if _, err := os.Stat(filepath.Join(path, build.ConfigFileName)); err == nil { + return KindBuild + } + } + if isEnvFileName(base) { + return KindEnv + } + return KindUnknown +} + +func hasKey(m map[string]any, key string) bool { + _, ok := m[key] + return ok +} + +// isManifestName reports whether the file name is a sling_project manifest. +func isManifestName(name string) bool { + n := strings.ToLower(name) + return n == "sling_project.yml" || n == "sling_project.yaml" +} + +// isBuildConfigName reports whether the file name is a sling_build config. +func isBuildConfigName(name string) bool { + return name == build.ConfigFileName || strings.EqualFold(name, "sling_build.yaml") +} + +// isEnvFileName reports whether the file name is an env.yaml connections file. +func isEnvFileName(name string) bool { + n := strings.ToLower(name) + return n == "env.yaml" || n == "env.yml" +} diff --git a/core/sling/validate/output.go b/core/sling/validate/output.go new file mode 100644 index 000000000..a8576f70d --- /dev/null +++ b/core/sling/validate/output.go @@ -0,0 +1,404 @@ +package validate + +import ( + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/flarco/g" + "github.com/mattn/go-isatty" + "github.com/spf13/cast" +) + +// GetOutput renders validate results as a string. --quiet is silent. JSON is the +// default when stdout is not a TTY. +func GetOutput(results []FileResult, opts Options) (string, error) { + if opts.Quiet { + return "", nil + } + if opts.NDJSON { + return ndjsonOutput(results) + } + if opts.JSON { + return jsonOutput(results) + } + if opts.Detailed { + return detailedOutput(results), nil + } + if !isStdoutTTY() { + return jsonOutput(results) + } + return tableOutput(results), nil +} + +func isStdoutTTY() bool { + return isatty.IsTerminal(os.Stdout.Fd()) || isatty.IsCygwinTerminal(os.Stdout.Fd()) +} + +// resultRow builds the JSON object for one file result. The JSON output keys +// rows by path, so it omits the inner "path" field; NDJSON includes it. +func resultRow(r FileResult, includePath bool) map[string]any { + row := map[string]any{ + "kind": r.Kind, + "ok": r.OK, + "compiled": r.Compiled, + } + if includePath { + row["path"] = r.Path + } + if r.Parsed != nil { + row["parsed"] = r.Parsed + } + if r.Error != "" { + row["error"] = r.Error + } + if len(r.Warnings) > 0 { + row["warnings"] = r.Warnings + } + return row +} + +func jsonOutput(results []FileResult) (string, error) { + out := map[string]any{} + for _, r := range results { + out[r.Path] = resultRow(r, false) + } + b, err := json.MarshalIndent(out, "", " ") + if err != nil { + return "", err + } + return string(b) + "\n", nil +} + +func ndjsonOutput(results []FileResult) (string, error) { + sb := strings.Builder{} + enc := json.NewEncoder(&sb) + for _, r := range results { + if err := enc.Encode(resultRow(r, true)); err != nil { + return "", err + } + } + return sb.String(), nil +} + +func tableOutput(results []FileResult) string { + header := []string{"path", "kind", "ok"} + rows := make([][]any, 0, len(results)) + var errs []string + for _, r := range results { + rows = append(rows, []any{r.Path, string(r.Kind), r.OK}) + if !r.OK { + msg := r.Error + if msg == "" { + msg = "validation failed" + } + errs = append(errs, fmt.Sprintf("%s: %s", r.Path, msg)) + } + } + out := g.PrettyTable(header, rows) + if w := warningLines(results); w != "" { + out += w + } + if len(errs) > 0 { + out += fmt.Sprintf("\nerrors:\n %s\n", strings.Join(errs, "\n ")) + } + return out +} + +// warningLines renders advisory findings under the file rows. Warnings +// never change the exit code. +func warningLines(results []FileResult) string { + sb := strings.Builder{} + for _, r := range results { + for _, w := range r.Warnings { + fmt.Fprintf(&sb, " warning: %s: %s\n", r.Path, w) + } + } + if sb.Len() == 0 { + return "" + } + return "\n" + sb.String() +} + +// detailedOutput renders one table per kind, each with fields useful +// for that kind. Files that failed to parse get their own error table. +func detailedOutput(results []FileResult) string { + sections := []struct { + title string + body string + }{ + {"Replications", replicationSection(results)}, + {"Pipelines", pipelineSection(results)}, + {"API Specs", apiSpecSection(results)}, + {"Connections", connectionSection(results)}, + {"Build Projects", buildSection(results)}, + {"Projects", projectSection(results)}, + {"Other Files", otherSection(results)}, + {"Errors", errorSection(results)}, + } + + sb := strings.Builder{} + for _, s := range sections { + if s.body == "" { + continue + } + if sb.Len() > 0 { + sb.WriteString("\n") + } + sb.WriteString(g.Colorize(g.ColorCyan, "# "+s.title) + "\n") + sb.WriteString(s.body) + } + if sb.Len() == 0 { + return "no files parsed\n" + } + if w := warningLines(results); w != "" { + sb.WriteString(w) + } + return sb.String() +} + +// projectSection lists the jobs each manifest declares. +func projectSection(results []FileResult) string { + rows := [][]any{} + for _, r := range kindResults(results, KindProject) { + m := asMapOrEmpty(r.Parsed) + jobs, ok := asMap(m["jobs"]) + if !ok || len(jobs) == 0 { + rows = append(rows, []any{r.Path, cast.ToString(m["name"]), "-", "-", "-"}) + continue + } + for _, key := range sortedKeys(jobs) { + jm := asMapOrEmpty(jobs[key]) + rows = append(rows, []any{ + r.Path, + cast.ToString(m["name"]), + key, + firstNonEmpty(cast.ToString(jm["file"]), "-"), + firstNonEmpty(strings.Join(toStringSlice(jm["schedules"]), ", "), "-"), + }) + } + } + if len(rows) == 0 { + return "" + } + return g.PrettyTable([]string{"path", "project", "job", "file", "schedules"}, rows) +} + +func kindResults(results []FileResult, kind Kind) []FileResult { + out := []FileResult{} + for _, r := range results { + if r.OK && r.Kind == kind { + out = append(out, r) + } + } + return out +} + +func replicationSection(results []FileResult) string { + rows := [][]any{} + for _, r := range kindResults(results, KindReplication) { + m := asMapOrEmpty(r.Parsed) + source := cast.ToString(m["source"]) + target := cast.ToString(m["target"]) + streams, ok := asMap(m["streams"]) + if !ok || len(streams) == 0 { + rows = append(rows, []any{r.Path, source, target, "-", "-", "-"}) + continue + } + defs := asMapOrEmpty(m["defaults"]) + for _, name := range sortedKeys(streams) { + sm, _ := asMap(streams[name]) + rows = append(rows, []any{ + r.Path, + source, + target, + name, + firstNonEmpty(cast.ToString(sm["mode"]), defaultsMode(defs), "-"), + boolLabel(sm["disabled"]), + }) + } + } + if len(rows) == 0 { + return "" + } + return g.PrettyTable([]string{"path", "source", "target", "stream", "mode", "disabled"}, rows) +} + +func pipelineSection(results []FileResult) string { + rows := [][]any{} + for _, r := range kindResults(results, KindPipeline) { + m := asMapOrEmpty(r.Parsed) + for _, step := range asSlice(m["steps"]) { + sm, ok := asMap(step) + if !ok { + continue + } + rows = append(rows, []any{ + r.Path, + stepType(sm), + firstNonEmpty(cast.ToString(sm["id"]), "-"), + }) + } + } + if len(rows) == 0 { + return "" + } + return g.PrettyTable([]string{"path", "type", "id"}, rows) +} + +// stepType resolves the step type, including the shorthand form +// where the type is the key (e.g. `- replication: path`). +func stepType(sm map[string]any) string { + if typ := strings.TrimSpace(cast.ToString(sm["type"])); typ != "" { + return typ + } + for _, k := range sortedKeys(sm) { + if knownStepTypes[k] { + return k + } + } + return "-" +} + +func apiSpecSection(results []FileResult) string { + rows := [][]any{} + for _, r := range kindResults(results, KindAPISpec) { + m := asMapOrEmpty(r.Parsed) + eps, ok := asMap(m["endpoints"]) + if !ok || len(eps) == 0 { + rows = append(rows, []any{r.Path, "-", "-", "-", "-"}) + continue + } + for _, name := range sortedKeys(eps) { + em, _ := asMap(eps[name]) + req, _ := asMap(em["request"]) + rows = append(rows, []any{ + r.Path, + name, + firstNonEmpty(cast.ToString(req["method"]), "GET"), + cast.ToString(req["url"]), + boolLabel(em["disabled"]), + }) + } + } + if len(rows) == 0 { + return "" + } + return g.PrettyTable([]string{"path", "endpoint", "method", "url", "disabled"}, rows) +} + +func connectionSection(results []FileResult) string { + rows := [][]any{} + for _, r := range kindResults(results, KindEnv) { + m := asMapOrEmpty(r.Parsed) + conns, ok := asMap(m["connections"]) + if !ok { + continue + } + for _, name := range sortedKeys(conns) { + cm, _ := asMap(conns[name]) + rows = append(rows, []any{ + r.Path, + name, + firstNonEmpty(cast.ToString(cm["type"]), "-"), + firstNonEmpty( + cast.ToString(cm["host"]), + cast.ToString(cm["bucket"]), + cast.ToString(cm["account"]), + cast.ToString(cm["url"]), + "-", + ), + firstNonEmpty(cast.ToString(cm["database"]), "-"), + firstNonEmpty(cast.ToString(cm["schema"]), "-"), + }) + } + } + if len(rows) == 0 { + return "" + } + return g.PrettyTable([]string{"path", "connection", "type", "host", "database", "schema"}, rows) +} + +func buildSection(results []FileResult) string { + rows := [][]any{} + for _, r := range kindResults(results, KindBuild) { + m := asMapOrEmpty(r.Parsed) + vars, _ := asMap(m["vars"]) + rows = append(rows, []any{ + r.Path, + firstNonEmpty(cast.ToString(m["target"]), "-"), + firstNonEmpty(cast.ToString(m["mode"]), defaultsMode(m["defaults"]), "-"), + countOrDash(m["models"]), + countOrDash(m["seeds"]), + len(vars), + }) + } + if len(rows) == 0 { + return "" + } + return g.PrettyTable([]string{"path", "target", "mode", "models", "seeds", "vars"}, rows) +} + +func otherSection(results []FileResult) string { + rows := [][]any{} + for _, r := range results { + if !r.OK { + continue + } + switch r.Kind { + case KindReplication, KindPipeline, KindEnv, KindAPISpec, KindBuild, KindProject: + continue + } + m := asMapOrEmpty(r.Parsed) + rows = append(rows, []any{r.Path, string(r.Kind), len(m)}) + } + if len(rows) == 0 { + return "" + } + return g.PrettyTable([]string{"path", "kind", "keys"}, rows) +} + +func errorSection(results []FileResult) string { + rows := [][]any{} + for _, r := range results { + if r.OK { + continue + } + msg := r.Error + if msg == "" { + msg = "parse failed" + } + rows = append(rows, []any{r.Path, string(r.Kind), msg}) + } + if len(rows) == 0 { + return "" + } + return g.PrettyTable([]string{"path", "kind", "error"}, rows) +} + +func defaultsMode(v any) string { + m, ok := asMap(v) + if !ok { + return "" + } + // compiled defaults use the Go field name + return firstNonEmpty(cast.ToString(m["mode"]), cast.ToString(m["Mode"])) +} + +func countOrDash(v any) any { + if s := asSlice(v); s != nil { + return len(s) + } + if m, ok := asMap(v); ok { + return len(m) + } + return "-" +} + +func boolLabel(v any) string { + if cast.ToBool(v) { + return "yes" + } + return "-" +} diff --git a/core/sling/validate/redact.go b/core/sling/validate/redact.go new file mode 100644 index 000000000..52d8b4604 --- /dev/null +++ b/core/sling/validate/redact.go @@ -0,0 +1,107 @@ +package validate + +import ( + "regexp" + "strings" + "sync" + + "github.com/slingdata-io/sling-cli/core/env" +) + +var envRefRe = regexp.MustCompile(`^\s*\$\{[A-Za-z_][A-Za-z0-9_]*\}\s*$`) + +var ( + secretKeyOnce sync.Once + secretKeySet map[string]struct{} +) + +// secretKeysLower returns the lowercased set of known secret key names, built once. +func secretKeysLower() map[string]struct{} { + secretKeyOnce.Do(func() { + secretKeySet = map[string]struct{}{} + for _, k := range env.SecretKeys { + secretKeySet[strings.ToLower(k)] = struct{}{} + } + secretKeySet["authentication"] = struct{}{} + }) + return secretKeySet +} + +// isEnvRef reports whether s is exactly a ${VAR} reference. +func isEnvRef(s string) bool { + return envRefRe.MatchString(s) +} + +// Redact replaces secret-shaped values with "***". ${VAR} refs pass through. +func Redact(v any) any { + return redactValue(v, false) +} + +// redactValue walks v recursively, masking strings under force or when the +// parent key is secret-shaped. +func redactValue(v any, force bool) any { + switch x := v.(type) { + case map[string]any: + out := make(map[string]any, len(x)) + keys := secretKeysLower() + for k, val := range x { + lk := strings.ToLower(k) + if lk == "authentication" { + out[k] = redactAuth(val) + continue + } + if lk == "secrets" { + out[k] = redactValue(val, true) + continue + } + _, secret := keys[lk] + out[k] = redactValue(val, force || secret) + } + return out + case []any: + out := make([]any, len(x)) + for i, item := range x { + out[i] = redactValue(item, force) + } + return out + case string: + if force { + if isEnvRef(x) || x == "" { + return x + } + return "***" + } + return x + default: + if force && v != nil { + if s, ok := v.(string); ok && isEnvRef(s) { + return s + } + return "***" + } + return v + } +} + +// redactAuth masks authentication values but keeps the map shape and the +// non-secret "type" field so output stays an object. +func redactAuth(v any) any { + if s, ok := v.(string); ok && isEnvRef(s) { + return s + } + if v == nil { + return v + } + if m, ok := v.(map[string]any); ok { + out, _ := redactValue(m, true).(map[string]any) + for k, val := range m { + if strings.EqualFold(k, "type") { + if s, ok := val.(string); ok { + out[k] = s + } + } + } + return out + } + return "***" +} diff --git a/core/sling/validate/util.go b/core/sling/validate/util.go new file mode 100644 index 000000000..8df31cd5b --- /dev/null +++ b/core/sling/validate/util.go @@ -0,0 +1,102 @@ +package validate + +import ( + "encoding/json" + "sort" + "strings" + + "github.com/spf13/cast" +) + +// Shared value-coercion helpers used across parsing and rendering. + +// asMap coerces v to map[string]any, converting map[any]any keys when needed. +func asMap(v any) (map[string]any, bool) { + switch t := v.(type) { + case map[string]any: + return t, true + case map[any]any: + out := make(map[string]any, len(t)) + for k, val := range t { + out[cast.ToString(k)] = val + } + return out, true + default: + return nil, false + } +} + +// asMapOrEmpty is asMap but returns an empty map instead of false on mismatch. +func asMapOrEmpty(v any) map[string]any { + if m, ok := asMap(v); ok { + return m + } + return map[string]any{} +} + +// asSlice coerces v to []any, wrapping []string items when needed. +func asSlice(v any) []any { + switch t := v.(type) { + case []any: + return t + case []string: + out := make([]any, len(t)) + for i, s := range t { + out[i] = s + } + return out + } + return nil +} + +// toStringSlice coerces a list or scalar to []string. Blank strings yield nil. +func toStringSlice(v any) []string { + switch t := v.(type) { + case []string: + return t + case []any: + out := make([]string, 0, len(t)) + for _, item := range t { + out = append(out, cast.ToString(item)) + } + return out + case string: + if strings.TrimSpace(t) == "" { + return nil + } + return []string{t} + } + return nil +} + +func sortedKeys(m map[string]any) []string { + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + return keys +} + +// firstNonEmpty returns the first value that is not blank after trimming. +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if strings.TrimSpace(v) != "" { + return v + } + } + return "" +} + +// dtoToMap round-trips a DTO through JSON so nested values become plain maps. +func dtoToMap(v any) map[string]any { + b, err := json.Marshal(v) + if err != nil { + return map[string]any{"error": err.Error()} + } + m := map[string]any{} + if err := json.Unmarshal(b, &m); err != nil { + return map[string]any{"error": err.Error()} + } + return m +} diff --git a/core/sling/validate/validate.go b/core/sling/validate/validate.go new file mode 100644 index 000000000..34a26d1ba --- /dev/null +++ b/core/sling/validate/validate.go @@ -0,0 +1,413 @@ +package validate + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/flarco/g" + "github.com/robfig/cron/v3" + "github.com/slingdata-io/sling-cli/core/env" + "github.com/slingdata-io/sling-cli/core/sling" + "github.com/slingdata-io/sling-cli/core/sling/build" + "github.com/spf13/cast" + "gopkg.in/yaml.v3" +) + +// Options controls parse behavior. +type Options struct { + Compile bool + Quiet bool + NDJSON bool + JSON bool + Detailed bool +} + +// FileResult is one validated path. +type FileResult struct { + Path string `json:"path,omitempty"` + Kind Kind `json:"kind"` + OK bool `json:"ok"` + Compiled bool `json:"compiled"` + Parsed any `json:"parsed,omitempty"` + Error string `json:"error,omitempty"` + Warnings []string `json:"warnings,omitempty"` +} + +// ParsePaths parses each path. Directories are walked. Parse errors +// stay in the result list so all files are reported. +func ParsePaths(paths []string, opts Options) []FileResult { + var results []FileResult + for _, p := range paths { + results = append(results, parseOnePath(p, opts)...) + } + return results +} + +// ParseFile parses one explicit file. Directories are rejected. +func ParseFile(path string, opts Options) FileResult { + abs, err := filepath.Abs(path) + if err != nil { + return FileResult{Path: displayPath(path, path), OK: false, Error: err.Error()} + } + info, err := os.Stat(abs) + if err != nil { + return FileResult{Path: displayPath(path, abs), OK: false, Kind: KindUnknown, Error: err.Error()} + } + if info.IsDir() { + return FileResult{Path: displayPath(path, abs), OK: false, Kind: KindUnknown, Error: "path is a directory"} + } + return parseExplicitFile(path, abs, opts) +} + +func parseOnePath(userPath string, opts Options) []FileResult { + abs, err := filepath.Abs(userPath) + if err != nil { + return []FileResult{{Path: userPath, OK: false, Kind: KindUnknown, Error: err.Error()}} + } + info, err := os.Stat(abs) + if err != nil { + return []FileResult{{Path: displayPath(userPath, abs), OK: false, Kind: KindUnknown, Error: err.Error()}} + } + if info.IsDir() { + return walkDir(userPath, abs, opts) + } + return []FileResult{parseExplicitFile(userPath, abs, opts)} +} + +func parseExplicitFile(userPath, abs string, opts Options) FileResult { + body, err := os.ReadFile(abs) + if err != nil { + return FileResult{Path: displayPath(userPath, abs), OK: false, Kind: KindUnknown, Error: err.Error()} + } + kind := DetectFileKind(body, abs) + return parseBody(displayPath(userPath, abs), abs, kind, body, opts) +} + +func walkDir(userPath, absDir string, opts Options) []FileResult { + files, err := g.ListDirRecursive(absDir) + if err != nil { + return []FileResult{{Path: displayPath(userPath, absDir), OK: false, Kind: KindUnknown, Error: err.Error()}} + } + + var results []FileResult + for _, file := range files { + if file.IsDir { + continue + } + name := strings.ToLower(file.Name) + if !strings.HasSuffix(name, ".yaml") && !strings.HasSuffix(name, ".yml") { + continue + } + + body, err := os.ReadFile(file.FullPath) + if err != nil { + disp := walkDisplayPath(userPath, absDir, file.FullPath) + results = append(results, FileResult{Path: disp, OK: false, Kind: KindUnknown, Error: err.Error()}) + continue + } + + kind := DetectFileKind(body, file.FullPath) + disp := walkDisplayPath(userPath, absDir, file.FullPath) + if kind == KindUnknown && !matchesWalkPattern(file.FullPath) && !inCanonicalFolder(file.FullPath) { + g.Debug("skipping unknown yaml in folder walk: %s", disp) + continue + } + results = append(results, parseBody(disp, file.FullPath, kind, body, opts)) + } + return results +} + +func parseBody(display, abs string, kind Kind, body []byte, opts Options) FileResult { + res := FileResult{Path: display, Kind: kind} + if kind == KindUnknown { + res.Error = "unknown kind" + return res + } + + if kind == KindProject { + return parseProjectFile(res, abs, body, opts) + } + + useCompile := opts.Compile && (kind == KindReplication || kind == KindPipeline || kind == KindBuild) + var ( + parsed any + err error + ) + if useCompile { + parsed, err = parseCompile(kind, abs, body) + } else { + parsed, err = parseDTO(kind, body) + } + if err != nil { + res.Error = err.Error() + return res + } + res.OK = true + res.Compiled = useCompile + res.Parsed = parsed + if kind == KindPipeline { + res.Warnings = append(res.Warnings, pipelineStepPathWarnings(abs, parsed)...) + } + if kind == KindReplication { + res.Warnings = append(res.Warnings, unknownConnWarnings(parsed)...) + } + return res +} + +func parseProjectFile(res FileResult, abs string, body []byte, opts Options) FileResult { + parsed, warnings, errs := validateProject(abs, body, opts) + res.Warnings = warnings + if len(errs) > 0 { + res.Error = strings.Join(errs, "; ") + return res + } + res.OK = true + res.Parsed = parsed + return res +} + +// 5-field cron, or a descriptor such as @daily / @every 1h. +var ( + cronSpecParser = cron.NewParser(cron.Minute | cron.Hour | cron.Dom | cron.Month | cron.Dow) + cronDescriptorParser = cron.NewParser(cron.Descriptor) +) + +func validateProject(abs string, body []byte, opts Options) (parsed any, warnings, errs []string) { + var raw struct { + Name string `yaml:"name"` + ProjectID string `yaml:"project_id"` + Jobs map[string]any `yaml:"jobs"` + } + if err := yaml.Unmarshal(body, &raw); err != nil { + return nil, nil, []string{fmt.Sprintf("invalid project yaml: %v", err)} + } + + dir := filepath.Dir(abs) + for _, key := range sortedKeys(raw.Jobs) { + jm, ok := asMap(raw.Jobs[key]) + if !ok { + errs = append(errs, fmt.Sprintf("job %q must be a mapping", key)) + continue + } + file := strings.TrimSpace(cast.ToString(jm["file"])) + if file == "" { + errs = append(errs, fmt.Sprintf("job %q is missing required key 'file'", key)) + continue + } + if err := validateModeIn(jm, "job "+key); err != nil { + errs = append(errs, err.Error()) + } + + schedules := toStringSlice(jm["schedules"]) + for _, sched := range schedules { + if err := validateCron(sched); err != nil { + errs = append(errs, fmt.Sprintf("job %q has an invalid schedule %q: %v", key, sched, err)) + } + } + if len(schedules) > 1 { + warnings = append(warnings, fmt.Sprintf("job %q has %d schedules; only the first cron fires", key, len(schedules))) + } + + absFile := file + if !filepath.IsAbs(absFile) { + absFile = filepath.Join(dir, file) + } + info, err := os.Stat(absFile) + if err != nil { + errs = append(errs, fmt.Sprintf("job %q file %s does not exist", key, file)) + continue + } + if info.IsDir() { + errs = append(errs, fmt.Sprintf("job %q file %s is a directory", key, file)) + continue + } + fileBody, err := os.ReadFile(absFile) + if err != nil { + errs = append(errs, fmt.Sprintf("job %q file %s: %s", key, file, err.Error())) + continue + } + kind := DetectFileKind(fileBody, absFile) + if kind != KindReplication && kind != KindPipeline { + errs = append(errs, fmt.Sprintf("job %q file %s is a %s; expected a replication or pipeline", key, file, kind)) + continue + } + jobRes := parseBody(file, absFile, kind, fileBody, opts) + if !jobRes.OK { + errs = append(errs, fmt.Sprintf("job %q file %s does not parse: %s", key, file, jobRes.Error)) + continue + } + if kind != KindReplication { + continue + } + streams, ok := asMap(asMapOrEmpty(jobRes.Parsed)["streams"]) + if !ok { + continue + } + for _, name := range toStringSlice(jm["streams"]) { + if _, found := streams[name]; !found { + warnings = append(warnings, fmt.Sprintf("job %q stream %q is not in %s", key, name, file)) + } + } + } + + dto := ProjectDTO{Name: raw.Name, ProjectID: raw.ProjectID, Jobs: raw.Jobs} + return Redact(dtoToMap(dto)), warnings, errs +} + +func validateCron(spec string) error { + expr := strings.TrimSpace(spec) + if expr == "" { + return fmt.Errorf("schedule is empty") + } + if strings.HasPrefix(expr, "@every") { + parts := strings.Split(expr, " ") + if len(parts) == 2 && strings.HasSuffix(parts[1], "d") { + if days := cast.ToInt(strings.TrimSuffix(parts[1], "d")); days > 0 { + expr = fmt.Sprintf("@every %dh", days*24) + } + } + } + if _, err := cronSpecParser.Parse(expr); err == nil { + return nil + } else if _, err2 := cronDescriptorParser.Parse(expr); err2 == nil { + return nil + } else { + return err + } +} + +func matchesWalkPattern(path string) bool { + base := strings.ToLower(filepath.Base(path)) + if isManifestName(filepath.Base(path)) || isBuildConfigName(filepath.Base(path)) || isEnvFileName(filepath.Base(path)) { + return true + } + yaml := strings.HasSuffix(base, ".yaml") || strings.HasSuffix(base, ".yml") + if !yaml { + return false + } + return strings.HasPrefix(base, "replication") || strings.HasPrefix(base, "pipeline") +} + +func inCanonicalFolder(path string) bool { + for _, part := range strings.Split(filepath.ToSlash(path), "/") { + switch part { + case "replications", "pipelines", "specs", "models": + return true + } + } + return false +} + +func displayPath(userPath, abs string) string { + if userPath == "" { + return filepath.ToSlash(abs) + } + if filepath.IsAbs(userPath) { + return filepath.ToSlash(abs) + } + return filepath.ToSlash(filepath.Clean(userPath)) +} + +func walkDisplayPath(userPath, absDir, absFile string) string { + rel, err := filepath.Rel(absDir, absFile) + if err != nil { + return filepath.ToSlash(absFile) + } + if filepath.IsAbs(userPath) { + return filepath.ToSlash(filepath.Join(absDir, rel)) + } + return filepath.ToSlash(filepath.Join(filepath.Clean(userPath), rel)) +} + +func AnyFailed(results []FileResult) bool { + for _, r := range results { + if !r.OK { + return true + } + } + return false +} + +func parseCompile(kind Kind, absPath string, body []byte) (any, error) { + env.LoadDotEnvSlingFrom(filepath.Dir(absPath)) + + switch kind { + case KindReplication: + cfg, err := sling.LoadReplicationConfigFromFile(absPath) + if err != nil { + return nil, err + } + if err = cfg.Compile(nil); err != nil { + return nil, err + } + return Redact(compiledReplication(cfg)), nil + case KindPipeline: + pipeline, err := sling.LoadPipelineConfigFromFile(absPath) + if err != nil { + return nil, err + } + if err := validatePipelineSteps(pipeline.Steps, ""); err != nil { + return nil, err + } + dto := PipelineDTO{Steps: pipeline.Steps, Env: pipeline.Env} + return Redact(dtoToMap(dto)), nil + case KindBuild: + projDir := absPath + if isBuildConfigName(filepath.Base(absPath)) { + projDir = filepath.Dir(absPath) + } + project, err := build.LoadProject(projDir) + if err != nil { + return nil, err + } + return Redact(compiledBuild(project)), nil + default: + return parseDTO(kind, body) + } +} + +func compiledReplication(cfg sling.ReplicationConfig) map[string]any { + streams := map[string]any{} + b, err := json.Marshal(cfg.Streams) + if err == nil { + _ = json.Unmarshal(b, &streams) + } + return dtoToMap(ReplicationDTO{ + Source: cfg.Source, + Target: cfg.Target, + Hooks: cfg.Hooks, + Defaults: cfg.Defaults, + Streams: streams, + Env: cfg.Env, + }) +} + +func compiledBuild(project *build.BuildProject) map[string]any { + models := make([]string, 0, len(project.Models)) + for name := range project.Models { + models = append(models, name) + } + sort.Strings(models) + seeds := make([]string, 0, len(project.Seeds)) + for name := range project.Seeds { + seeds = append(seeds, name) + } + sort.Strings(seeds) + + out := map[string]any{ + "dir": project.Dir, + "models": models, + "seeds": seeds, + "mode": project.Mode, + } + if project.Config != nil { + out["target"] = project.Config.Target + out["defaults"] = project.Config.Defaults + out["vars"] = project.Config.Vars + } + return out +} diff --git a/core/sling/validate/validate_test.go b/core/sling/validate/validate_test.go new file mode 100644 index 000000000..2801764c8 --- /dev/null +++ b/core/sling/validate/validate_test.go @@ -0,0 +1,564 @@ +package validate + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/slingdata-io/sling-cli/core/sling/build" +) + +func TestParseEnvNoInterpolate(t *testing.T) { + os.Setenv("MY_VALIDATE_SECRET", "SuperSecretValueXYZ") + t.Cleanup(func() { os.Unsetenv("MY_VALIDATE_SECRET") }) + + body := []byte(` +connections: + MY_PG: + type: postgres + password: LiteralPass123ABC + secret_access_key: ${MY_VALIDATE_SECRET} +`) + parsed, err := parseDTO(KindEnv, body) + if err != nil { + t.Fatal(err) + } + s := mustJSON(t, parsed) + if strings.Contains(s, "SuperSecretValueXYZ") { + t.Fatal("interpolated env secret into parse output") + } + if strings.Contains(s, "LiteralPass123ABC") { + t.Fatal("literal password leaked") + } + if !strings.Contains(s, "${MY_VALIDATE_SECRET}") { + t.Fatal("env ref was not passed through") + } +} + +func TestParseUnknownExplicit(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "foo.yaml") + if err := os.WriteFile(path, []byte("foo: bar\n"), 0644); err != nil { + t.Fatal(err) + } + res := ParseFile(path, Options{}) + if res.OK || res.Kind != KindUnknown { + t.Fatalf("got ok=%v kind=%q", res.OK, res.Kind) + } +} + +func mustJSON(t *testing.T, v any) string { + t.Helper() + b, err := json.Marshal(v) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestDetectFileKindOrder(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + body string + path string + want Kind + }{ + { + name: "steps beats connections", + body: "steps:\n - type: log\n message: hi\nconnections:\n PG:\n type: postgres\n", + path: "env.yaml", + want: KindPipeline, + }, + { + name: "replication needs source target streams", + body: "source: PG\ntarget: SF\nstreams:\n public.t:\n object: t\n", + path: "foo.yaml", + want: KindReplication, + }, + { + name: "api spec name plus endpoints", + body: "name: demo\nendpoints:\n ping:\n request:\n url: https://example.com\n", + path: "spec.yaml", + want: KindAPISpec, + }, + { + name: "api spec name plus dynamic_endpoints", + body: "name: demo\ndynamic_endpoints:\n - iterate: []\n", + path: "spec.yaml", + want: KindAPISpec, + }, + { + name: "monitor connection plus objects", + body: "connection: PG\nobjects:\n - public.t\n", + path: "mon.yaml", + want: KindMonitor, + }, + { + name: "routine", + body: "routines:\n nightly:\n steps: []\n", + path: "r.yaml", + want: KindRoutine, + }, + { + name: "env connections", + body: "connections:\n PG:\n type: postgres\n", + path: "other.yaml", + want: KindEnv, + }, + { + name: "content wins over env.yaml name", + body: "steps:\n - type: log\n message: hi\n", + path: "env.yaml", + want: KindPipeline, + }, + { + name: "env.yaml fallback when content unknown", + body: "foo: bar\n", + path: "env.yaml", + want: KindEnv, + }, + { + name: "sling_build.yml path rule", + body: "target: POSTGRES\ndefaults:\n mode: full-refresh\n", + path: "sling_build.yml", + want: KindBuild, + }, + { + name: "unknown yaml stays unknown", + body: "services:\n db:\n image: postgres\n", + path: "docker-compose.yml", + want: KindUnknown, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := DetectFileKind([]byte(tt.body), tt.path) + if got != tt.want { + t.Fatalf("DetectFileKind() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestDetectFileKindBuildDir(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := filepath.Join(dir, build.ConfigFileName) + if err := os.WriteFile(path, []byte("target: POSTGRES\n"), 0644); err != nil { + t.Fatal(err) + } + got := DetectFileKind(nil, dir) + if got != KindBuild { + t.Fatalf("directory with sling_build.yml: got %q want %q", got, KindBuild) + } +} + +func TestRedactEnvRefsPassThrough(t *testing.T) { + t.Parallel() + in := map[string]any{ + "connections": map[string]any{ + "PG": map[string]any{ + "password": "LiteralPass123ABC", + "secret_access_key": "${MY_VALIDATE_SECRET}", + }, + }, + } + out, _ := Redact(in).(map[string]any) + conns := out["connections"].(map[string]any) + pg := conns["PG"].(map[string]any) + if pg["password"] != "***" { + t.Fatalf("password = %#v, want ***", pg["password"]) + } + if pg["secret_access_key"] != "${MY_VALIDATE_SECRET}" { + t.Fatalf("ref = %#v, want ${MY_VALIDATE_SECRET}", pg["secret_access_key"]) + } +} + +func writeFile(t *testing.T, dir, name, body string) string { + t.Helper() + path := filepath.Join(dir, name) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(body), 0644); err != nil { + t.Fatal(err) + } + return path +} + +const testReplication = ` +source: '{source}' +target: LOCAL +streams: + main.example: + object: raw.example +` + +func TestProjectManifestExplicit(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "replications/r.yaml", testReplication) + path := writeFile(t, dir, "sling_project.yml", ` +name: demo +jobs: + daily: + file: replications/r.yaml + schedules: ["0 6 * * *"] +`) + res := ParseFile(path, Options{}) + if !res.OK || res.Kind != KindProject { + t.Fatalf("got ok=%v kind=%q err=%s", res.OK, res.Kind, res.Error) + } + if len(res.Warnings) != 0 { + t.Fatalf("unexpected warnings: %v", res.Warnings) + } +} + +func TestProjectManifestInWalk(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "replications/r.yaml", testReplication) + writeFile(t, dir, "sling_project.yml", "name: demo\n") + + var found bool + for _, r := range ParsePaths([]string{dir}, Options{}) { + if r.Kind == KindProject { + found = true + if !r.OK { + t.Fatalf("manifest failed: %s", r.Error) + } + } + } + if !found { + t.Fatal("walk did not include the manifest") + } +} + +func TestProjectBadCron(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "replications/r.yaml", testReplication) + path := writeFile(t, dir, "sling_project.yml", ` +jobs: + daily: + file: replications/r.yaml + schedules: ["not a cron"] +`) + res := ParseFile(path, Options{}) + if res.OK { + t.Fatal("bad cron should fail") + } + if !strings.Contains(res.Error, "invalid schedule") { + t.Fatalf("unclear error: %s", res.Error) + } +} + +func TestProjectCronDescriptors(t *testing.T) { + for _, spec := range []string{"@daily", "@hourly", "@every 1h", "@every 2d", "0 6 * * *", "*/5 * * * *"} { + if err := validateCron(spec); err != nil { + t.Fatalf("%q should be valid: %s", spec, err) + } + } + for _, spec := range []string{"", "not a cron", "0 6 * *", "@nope"} { + if err := validateCron(spec); err == nil { + t.Fatalf("%q should be invalid", spec) + } + } +} + +func TestProjectMultiScheduleWarns(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "replications/r.yaml", testReplication) + path := writeFile(t, dir, "sling_project.yml", ` +jobs: + daily: + file: replications/r.yaml + schedules: ["0 6 * * *", "0 7 * * *"] +`) + res := ParseFile(path, Options{}) + if !res.OK { + t.Fatalf("multi schedule should warn, not fail: %s", res.Error) + } + if len(res.Warnings) != 1 || !strings.Contains(res.Warnings[0], "only the first cron fires") { + t.Fatalf("got warnings %v", res.Warnings) + } + if AnyFailed([]FileResult{res}) { + t.Fatal("warnings must not fail the run") + } +} + +func TestProjectEmptyJobFile(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "sling_project.yml", ` +jobs: + daily: + schedules: ["0 6 * * *"] +`) + res := ParseFile(path, Options{}) + if res.OK { + t.Fatal("job without file should fail") + } + if !strings.Contains(res.Error, "'file'") { + t.Fatalf("unclear error: %s", res.Error) + } +} + +func TestProjectJobMissingFile(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "sling_project.yml", ` +jobs: + daily: + file: replications/nope.yaml +`) + res := ParseFile(path, Options{}) + if res.OK { + t.Fatal("missing job file should fail") + } + if !strings.Contains(res.Error, "does not exist") { + t.Fatalf("unclear error: %s", res.Error) + } +} + +func TestProjectJobWrongKind(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "env.yaml", "connections:\n MY_PG:\n type: postgres\n") + path := writeFile(t, dir, "sling_project.yml", ` +jobs: + daily: + file: env.yaml +`) + res := ParseFile(path, Options{}) + if res.OK { + t.Fatal("env-kind job file should fail") + } + if !strings.Contains(res.Error, "expected a replication or pipeline") { + t.Fatalf("unclear error: %s", res.Error) + } +} + +func TestProjectStreamOverrideMissWarns(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "replications/r.yaml", testReplication) + path := writeFile(t, dir, "sling_project.yml", ` +jobs: + daily: + file: replications/r.yaml + streams: [main.example, not_there] +`) + res := ParseFile(path, Options{}) + if !res.OK { + t.Fatalf("stream miss should warn, not fail: %s", res.Error) + } + if len(res.Warnings) != 1 || !strings.Contains(res.Warnings[0], "not_there") { + t.Fatalf("got warnings %v", res.Warnings) + } +} + +func TestPipelineStepBadPathWarns(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "pipelines/p.yaml", ` +steps: + - type: replication + path: replications/nope.yaml +`) + res := ParseFile(path, Options{}) + if !res.OK { + t.Fatalf("bad step path should warn, not fail: %s", res.Error) + } + if len(res.Warnings) != 1 || !strings.Contains(res.Warnings[0], "does not exist") { + t.Fatalf("got warnings %v", res.Warnings) + } +} + +func TestPipelineStepGoodPathNoWarn(t *testing.T) { + dir := t.TempDir() + writeFile(t, dir, "pipelines/replications/r.yaml", testReplication) + path := writeFile(t, dir, "pipelines/p.yaml", ` +steps: + - type: replication + path: replications/r.yaml +`) + res := ParseFile(path, Options{}) + if !res.OK { + t.Fatalf("failed: %s", res.Error) + } + if len(res.Warnings) != 0 { + t.Fatalf("unexpected warnings: %v", res.Warnings) + } +} + +func TestParsePipelineNestedUnknownStepType(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + body string + compile bool + wantErr string + }{ + { + name: "top-level unknown type", + body: ` +steps: + - type: run + source: LOCAL +`, + wantErr: `pipeline step 0 has unknown type "run"`, + }, + { + name: "nested group unknown type", + body: ` +steps: + - type: group + loop: [1] + steps: + - type: set + key: table_name + value: t + - type: run + source: LOCAL + target: DUCKDB + - type: log + message: done +`, + wantErr: `pipeline step 0.steps.1 has unknown type "run"`, + }, + { + name: "nested group unknown type compile", + body: ` +steps: + - type: group + loop: [1] + steps: + - type: log + message: hi + - type: run + source: LOCAL +`, + compile: true, + wantErr: `pipeline step 0.steps.1 has unknown type "run"`, + }, + { + name: "doubly nested group unknown type", + body: ` +steps: + - type: group + steps: + - type: group + steps: + - type: run + source: LOCAL +`, + wantErr: `pipeline step 0.steps.0.steps.0 has unknown type "run"`, + }, + { + name: "shorthand group unknown nested type", + body: ` +steps: + - group: + loop: [1] + steps: + - type: run + source: LOCAL +`, + wantErr: `pipeline step 0.steps.0 has unknown type "run"`, + }, + { + name: "nested non-mapping step", + body: ` +steps: + - type: group + steps: + - not-a-mapping +`, + wantErr: `pipeline step 0.steps.0 must be a mapping`, + }, + { + name: "valid nested group", + body: ` +steps: + - type: group + loop: [1] + steps: + - type: log + message: hi + - type: query + connection: DUCKDB + query: select 1 +`, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + dir := t.TempDir() + path := writeFile(t, dir, "loop.yaml", tt.body) + res := ParseFile(path, Options{Compile: tt.compile}) + if tt.wantErr == "" { + if !res.OK { + t.Fatalf("valid nested group failed: %s", res.Error) + } + return + } + if res.OK { + t.Fatal("expected unknown nested step type to fail") + } + if !strings.Contains(res.Error, tt.wantErr) { + t.Fatalf("error %q does not contain %q", res.Error, tt.wantErr) + } + }) + } +} + +func TestLintReplicationUnknownConnWarns(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "r.yaml", ` +source: WAREHOUSE_PROD_EVAL_XYZ +target: LOCAL +defaults: + object: main.{stream_table} +streams: + public.orders: + mode: full-refresh +`) + res := ParseFile(path, Options{}) + if !res.OK { + t.Fatalf("unknown conn should warn, not fail: %s", res.Error) + } + if len(res.Warnings) != 1 || !strings.Contains(res.Warnings[0], "WAREHOUSE_PROD_EVAL_XYZ") { + t.Fatalf("got warnings %v", res.Warnings) + } +} + +func TestLintAPISpecMissingURL(t *testing.T) { + dir := t.TempDir() + path := writeFile(t, dir, "spec.yaml", ` +name: missing-url +endpoints: + ping: + request: + method: GET +`) + res := ParseFile(path, Options{}) + if res.OK { + t.Fatal("missing request url should fail") + } + if !strings.Contains(strings.ToLower(res.Error), "url") { + t.Fatalf("unclear error: %s", res.Error) + } +} + +func TestResultRowCompiled(t *testing.T) { + row := resultRow(FileResult{Kind: KindReplication, OK: true, Compiled: false}, false) + if row["compiled"] != false { + t.Fatalf("compiled = %#v, want false", row["compiled"]) + } + row = resultRow(FileResult{Kind: KindReplication, OK: true, Compiled: true}, false) + if row["compiled"] != true { + t.Fatalf("compiled = %#v, want true", row["compiled"]) + } +} diff --git a/tests/suite.cli.validate.yaml b/tests/suite.cli.validate.yaml new file mode 100644 index 000000000..2b01c9e6d --- /dev/null +++ b/tests/suite.cli.validate.yaml @@ -0,0 +1,354 @@ +- id: 500 + name: 'sling validate replication fixture' + run: 'sling validate tests/replications/r.00.yaml --json' + output_contains: + - '"kind": "replication"' + - '"ok": true' + - '"compiled": true' + - 'tests/replications/r.00.yaml' + - '"source"' + - '"streams"' + +- id: 501 + name: 'sling validate pipeline fixture' + run: 'sling validate tests/pipelines/p.01.yaml --json' + output_contains: + - '"kind": "pipeline"' + - '"ok": true' + - 'tests/pipelines/p.01.yaml' + - '"steps"' + +- id: 502 + name: 'sling validate env fixture' + run: 'sling validate tests/validate/env.yaml --json' + output_contains: + - '"kind": "env"' + - '"ok": true' + - 'tests/validate/env.yaml' + - '"connections"' + - 'MY_PG' + +- id: 503 + name: 'sling validate api spec fixture' + run: 'sling validate tests/validate/spec.yaml --json' + output_contains: + - '"kind": "api_spec"' + - '"ok": true' + - 'validate-test-api' + - '"endpoints"' + - 'ping' + output_does_not_contain: + - 'LiteralTokenDoNotLeak' + +- id: 504 + name: 'sling validate build project sling_build.yml' + run: 'sling validate tests/build/sample_project/sling_build.yml --json' + output_contains: + - '"kind": "build"' + - '"ok": true' + - 'POSTGRES' + - 'sling_build.yml' + +- id: 505 + name: 'sling validate multi-file one invocation' + run: 'sling validate tests/replications/r.00.yaml tests/pipelines/p.01.yaml tests/validate/env.yaml --json' + output_contains: + - '"kind": "replication"' + - '"kind": "pipeline"' + - '"kind": "env"' + - 'tests/replications/r.00.yaml' + - 'tests/pipelines/p.01.yaml' + - 'tests/validate/env.yaml' + +- id: 506 + name: 'sling validate folder walk skips docker-compose.yml' + run: 'sling validate tests/validate/walk --json' + output_contains: + - '"kind": "replication"' + - 'replication_ok.yaml' + - '"ok": true' + output_does_not_contain: + - 'docker-compose.yml' + - 'ComposePassDoNotParse' + - 'notes.yaml' + +- id: 507 + name: 'sling validate -q success' + run: 'sling validate -q tests/validate/env.yaml tests/validate/spec.yaml' + output_does_not_contain: + - '"kind"' + - 'LiteralPass123ABC' + +- id: 508 + name: 'sling validate -q fail unknown kind' + err: true + run: 'sling validate -q tests/validate/unknown.yaml' + output_contains: + - 'unknown kind' + +- id: 509 + name: 'sling validate unknown-kind explicit file error' + err: true + run: 'sling validate tests/validate/unknown.yaml --json' + output_contains: + - '"kind": "unknown"' + - '"ok": false' + - 'unknown kind' + - 'tests/validate/unknown.yaml' + +- id: 510 + name: 'sling validate env redacts password value' + run: 'sling validate tests/validate/env.yaml --json' + output_contains: + - '"password": "***"' + - '"token": "***"' + - '${MY_VALIDATE_SECRET}' + output_does_not_contain: + - 'LiteralPass123ABC' + - 'LiteralTokenXYZ' + +- id: 511 + name: 'sling validate env does not interpolate secret env var' + env: + MY_VALIDATE_SECRET: SuperSecretValueXYZ + run: 'sling validate tests/validate/env.yaml --json' + output_contains: + - '${MY_VALIDATE_SECRET}' + - '"kind": "env"' + output_does_not_contain: + - 'SuperSecretValueXYZ' + - 'LiteralPass123ABC' + +- id: 512 + name: 'sling validate --parse-only keeps ${VAR} un-interpolated' + env: + MY_VALIDATE_SECRET: SuperSecretValueXYZ + run: 'sling validate --parse-only tests/replications/r.00.yaml --json' + output_contains: + - '"kind": "replication"' + - '"ok": true' + - '"compiled": false' + - '{source}' + output_does_not_contain: + - 'SuperSecretValueXYZ' + +- id: 513 + name: 'sling validate --detailed renders per-kind tables' + run: 'sling validate --detailed tests/replications/r.00.yaml tests/pipelines/p.01.yaml tests/validate/env.yaml tests/validate/spec.yaml tests/build/sample_project/sling_build.yml' + output_contains: + - '# Replications' + - '# Pipelines' + - '# API Specs' + - '# Connections' + - '# Build Projects' + - 'SQLITE' + - 'test1.1.csv' + - 'full-refresh' + - 'ping' + - 'https://example.com/ping' + - 'MY_PG' + - 'postgres' + - 'POSTGRES' + output_does_not_contain: + - 'LiteralPass123ABC' + - 'LiteralTokenXYZ' + - 'LiteralTokenDoNotLeak' + +- id: 514 + name: 'sling validate --detailed shows pipeline step type and id' + run: 'sling validate --detailed --parse-only tests/pipelines/p.04.test_inspect_hook.yaml' + output_contains: + - '# Pipelines' + - 'create_test_table' + - 'inspect_postgres_table' + - 'inspect' + - 'query' + output_does_not_contain: + - 'ORDER' + - 'ON_FAILURE' + +- id: 515 + name: 'sling validate --detailed reports errors in a table' + err: true + run: 'sling validate --detailed tests/validate/unknown.yaml' + output_contains: + - '# Errors' + - 'unknown kind' + - 'tests/validate/unknown.yaml' + +- id: 516 + name: 'sling validate --help shows the new verb and -q' + run: 'sling validate --help' + output_contains: + - 'Validate Sling files' + - '-q --quiet' + - 'project' + - '--parse-only' + - 'ready to run' + output_does_not_contain: + - '--no-compile' + +- id: 517 + name: 'sling parse no longer validates' + run: 'sling parse tests/validate/env.yaml --json' + output_contains: + - 'Slings data from a data source to a data target.' + output_does_not_contain: + - '"kind"' + - 'MY_PG' + +- id: 518 + name: 'sling validate bare at a project root' + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + sling init --target POSTGRES --source SQLITE --yes + sling validate --json + env: + SLING_PROJECT_TOKEN: '' + output_contains: + - '"kind": "project"' + - '"kind": "pipeline"' + - '"kind": "replication"' + - '"ok": true' + - sling_project.yml + +- id: 519 + name: 'sling validate bare from a subfolder' + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + sling init --target POSTGRES --source SQLITE --yes + cd replications + sling validate --json + env: + SLING_PROJECT_TOKEN: '' + output_contains: + - '"kind": "project"' + - '"ok": true' + +- id: 570 + name: 'sling validate project job with a missing file and a bad cron' + err: true + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + sling init --target POSTGRES --source SQLITE --yes + printf 'jobs:\n gone:\n file: replications/nope.yaml\n' >> sling_project.yml + sling validate sling_project.yml --json + env: + SLING_PROJECT_TOKEN: '' + output_contains: + - '"ok": false' + - does not exist + +- id: 571 + name: 'sling validate project bad cron fails' + err: true + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + sling init --target POSTGRES --source SQLITE --yes + printf 'jobs:\n daily:\n file: pipelines/daily.yaml\n schedules: ["not a cron"]\n' >> sling_project.yml + sling validate sling_project.yml --json + env: + SLING_PROJECT_TOKEN: '' + output_contains: + - invalid schedule + - '"ok": false' + +- id: 572 + name: 'sling validate warns on two schedules without failing' + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + sling init --target POSTGRES --source SQLITE --yes + printf 'jobs:\n daily:\n file: pipelines/daily.yaml\n schedules: ["0 6 * * *", "0 7 * * *"]\n' >> sling_project.yml + sling validate sling_project.yml --json + env: + SLING_PROJECT_TOKEN: '' + output_contains: + - '"warnings"' + - only the first cron fires + - '"ok": true' + +- id: 573 + name: 'sling validate warns on a pipeline step with a bad path' + run: | + DIR=$(mktemp -d) + cd "$DIR" + printf 'steps:\n - type: replication\n path: replications/nope.yaml\n' > p.yaml + sling validate p.yaml --json + output_contains: + - '"warnings"' + - does not exist + - '"ok": true' + +- id: 574 + name: 'MCP replication action validate works and parse errors' + run: | + printf '%s\n' \ + '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2024-11-05","capabilities":{},"clientInfo":{"name":"t","version":"1"}}}' \ + '{"jsonrpc":"2.0","method":"notifications/initialized"}' \ + '{"jsonrpc":"2.0","id":2,"method":"tools/call","params":{"name":"replication","arguments":{"action":"validate","input":{"file_path":"tests/replications/r.00.yaml","compile":false}}}}' \ + '{"jsonrpc":"2.0","id":3,"method":"tools/call","params":{"name":"replication","arguments":{"action":"parse","input":{"file_path":"tests/replications/r.00.yaml","compile":false}}}}' \ + | sling mcp 2>/dev/null + output_contains: + - "'invalid' action: parse" + - STREAM_NAME + - 'Parse-only' + - 'compiled: false' + +- id: 580 + name: 'sling validate compile fails without object' + err: true + run: 'sling validate tests/validate/repl_missing_object.yaml --json' + output_contains: + - '"ok": false' + - 'need to specify `object`' + +- id: 583 + name: 'sling validate --parse-only warns on unknown connection' + run: 'sling validate --parse-only tests/validate/repl_unknown_conn.yaml --json' + output_contains: + - '"ok": true' + - '"compiled": false' + - '"warnings"' + - WAREHOUSE_PROD_EVAL_XYZ + +- id: 584 + name: 'sling validate --parse-only fails api spec without url' + err: true + run: 'sling validate --parse-only tests/validate/spec_missing_url.yaml --json' + output_contains: + - '"ok": false' + - 'url' + +- id: 585 + name: 'sling validate --parse-only fails api spec with unknown depends_on' + err: true + run: 'sling validate --parse-only tests/validate/spec_bad_depends.yaml --json' + output_contains: + - '"ok": false' + - 'does_not_exist' + +- id: 586 + name: 'sling validate fails on nested group unknown step type' + err: true + run: 'sling validate tests/validate/pipeline_nested_bad_type.yaml --json' + output_contains: + - '"ok": false' + - 'pipeline step 0.steps.1 has unknown type "run"' + +- id: 587 + name: 'sling validate --parse-only fails on nested group unknown step type' + err: true + run: 'sling validate --parse-only tests/validate/pipeline_nested_bad_type.yaml --json' + output_contains: + - '"ok": false' + - 'pipeline step 0.steps.1 has unknown type "run"' diff --git a/tests/validate/env.yaml b/tests/validate/env.yaml new file mode 100644 index 000000000..8a58951d1 --- /dev/null +++ b/tests/validate/env.yaml @@ -0,0 +1,14 @@ +connections: + MY_PG: + type: postgres + host: localhost + user: sling + database: sling + password: LiteralPass123ABC + secret_access_key: ${MY_VALIDATE_SECRET} + MY_API: + type: api + secrets: + token: LiteralTokenXYZ +env: + FOO: bar diff --git a/tests/validate/pipeline_nested_bad_type.yaml b/tests/validate/pipeline_nested_bad_type.yaml new file mode 100644 index 000000000..8eba58b2a --- /dev/null +++ b/tests/validate/pipeline_nested_bad_type.yaml @@ -0,0 +1,12 @@ +steps: + - type: group + loop: [1] + steps: + - type: set + key: table_name + value: t + - type: run + source: LOCAL + target: DUCKDB + - type: log + message: done diff --git a/tests/validate/repl_missing_object.yaml b/tests/validate/repl_missing_object.yaml new file mode 100644 index 000000000..e56e5f6fa --- /dev/null +++ b/tests/validate/repl_missing_object.yaml @@ -0,0 +1,5 @@ +source: LOCAL +target: SQLITE +streams: + public.orders: + mode: full-refresh diff --git a/tests/validate/repl_unknown_conn.yaml b/tests/validate/repl_unknown_conn.yaml new file mode 100644 index 000000000..1a183978e --- /dev/null +++ b/tests/validate/repl_unknown_conn.yaml @@ -0,0 +1,7 @@ +source: WAREHOUSE_PROD_EVAL_XYZ +target: LOCAL +defaults: + object: main.{stream_table} +streams: + public.orders: + mode: full-refresh diff --git a/tests/validate/spec.yaml b/tests/validate/spec.yaml new file mode 100644 index 000000000..6e9ca7e55 --- /dev/null +++ b/tests/validate/spec.yaml @@ -0,0 +1,12 @@ +name: validate-test-api +description: tiny spec for validate tests +authentication: + type: static + headers: + Authorization: Bearer LiteralTokenDoNotLeak +endpoints: + ping: + description: ping + request: + url: https://example.com/ping + method: GET diff --git a/tests/validate/spec_bad_depends.yaml b/tests/validate/spec_bad_depends.yaml new file mode 100644 index 000000000..5d602e37b --- /dev/null +++ b/tests/validate/spec_bad_depends.yaml @@ -0,0 +1,6 @@ +name: bad-depends +endpoints: + child: + request: + url: https://example.com/child + depends_on: [does_not_exist] diff --git a/tests/validate/spec_missing_url.yaml b/tests/validate/spec_missing_url.yaml new file mode 100644 index 000000000..132f9f9a6 --- /dev/null +++ b/tests/validate/spec_missing_url.yaml @@ -0,0 +1,5 @@ +name: missing-url +endpoints: + ping: + request: + method: GET diff --git a/tests/validate/unknown.yaml b/tests/validate/unknown.yaml new file mode 100644 index 000000000..1b4817b69 --- /dev/null +++ b/tests/validate/unknown.yaml @@ -0,0 +1,2 @@ +foo: bar +hello: world diff --git a/tests/validate/walk/docker-compose.yml b/tests/validate/walk/docker-compose.yml new file mode 100644 index 000000000..20a78b07e --- /dev/null +++ b/tests/validate/walk/docker-compose.yml @@ -0,0 +1,5 @@ +services: + db: + image: postgres:16 + environment: + POSTGRES_PASSWORD: ComposePassDoNotParse diff --git a/tests/validate/walk/notes.yaml b/tests/validate/walk/notes.yaml new file mode 100644 index 000000000..02d3320e7 --- /dev/null +++ b/tests/validate/walk/notes.yaml @@ -0,0 +1,3 @@ +# not a sling file +version: "3" +x-unused: true diff --git a/tests/validate/walk/replication_ok.yaml b/tests/validate/walk/replication_ok.yaml new file mode 100644 index 000000000..cf5de08c0 --- /dev/null +++ b/tests/validate/walk/replication_ok.yaml @@ -0,0 +1,7 @@ +source: LOCAL +target: SQLITE +streams: + file://./tests/files/test1.csv: + object: main.test +defaults: + mode: full-refresh From a7eec7a58b7214d4e47620de8b93f786ea65b810 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Thu, 27 Aug 2026 10:55:45 -0300 Subject: [PATCH 08/30] feat(project): add job planning and project scaffolding Introduce JobSpec and StampedJob types along with PlanJobs, which diffs manifest jobs against stamped platform jobs and produces a plan of create/update/delete/orphan/rename/keep actions, with support for rename mapping and prune options. Add scaffold templates for bootstrapping new projects (sling_project.yml, replication, daily pipeline, and staging model templates) plus scaffold tests covering file generation and placeholder rendering. --- core/sling/project/jobs.go | 317 ++++++++++++++++++ core/sling/project/jobs_test.go | 287 ++++++++++++++++ core/sling/project/manifest.go | 260 ++++++++++++++ core/sling/project/scaffold.go | 207 ++++++++++++ core/sling/project/scaffold/gitignore | 5 + .../scaffold/models/marts/fct_ENTITY.sql.tmpl | 9 + .../scaffold/models/sling_build.yml.tmpl | 7 + .../staging/stg_SOURCE_SLUG__ENTITY.sql.tmpl | 5 + .../scaffold/pipelines/daily.yaml.tmpl | 20 ++ .../replications/SOURCE_SLUG.yaml.tmpl | 10 + .../project/scaffold/sling_project.yml.tmpl | 9 + core/sling/project/scaffold_test.go | 121 +++++++ tests/suite.cli.project.yaml | 202 +++++++++++ 13 files changed, 1459 insertions(+) create mode 100644 core/sling/project/jobs.go create mode 100644 core/sling/project/jobs_test.go create mode 100644 core/sling/project/manifest.go create mode 100644 core/sling/project/scaffold.go create mode 100644 core/sling/project/scaffold/gitignore create mode 100644 core/sling/project/scaffold/models/marts/fct_ENTITY.sql.tmpl create mode 100644 core/sling/project/scaffold/models/sling_build.yml.tmpl create mode 100644 core/sling/project/scaffold/models/staging/stg_SOURCE_SLUG__ENTITY.sql.tmpl create mode 100644 core/sling/project/scaffold/pipelines/daily.yaml.tmpl create mode 100644 core/sling/project/scaffold/replications/SOURCE_SLUG.yaml.tmpl create mode 100644 core/sling/project/scaffold/sling_project.yml.tmpl create mode 100644 core/sling/project/scaffold_test.go create mode 100644 tests/suite.cli.project.yaml diff --git a/core/sling/project/jobs.go b/core/sling/project/jobs.go new file mode 100644 index 000000000..69347f62b --- /dev/null +++ b/core/sling/project/jobs.go @@ -0,0 +1,317 @@ +package project + +import ( + "fmt" + "sort" + "strings" + + "github.com/flarco/g" +) + +const multiScheduleWarn = "only the first cron fires" + +// JobSpec is one entry under manifest jobs:. Extra YAML keys are ignored. +type JobSpec struct { + File string `yaml:"file,omitempty" json:"file,omitempty"` + Schedules []string `yaml:"schedules,omitempty" json:"schedules,omitempty"` + Streams []string `yaml:"streams,omitempty" json:"streams,omitempty"` + Mode string `yaml:"mode,omitempty" json:"mode,omitempty"` + Variables map[string]string `yaml:"variables,omitempty" json:"variables,omitempty"` + Retries *int `yaml:"retries,omitempty" json:"retries,omitempty"` + Timezone string `yaml:"timezone,omitempty" json:"timezone,omitempty"` +} + +// StampedJob is a platform job keyed by source_key. +type StampedJob struct { + SourceKey string + Name string + File string + Schedules []string + Streams []string + Mode string + Variables map[string]string + Retries *int + Timezone string +} + +// PlanAction is one row in a job deploy plan. +type PlanAction string + +const ( + PlanCreate PlanAction = "create" + PlanUpdate PlanAction = "update" + PlanDelete PlanAction = "delete" + PlanOrphan PlanAction = "orphan" + PlanRename PlanAction = "rename" + PlanKeep PlanAction = "keep" +) + +// PlanItem is one planned change. +type PlanItem struct { + Action PlanAction + SourceKey string + OldKey string + Spec JobSpec + Warning string +} + +// PlanOptions controls rename mapping and prune. +type PlanOptions struct { + Renames map[string]string + Prune bool +} + +// JobPlan is the diff of manifest jobs vs stamped jobs. +type JobPlan struct { + Items []PlanItem + Warnings []string +} + +// PlanJobs diffs manifest jobs against the stamped (source_key) set. +func PlanJobs(m Manifest, stamped []StampedJob, opts PlanOptions) JobPlan { + plan := JobPlan{} + jobs := m.Jobs + if jobs == nil { + jobs = map[string]JobSpec{} + } + + stampedByKey := map[string]StampedJob{} + for _, s := range stamped { + if strings.TrimSpace(s.SourceKey) == "" { + continue + } + stampedByKey[s.SourceKey] = s + } + + renames := opts.Renames + if renames == nil { + renames = map[string]string{} + } + oldOfNew := map[string]string{} + for oldKey, newKey := range renames { + if oldKey == "" || newKey == "" || oldKey == newKey { + continue + } + sj, ok := stampedByKey[oldKey] + if !ok { + plan.Warnings = append(plan.Warnings, g.F("rename source %s is not a managed job", oldKey)) + continue + } + if _, exists := jobs[newKey]; !exists { + plan.Warnings = append(plan.Warnings, g.F("rename target %s is not in the manifest", newKey)) + continue + } + delete(stampedByKey, oldKey) + sj.SourceKey = newKey + stampedByKey[newKey] = sj + oldOfNew[newKey] = oldKey + } + + keys := make([]string, 0, len(jobs)) + for k := range jobs { + keys = append(keys, k) + } + sort.Strings(keys) + + used := map[string]bool{} + for _, key := range keys { + spec := jobs[key] + if w := spec.scheduleWarning(key); w != "" { + plan.Warnings = append(plan.Warnings, w) + } + if strings.TrimSpace(spec.File) == "" { + plan.Warnings = append(plan.Warnings, g.F("job %s has no file", key)) + } + sj, ok := stampedByKey[key] + if !ok { + plan.Items = append(plan.Items, PlanItem{Action: PlanCreate, SourceKey: key, Spec: spec}) + continue + } + used[key] = true + item := PlanItem{SourceKey: key, Spec: spec} + if oldKey, renamed := oldOfNew[key]; renamed { + item.Action = PlanRename + item.OldKey = oldKey + plan.Items = append(plan.Items, item) + continue + } + if jobSpecEqual(spec, sj) { + item.Action = PlanKeep + } else { + item.Action = PlanUpdate + } + plan.Items = append(plan.Items, item) + } + + orphanKeys := make([]string, 0, len(stampedByKey)) + for key := range stampedByKey { + if used[key] { + continue + } + if _, inManifest := jobs[key]; inManifest { + continue + } + orphanKeys = append(orphanKeys, key) + } + sort.Strings(orphanKeys) + for _, key := range orphanKeys { + sj := stampedByKey[key] + action := PlanDelete + if !opts.Prune { + action = PlanOrphan + plan.Warnings = append(plan.Warnings, g.F("orphan managed job %s (use --prune to delete)", key)) + } + plan.Items = append(plan.Items, PlanItem{ + Action: action, + SourceKey: key, + Spec: JobSpec{ + File: sj.File, + Schedules: sj.Schedules, + Streams: sj.Streams, + Mode: sj.Mode, + Variables: sj.Variables, + Retries: sj.Retries, + Timezone: sj.Timezone, + }, + }) + } + + return plan +} + +func (s JobSpec) scheduleWarning(key string) string { + if len(s.Schedules) > 1 { + return g.F("job %s has %d schedules; %s", key, len(s.Schedules), multiScheduleWarn) + } + return "" +} + +func jobSpecEqual(spec JobSpec, sj StampedJob) bool { + if spec.File != sj.File { + return false + } + if spec.Mode != sj.Mode || spec.Timezone != sj.Timezone { + return false + } + if !strSliceEqual(spec.Schedules, sj.Schedules) { + return false + } + if !strSliceEqual(spec.Streams, sj.Streams) { + return false + } + if !strMapEqual(spec.Variables, sj.Variables) { + return false + } + return intPtrEqual(spec.Retries, sj.Retries) +} + +func strSliceEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if a[i] != b[i] { + return false + } + } + return true +} + +func strMapEqual(a, b map[string]string) bool { + if len(a) == 0 && len(b) == 0 { + return true + } + if len(a) != len(b) { + return false + } + for k, v := range a { + if b[k] != v { + return false + } + } + return true +} + +func intPtrEqual(a, b *int) bool { + if a == nil && b == nil { + return true + } + if a == nil || b == nil { + return false + } + return *a == *b +} + +// FormatPlan prints a job plan as a table plus warnings. +func FormatPlan(p JobPlan) string { + rows := [][]any{} + for _, it := range p.Items { + if it.Action == PlanKeep { + continue + } + key := it.SourceKey + if it.Action == PlanRename && it.OldKey != "" { + key = it.OldKey + " -> " + it.SourceKey + } + rows = append(rows, []any{string(it.Action), key, it.Spec.File, strings.Join(it.Spec.Schedules, ", ")}) + } + + var b strings.Builder + if len(rows) == 0 { + b.WriteString("No job changes.") + } else { + b.WriteString(g.PrettyTable([]string{"Action", "Job", "File", "Schedules"}, rows)) + } + for _, w := range p.Warnings { + fmt.Fprintf(&b, "\nwarning: %s", w) + } + return b.String() +} + +// HasPushActions reports whether the plan creates, updates, renames, or (when prune) deletes. +func (p JobPlan) HasPushActions() bool { + for _, it := range p.Items { + switch it.Action { + case PlanCreate, PlanUpdate, PlanRename, PlanDelete: + return true + } + } + return false +} + +// ResolveJob finds the project root from startDir and returns the job spec +// for key. The error lists the available keys when the key misses. +func ResolveJob(startDir, key string) (root string, spec JobSpec, err error) { + key = strings.TrimSpace(key) + if key == "" { + return "", spec, g.Error("job key is empty") + } + + root, err = FindRoot(startDir) + if err != nil { + return "", spec, g.Error("no sling project found; run `sling init` or pass a file path") + } + + m, err := Load(root) + if err != nil { + return root, spec, g.Error(err, "could not load the project manifest") + } + + spec, ok := m.Jobs[key] + if !ok { + return root, spec, g.Error("job %s is not in the manifest%s", key, availableKeysSuffix(m.Jobs)) + } + return root, spec, nil +} + +func availableKeysSuffix(jobs map[string]JobSpec) string { + if len(jobs) == 0 { + return ". The manifest has no jobs" + } + keys := make([]string, 0, len(jobs)) + for k := range jobs { + keys = append(keys, k) + } + sort.Strings(keys) + return ". Available jobs: " + strings.Join(keys, ", ") +} diff --git a/core/sling/project/jobs_test.go b/core/sling/project/jobs_test.go new file mode 100644 index 000000000..9a790e2b9 --- /dev/null +++ b/core/sling/project/jobs_test.go @@ -0,0 +1,287 @@ +package project_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/slingdata-io/sling-cli/core/sling/project" +) + +func TestParseJobsUnknownFields(t *testing.T) { + body := []byte(` +name: demo +jobs: + orders_hourly: + file: replications/postgres_orders.yaml + schedules: ["0 * * * *"] + streams: [public.orders] + mode: incremental + retries: 2 + timezone: UTC + extra_ignored: true +`) + m, err := project.Parse(body) + if err != nil { + t.Fatal(err) + } + spec, ok := m.Jobs["orders_hourly"] + if !ok { + t.Fatal("missing orders_hourly") + } + if spec.File != "replications/postgres_orders.yaml" { + t.Fatalf("file = %q", spec.File) + } + if len(spec.Schedules) != 1 || spec.Schedules[0] != "0 * * * *" { + t.Fatalf("schedules = %#v", spec.Schedules) + } + if spec.Mode != "incremental" { + t.Fatalf("mode = %q", spec.Mode) + } + if spec.Retries == nil || *spec.Retries != 2 { + t.Fatalf("retries = %#v", spec.Retries) + } +} + +func TestLoadJobsFromFile(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, project.ManifestFileName) + body := []byte("name: fromfile\njobs:\n daily:\n file: pipelines/daily.yaml\n schedules: [\"0 6 * * *\"]\n") + if err := os.WriteFile(path, body, 0644); err != nil { + t.Fatal(err) + } + m, err := project.Load(dir) + if err != nil { + t.Fatal(err) + } + if m.Jobs["daily"].File != "pipelines/daily.yaml" { + t.Fatalf("jobs = %#v", m.Jobs) + } +} + +func TestPlanJobsCreateUpdateOrphanRename(t *testing.T) { + m := project.Manifest{ + Jobs: map[string]project.JobSpec{ + "orders_hourly": { + File: "replications/postgres_orders.yaml", + Schedules: []string{"0 * * * *"}, + Streams: []string{"public.orders"}, + }, + "daily": { + File: "pipelines/daily.yaml", + Schedules: []string{"0 6 * * *", "0 18 * * *"}, + }, + }, + } + + empty := project.PlanJobs(m, nil, project.PlanOptions{}) + if got := actions(empty); strings.Join(got, ",") != "create,create" { + t.Fatalf("empty stamped actions = %v plan=\n%s", got, project.FormatPlan(empty)) + } + out := project.FormatPlan(empty) + if !strings.Contains(out, "orders_hourly") || !strings.Contains(out, "create") { + t.Fatalf("plan output missing create:\n%s", out) + } + if !strings.Contains(out, "only the first cron fires") { + t.Fatalf("expected multi-schedule warning:\n%s", out) + } + + stamped := []project.StampedJob{ + {SourceKey: "orders_hourly", File: "replications/old.yaml", Schedules: []string{"0 * * * *"}}, + {SourceKey: "legacy", File: "replications/legacy.yaml"}, + {SourceKey: "unrelated", File: "pipelines/old_daily.yaml", Schedules: []string{"0 6 * * *"}}, + } + updated := project.PlanJobs(m, stamped, project.PlanOptions{}) + got := actions(updated) + if !contains(got, "update") || !contains(got, "create") || !contains(got, "orphan") { + t.Fatalf("actions = %v\n%s", got, project.FormatPlan(updated)) + } + if contains(got, "delete") { + t.Fatalf("unpruned plan must not label orphans as delete: %v", got) + } + if !strings.Contains(project.FormatPlan(updated), "orphan managed job") { + t.Fatal("expected orphan warning without prune") + } + + pruned := project.PlanJobs(m, stamped, project.PlanOptions{Prune: true}) + if !contains(actions(pruned), "delete") { + t.Fatalf("prune plan missing delete: %v", actions(pruned)) + } + + renamed := project.PlanJobs( + project.Manifest{Jobs: map[string]project.JobSpec{ + "orders_v2": {File: "replications/postgres_orders.yaml", Schedules: []string{"0 * * * *"}}, + }}, + []project.StampedJob{{SourceKey: "orders_hourly", File: "replications/postgres_orders.yaml", Schedules: []string{"0 * * * *"}}}, + project.PlanOptions{Renames: map[string]string{"orders_hourly": "orders_v2"}}, + ) + if !contains(actions(renamed), "rename") { + t.Fatalf("rename plan = %v\n%s", actions(renamed), project.FormatPlan(renamed)) + } + if contains(actions(renamed), "create") || contains(actions(renamed), "delete") { + t.Fatalf("rename should not create/delete: %v", actions(renamed)) + } +} + +func TestPlanJobsUnmanagedEmptySourceKeyIgnored(t *testing.T) { + m := project.Manifest{Jobs: map[string]project.JobSpec{ + "a": {File: "replications/a.yaml"}, + }} + plan := project.PlanJobs(m, []project.StampedJob{ + {SourceKey: "", Name: "Default Job (a.yaml)", File: "replications/a.yaml"}, + }, project.PlanOptions{}) + if contains(actions(plan), "delete") { + t.Fatalf("unmanaged empty source_key treated as stamped: %v", actions(plan)) + } +} + +func actions(p project.JobPlan) []string { + out := make([]string, 0, len(p.Items)) + for _, it := range p.Items { + if it.Action == project.PlanKeep { + continue + } + out = append(out, string(it.Action)) + } + return out +} + +func contains(items []string, want string) bool { + for _, s := range items { + if s == want { + return true + } + } + return false +} + +func TestSetProjectIDKeepsJobsComment(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, project.ManifestFileName) + body := "name: demo\n# project_id: set by `sling project link`\n\n# jobs:\n# daily:\n# file: pipelines/daily.yaml\n# schedules: [\"0 6 * * *\"]\n" + if err := os.WriteFile(path, []byte(body), 0644); err != nil { + t.Fatal(err) + } + if err := project.SetProjectID(dir, "proj_123"); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + s := string(got) + if !strings.Contains(s, "proj_123") { + t.Fatalf("missing project_id:\n%s", s) + } + if !strings.Contains(s, "# jobs:") { + t.Fatalf("lost # jobs: comment:\n%s", s) + } +} + +func TestResolveLinkProjectID(t *testing.T) { + id, err := project.ResolveLinkProjectID("from-token", nil, nil) + if err != nil || id != "from-token" { + t.Fatalf("token id: %q %v", id, err) + } + + one := []project.LinkProject{{ID: "only", Name: "Only"}} + id, err = project.ResolveLinkProjectID("", one, nil) + if err != nil || id != "only" { + t.Fatalf("single listed: %q %v", id, err) + } + + many := []project.LinkProject{{ID: "a"}, {ID: "b"}} + _, err = project.ResolveLinkProjectID("", many, nil) + if err == nil { + t.Fatal("expected error for multiple without pick") + } + id, err = project.ResolveLinkProjectID("", many, func(listed []project.LinkProject) (string, error) { + return listed[1].ID, nil + }) + if err != nil || id != "b" { + t.Fatalf("pick: %q %v", id, err) + } +} + +func writeManifest(t *testing.T, dir, body string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, project.ManifestFileName), []byte(body), 0644); err != nil { + t.Fatal(err) + } +} + +func TestResolveJobHit(t *testing.T) { + dir := t.TempDir() + writeManifest(t, dir, ` +name: demo +jobs: + daily: + file: replications/r.yaml + mode: truncate + streams: [public.orders] +`) + root, spec, err := project.ResolveJob(dir, "daily") + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(root, filepath.Base(dir)) { + t.Fatalf("unexpected root %s", root) + } + if spec.File != "replications/r.yaml" || spec.Mode != "truncate" { + t.Fatalf("unexpected spec %+v", spec) + } + if len(spec.Streams) != 1 || spec.Streams[0] != "public.orders" { + t.Fatalf("unexpected streams %v", spec.Streams) + } +} + +func TestResolveJobFromSubfolder(t *testing.T) { + dir := t.TempDir() + writeManifest(t, dir, "jobs:\n daily:\n file: r.yaml\n") + sub := filepath.Join(dir, "replications", "nested") + if err := os.MkdirAll(sub, 0755); err != nil { + t.Fatal(err) + } + if _, _, err := project.ResolveJob(sub, "daily"); err != nil { + t.Fatal(err) + } +} + +func TestResolveJobMissListsKeys(t *testing.T) { + dir := t.TempDir() + writeManifest(t, dir, "jobs:\n daily:\n file: a.yaml\n hourly:\n file: b.yaml\n") + _, _, err := project.ResolveJob(dir, "nope") + if err == nil { + t.Fatal("expected an error") + } + msg := err.Error() + if !strings.Contains(msg, "daily") || !strings.Contains(msg, "hourly") { + t.Fatalf("error does not list the keys: %s", msg) + } +} + +func TestResolveJobNoJobs(t *testing.T) { + dir := t.TempDir() + writeManifest(t, dir, "name: demo\n") + _, _, err := project.ResolveJob(dir, "daily") + if err == nil || !strings.Contains(err.Error(), "no jobs") { + t.Fatalf("got %v", err) + } +} + +func TestResolveJobNoProject(t *testing.T) { + dir := t.TempDir() + _, _, err := project.ResolveJob(dir, "daily") + if err == nil || !strings.Contains(err.Error(), "no sling project found") { + t.Fatalf("got %v", err) + } +} + +func TestResolveJobEmptyKey(t *testing.T) { + dir := t.TempDir() + writeManifest(t, dir, "jobs:\n daily:\n file: a.yaml\n") + if _, _, err := project.ResolveJob(dir, " "); err == nil { + t.Fatal("expected an error for an empty key") + } +} diff --git a/core/sling/project/manifest.go b/core/sling/project/manifest.go new file mode 100644 index 000000000..b38167937 --- /dev/null +++ b/core/sling/project/manifest.go @@ -0,0 +1,260 @@ +package project + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + + "github.com/flarco/g" + "gopkg.in/yaml.v3" +) + +const ( + // ManifestFileName is the canonical project file. + ManifestFileName = "sling_project.yml" + // LegacyFileName is the older JSON project file. + LegacyFileName = ".sling.json" +) + +// Manifest is a local Sling project file. +// Unknown YAML/JSON keys are ignored. +type Manifest struct { + Name string `yaml:"name,omitempty" json:"name,omitempty"` + ProjectID string `yaml:"project_id,omitempty" json:"id,omitempty"` + Paths []string `yaml:"paths,omitempty" json:"paths,omitempty"` + Jobs map[string]JobSpec `yaml:"jobs,omitempty" json:"jobs,omitempty"` + Root string `yaml:"-" json:"-"` + Path string `yaml:"-" json:"-"` +} + +// Parse unmarshals a sling_project.yml body. Extra keys are ignored. +func Parse(body []byte) (Manifest, error) { + var m Manifest + if err := yaml.Unmarshal(body, &m); err != nil { + return m, g.Error(err, "could not parse %s", ManifestFileName) + } + return m, nil +} + +// HasManifest reports whether dir has sling_project.yml or .sling.json. +func HasManifest(dir string) bool { + if dir == "" { + return false + } + if g.PathExists(filepath.Join(dir, ManifestFileName)) { + return true + } + return g.PathExists(filepath.Join(dir, LegacyFileName)) +} + +// FindRoot walks up from start and returns the nearest folder with a manifest. +func FindRoot(start string) (string, error) { + if start == "" { + wd, err := os.Getwd() + if err != nil { + return "", g.Error(err, "could not get working directory") + } + start = wd + } + dir, err := filepath.Abs(start) + if err != nil { + return "", g.Error(err, "could not resolve path") + } + for { + if HasManifest(dir) { + return dir, nil + } + parent := filepath.Dir(dir) + if parent == dir { + return "", g.Error("no sling project found") + } + dir = parent + } +} + +// Load reads sling_project.yml first, then .sling.json. +func Load(folderPath string) (Manifest, error) { + var m Manifest + if folderPath == "" { + return m, g.Error("folder path is empty") + } + abs, err := filepath.Abs(folderPath) + if err != nil { + return m, g.Error(err, "could not resolve folder") + } + + ymlPath := filepath.Join(abs, ManifestFileName) + jsonPath := filepath.Join(abs, LegacyFileName) + + switch { + case g.PathExists(ymlPath): + body, err := os.ReadFile(ymlPath) + if err != nil { + return m, g.Error(err, "could not read file %s", ymlPath) + } + if err = yaml.Unmarshal(body, &m); err != nil { + return m, g.Error(err, "could not parse file %s", ymlPath) + } + m.Path = ymlPath + case g.PathExists(jsonPath): + body, err := os.ReadFile(jsonPath) + if err != nil { + return m, g.Error(err, "could not read file %s", jsonPath) + } + if err = json.Unmarshal(body, &m); err != nil { + return m, g.Error(err, "could not parse file %s", jsonPath) + } + m.Path = jsonPath + default: + return m, g.Error("did not find %s or %s", ManifestFileName, LegacyFileName) + } + + m.Root = abs + return m, nil +} + +// Linked reports whether the manifest points at a platform project. +func (m Manifest) Linked() bool { + return strings.TrimSpace(m.ProjectID) != "" +} + +// LinkProject is one platform project a token can attach to. +type LinkProject struct { + ID string + Name string +} + +// ResolveLinkProjectID picks the platform project id to attach a folder to. +// A token-scoped id wins. Otherwise one listed project is used, or pick() when +// more than one exists. +func ResolveLinkProjectID(tokenID string, listed []LinkProject, pick func([]LinkProject) (string, error)) (string, error) { + if id := strings.TrimSpace(tokenID); id != "" { + return id, nil + } + if len(listed) == 0 { + return "", g.Error("no platform projects found for this token") + } + if len(listed) == 1 { + return listed[0].ID, nil + } + if pick == nil { + return "", g.Error("multiple platform projects; use a project-scoped token or pick one") + } + return pick(listed) +} + +// SetProjectID writes project_id into sling_project.yml and keeps comments. +func SetProjectID(folderPath, projectID string) error { + if strings.TrimSpace(projectID) == "" { + return g.Error("project id is empty") + } + abs, err := filepath.Abs(folderPath) + if err != nil { + return g.Error(err, "could not resolve folder") + } + ymlPath := filepath.Join(abs, ManifestFileName) + + var doc yaml.Node + if g.PathExists(ymlPath) { + body, err := os.ReadFile(ymlPath) + if err != nil { + return g.Error(err, "could not read file %s", ymlPath) + } + if err = yaml.Unmarshal(body, &doc); err != nil { + return g.Error(err, "could not parse file %s", ymlPath) + } + } + if err = setYAMLMapString(&doc, "project_id", projectID); err != nil { + return err + } + if mapping := yamlMapping(&doc); mapping != nil && yamlMapValue(mapping, "name") == "" { + name := filepath.Base(abs) + if m, loadErr := Load(abs); loadErr == nil && m.Name != "" { + name = m.Name + } + _ = setYAMLMapString(&doc, "name", name) + } + + var buf strings.Builder + enc := yaml.NewEncoder(&buf) + enc.SetIndent(2) + if err = enc.Encode(&doc); err != nil { + _ = enc.Close() + return g.Error(err, "could not marshal %s", ManifestFileName) + } + if err = enc.Close(); err != nil { + return g.Error(err, "could not finalize %s", ManifestFileName) + } + if err = os.WriteFile(ymlPath, []byte(buf.String()), 0644); err != nil { + return g.Error(err, "could not write %s", ymlPath) + } + + jsonPath := filepath.Join(abs, LegacyFileName) + if g.PathExists(jsonPath) { + m, loadErr := Load(abs) + if loadErr == nil { + m.ProjectID = projectID + body := []byte(g.Pretty(map[string]any{ + "id": m.ProjectID, + "name": m.Name, + "paths": m.Paths, + })) + _ = os.WriteFile(jsonPath, body, 0644) + } + } + + return nil +} + +func yamlMapping(n *yaml.Node) *yaml.Node { + if n == nil { + return nil + } + if n.Kind == yaml.DocumentNode && len(n.Content) > 0 { + n = n.Content[0] + } + if n.Kind == yaml.MappingNode { + return n + } + return nil +} + +func yamlMapValue(m *yaml.Node, key string) string { + if m == nil { + return "" + } + for i := 0; i < len(m.Content)-1; i += 2 { + if m.Content[i].Value == key { + return m.Content[i+1].Value + } + } + return "" +} + +func setYAMLMapString(doc *yaml.Node, key, value string) error { + if doc == nil { + return g.Error("yaml document is nil") + } + if doc.Kind == 0 { + doc.Kind = yaml.DocumentNode + doc.Content = []*yaml.Node{{Kind: yaml.MappingNode}} + } + m := yamlMapping(doc) + if m == nil { + return g.Error("sling_project.yml root is not a mapping") + } + for i := 0; i < len(m.Content)-1; i += 2 { + if m.Content[i].Value == key { + m.Content[i+1].Kind = yaml.ScalarNode + m.Content[i+1].Tag = "!!str" + m.Content[i+1].Value = value + return nil + } + } + m.Content = append(m.Content, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: key}, + &yaml.Node{Kind: yaml.ScalarNode, Tag: "!!str", Value: value}, + ) + return nil +} diff --git a/core/sling/project/scaffold.go b/core/sling/project/scaffold.go new file mode 100644 index 000000000..de1192b17 --- /dev/null +++ b/core/sling/project/scaffold.go @@ -0,0 +1,207 @@ +package project + +import ( + "bufio" + "embed" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + + "github.com/flarco/g" +) + +//go:embed all:scaffold +var scaffoldFS embed.FS + +const defaultEntity = "example" + +// Options controls project scaffolding. +type Options struct { + Dir string + Name string + Source string + Target string + Entity string + Yes bool + Force bool +} + +// Result lists files written by Init. +type Result struct { + Dir string + Name string + Files []string +} + +// Init writes a canonical Sling project into dir. +func Init(opts Options) (*Result, error) { + dir := opts.Dir + if dir == "" { + wd, err := os.Getwd() + if err != nil { + return nil, g.Error(err, "could not get working directory") + } + dir = wd + } + abs, err := filepath.Abs(dir) + if err != nil { + return nil, g.Error(err, "could not resolve project directory") + } + + if err := os.MkdirAll(abs, 0755); err != nil { + return nil, g.Error(err, "could not create project directory") + } + + if !opts.Force { + if HasManifest(abs) { + return nil, g.Error("project already exists in %s; re-run with --force to overwrite", abs) + } + if root, findErr := FindRoot(abs); findErr == nil && root != "" && root != abs { + return nil, g.Error("already inside a project at %s; re-run with --force", root) + } + } + + source := strings.TrimSpace(opts.Source) + target := strings.TrimSpace(opts.Target) + if source == "" || target == "" { + return nil, g.Error("source and target are required") + } + + name := strings.TrimSpace(opts.Name) + if name == "" { + name = filepath.Base(abs) + } + entity := strings.TrimSpace(opts.Entity) + if entity == "" { + entity = defaultEntity + } + sourceSlug := slug(source) + + replacer := strings.NewReplacer( + "{{SOURCE_SLUG}}", sourceSlug, + "{{SOURCE}}", source, + "{{TARGET}}", target, + "{{NAME}}", name, + "{{ENTITY}}", entity, + ) + + planned, err := plannedFiles(abs, sourceSlug, entity, replacer) + if err != nil { + return nil, err + } + + if err := confirmOverwrite(planned, opts.Yes); err != nil { + return nil, err + } + + written := make([]string, 0, len(planned)) + for _, f := range planned { + if err := os.MkdirAll(filepath.Dir(f.path), 0755); err != nil { + return nil, g.Error(err, "could not create directory %s", filepath.Dir(f.path)) + } + if err := os.WriteFile(f.path, f.body, 0644); err != nil { + return nil, g.Error(err, "could not write %s", f.path) + } + rel, _ := filepath.Rel(abs, f.path) + written = append(written, filepath.ToSlash(rel)) + } + + return &Result{Dir: abs, Name: name, Files: written}, nil +} + +type plannedFile struct { + path string + body []byte +} + +func plannedFiles(abs, sourceSlug, entity string, replacer *strings.Replacer) ([]plannedFile, error) { + var out []plannedFile + err := fs.WalkDir(scaffoldFS, "scaffold", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + return nil + } + body, err := fs.ReadFile(scaffoldFS, path) + if err != nil { + return g.Error(err, "could not read scaffold file %s", path) + } + rel, err := filepath.Rel("scaffold", path) + if err != nil { + return err + } + rel = filepath.ToSlash(rel) + rel = strings.TrimSuffix(rel, ".tmpl") + rel = strings.ReplaceAll(rel, "SOURCE_SLUG", sourceSlug) + rel = strings.ReplaceAll(rel, "ENTITY", entity) + if rel == "gitignore" { + rel = ".gitignore" + } + out = append(out, plannedFile{ + path: filepath.Join(abs, filepath.FromSlash(rel)), + body: []byte(replacer.Replace(string(body))), + }) + return nil + }) + if err != nil { + return nil, err + } + return out, nil +} + +func confirmOverwrite(files []plannedFile, yes bool) error { + existing := []string{} + for _, f := range files { + if g.PathExists(f.path) { + existing = append(existing, f.path) + } + } + if len(existing) == 0 { + return nil + } + if yes { + return nil + } + if !isInteractive() { + return g.Error("project files already exist; re-run with --yes to overwrite in non-interactive mode") + } + + fmt.Print("Project files already exist. Overwrite? [y/N]: ") + reader := bufio.NewReader(os.Stdin) + answer, _ := reader.ReadString('\n') + answer = strings.TrimSpace(strings.ToLower(answer)) + if answer != "y" && answer != "yes" { + return g.Error("aborted") + } + return nil +} + +func isInteractive() bool { + fi, err := os.Stdin.Stat() + if err != nil { + return false + } + return fi.Mode()&os.ModeCharDevice != 0 +} + +func slug(name string) string { + name = strings.ToLower(strings.TrimSpace(name)) + var b strings.Builder + prevUnderscore := false + for _, r := range name { + ok := (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') + if ok { + b.WriteRune(r) + prevUnderscore = false + continue + } + if !prevUnderscore && b.Len() > 0 { + b.WriteByte('_') + prevUnderscore = true + } + } + return strings.Trim(b.String(), "_") +} diff --git a/core/sling/project/scaffold/gitignore b/core/sling/project/scaffold/gitignore new file mode 100644 index 000000000..2cdcdb167 --- /dev/null +++ b/core/sling/project/scaffold/gitignore @@ -0,0 +1,5 @@ +.sling/ +.env +*.log +.DS_Store +.sling.json diff --git a/core/sling/project/scaffold/models/marts/fct_ENTITY.sql.tmpl b/core/sling/project/scaffold/models/marts/fct_ENTITY.sql.tmpl new file mode 100644 index 000000000..9180758b3 --- /dev/null +++ b/core/sling/project/scaffold/models/marts/fct_ENTITY.sql.tmpl @@ -0,0 +1,9 @@ +/** +mode: full-refresh +tests: + - not_null: [id] +**/ +SELECT + id, + name +FROM {{ ref('stg_{{SOURCE_SLUG}}__{{ENTITY}}') }} diff --git a/core/sling/project/scaffold/models/sling_build.yml.tmpl b/core/sling/project/scaffold/models/sling_build.yml.tmpl new file mode 100644 index 000000000..71e330fb6 --- /dev/null +++ b/core/sling/project/scaffold/models/sling_build.yml.tmpl @@ -0,0 +1,7 @@ +target: {{TARGET}} + +defaults: + mode: full-refresh + +dev: + schema: dev diff --git a/core/sling/project/scaffold/models/staging/stg_SOURCE_SLUG__ENTITY.sql.tmpl b/core/sling/project/scaffold/models/staging/stg_SOURCE_SLUG__ENTITY.sql.tmpl new file mode 100644 index 000000000..51900b79b --- /dev/null +++ b/core/sling/project/scaffold/models/staging/stg_SOURCE_SLUG__ENTITY.sql.tmpl @@ -0,0 +1,5 @@ +-- {mode: view} +SELECT + id, + name +FROM {{ src('raw_{{SOURCE_SLUG}}.{{ENTITY}}') }} diff --git a/core/sling/project/scaffold/pipelines/daily.yaml.tmpl b/core/sling/project/scaffold/pipelines/daily.yaml.tmpl new file mode 100644 index 000000000..9dfde9fa8 --- /dev/null +++ b/core/sling/project/scaffold/pipelines/daily.yaml.tmpl @@ -0,0 +1,20 @@ +steps: + - type: log + message: "Start daily run" + + - type: replication + path: replications/{{SOURCE_SLUG}}.yaml + id: load_raw + + - type: build + build: models + prod: true + id: transform + + - type: check + check: state.transform.failed == 0 + failure_message: "Build reported failures" + on_failure: abort + + - type: log + message: "Done. build ok={state.transform.ok}" diff --git a/core/sling/project/scaffold/replications/SOURCE_SLUG.yaml.tmpl b/core/sling/project/scaffold/replications/SOURCE_SLUG.yaml.tmpl new file mode 100644 index 000000000..b53fd157a --- /dev/null +++ b/core/sling/project/scaffold/replications/SOURCE_SLUG.yaml.tmpl @@ -0,0 +1,10 @@ +source: {{SOURCE}} +target: {{TARGET}} + +defaults: + mode: full-refresh + object: raw_{{SOURCE_SLUG}}.{stream_table} + +streams: + main.{{ENTITY}}: + object: raw_{{SOURCE_SLUG}}.{{ENTITY}} diff --git a/core/sling/project/scaffold/sling_project.yml.tmpl b/core/sling/project/scaffold/sling_project.yml.tmpl new file mode 100644 index 000000000..ad8524325 --- /dev/null +++ b/core/sling/project/scaffold/sling_project.yml.tmpl @@ -0,0 +1,9 @@ +name: {{NAME}} +# project_id: set by `sling project deploy` when this folder is linked + +# jobs: +# daily: +# file: pipelines/daily.yaml +# schedules: ["0 6 * * *"] +# # run locally on demand: sling run -j daily +# # schedules fire on the platform after 'sling project deploy' diff --git a/core/sling/project/scaffold_test.go b/core/sling/project/scaffold_test.go new file mode 100644 index 000000000..c70412078 --- /dev/null +++ b/core/sling/project/scaffold_test.go @@ -0,0 +1,121 @@ +package project_test + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/slingdata-io/sling-cli/core/sling/build" + "github.com/slingdata-io/sling-cli/core/sling/validate" + "github.com/slingdata-io/sling-cli/core/sling/project" +) + +func TestScaffoldParseAndLoad(t *testing.T) { + dir := t.TempDir() + res, err := project.Init(project.Options{ + Dir: dir, + Name: "wave45", + Source: "SQLITE", + Target: "POSTGRES", + Yes: true, + }) + if err != nil { + t.Fatal(err) + } + if res == nil || len(res.Files) == 0 { + t.Fatal("expected scaffolded files") + } + + required := []string{ + "sling_project.yml", + "models/sling_build.yml", + "pipelines/daily.yaml", + "replications/sqlite.yaml", + "models/staging/stg_sqlite__example.sql", + "models/marts/fct_example.sql", + ".gitignore", + } + for _, rel := range required { + path := filepath.Join(dir, filepath.FromSlash(rel)) + if _, err := os.Stat(path); err != nil { + t.Fatalf("missing %s: %v", rel, err) + } + } + + m, err := project.Load(dir) + if err != nil { + t.Fatal(err) + } + if m.Name != "wave45" { + t.Fatalf("manifest name = %q", m.Name) + } + + yamlRoots := []string{ + filepath.Join(dir, "replications"), + filepath.Join(dir, "pipelines"), + filepath.Join(dir, "models"), + } + results := validate.ParsePaths(yamlRoots, validate.Options{}) + if len(results) == 0 { + t.Fatal("parse returned no yaml files") + } + var kinds []validate.Kind + for _, r := range results { + if !r.OK { + t.Errorf("parse failed for %s (%s): %s", r.Path, r.Kind, r.Error) + continue + } + kinds = append(kinds, r.Kind) + } + if !hasKind(kinds, validate.KindReplication) { + t.Error("expected a replication yaml") + } + if !hasKind(kinds, validate.KindPipeline) { + t.Error("expected a pipeline yaml") + } + if !hasKind(kinds, validate.KindBuild) { + t.Error("expected a build yaml") + } + + proj, err := build.LoadProject(filepath.Join(dir, "models")) + if err != nil { + t.Fatal(err) + } + if _, ok := proj.Models["stg_sqlite__example"]; !ok { + t.Fatalf("missing staging model, have %v", modelNames(proj)) + } + if _, ok := proj.Models["fct_example"]; !ok { + t.Fatalf("missing mart model, have %v", modelNames(proj)) + } + + _, err = project.Init(project.Options{ + Dir: dir, + Source: "SQLITE", + Target: "POSTGRES", + Yes: true, + }) + if err == nil { + t.Fatal("expected refuse inside existing project without --force") + } + if !strings.Contains(err.Error(), "--force") { + t.Fatalf("unexpected error: %s", err) + } +} + +func hasKind(kinds []validate.Kind, want validate.Kind) bool { + for _, k := range kinds { + if k == want { + return true + } + } + return false +} + +func modelNames(proj *build.BuildProject) []string { + names := make([]string, 0, len(proj.Models)) + for name := range proj.Models { + names = append(names, name) + } + return names +} diff --git a/tests/suite.cli.project.yaml b/tests/suite.cli.project.yaml new file mode 100644 index 000000000..3f16b5471 --- /dev/null +++ b/tests/suite.cli.project.yaml @@ -0,0 +1,202 @@ +- id: 530 + name: 'sling init succeeds with no token' + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + unset SLING_CLI_TOKEN + sling init --target POSTGRES --source SQLITE --yes + env: + SLING_PROJECT_TOKEN: '' + output_contains: + - sling_project.yml + - models/sling_build.yml + output_does_not_contain: + - invalid project token + - did not provide the SLING_PROJECT_TOKEN + +- id: 531 + name: 'sling init succeeds with leftover token and no network' + run: | + DIR=$(mktemp -d) + cd "$DIR" + sling init --target POSTGRES --source SQLITE --yes + env: + SLING_PROJECT_TOKEN: dummy-token-must-not-call-network + SLING_CLI_TOKEN: '' + HTTPS_PROXY: 'http://127.0.0.1:1' + HTTP_PROXY: 'http://127.0.0.1:1' + SLING_PLATFORM_HOST: 'https://127.0.0.1:1' + output_contains: + - sling_project.yml + - sling project deploy --check + output_does_not_contain: + - invalid project token + - error validating project token + +- id: 532 + name: 'sling init non-interactive writes canonical layout' + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + sling init --target POSTGRES --source SQLITE --yes + test -f sling_project.yml + test -f models/sling_build.yml + test -f pipelines/daily.yaml + test -f replications/sqlite.yaml + test -f models/staging/stg_sqlite__example.sql + test -f models/marts/fct_example.sql + echo "SUCCESS: canonical layout" + env: + SLING_PROJECT_TOKEN: '' + output_contains: + - 'SUCCESS: canonical layout' + - sling_project.yml + +- id: 534 + name: 'sling validate succeeds on scaffolded pipeline' + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + sling init --target POSTGRES --source SQLITE --yes + sling validate pipelines/daily.yaml --json + env: + SLING_PROJECT_TOKEN: '' + output_contains: + - '"kind": "pipeline"' + - '"ok": true' + - daily.yaml + +- id: 535 + name: 'sling project status without a token fails fast' + err: true + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + sling init --target POSTGRES --source SQLITE --yes + sling project status + env: + SLING_PROJECT_TOKEN: '' + output_contains: + - SLING_PROJECT_TOKEN + output_does_not_contain: + - Last Run + - Project Root + +- id: 560 + name: 'sling project deploy --check prints local job plan' + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + unset SLING_CLI_TOKEN + sling init --target POSTGRES --source SQLITE --yes + echo 'jobs:' >> sling_project.yml + echo ' orders_hourly:' >> sling_project.yml + echo ' file: replications/sqlite.yaml' >> sling_project.yml + echo ' schedules: ["0 * * * *"]' >> sling_project.yml + echo ' streams: [public.orders]' >> sling_project.yml + sling project deploy --check + env: + SLING_PROJECT_TOKEN: '' + SLING_CLI_TOKEN: '' + output_contains: + - create + - orders_hourly + - replications/sqlite.yaml + output_does_not_contain: + - not implemented yet + +- id: 561 + name: 'sling run -j applies the job streams and mode' + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + sling init --target POSTGRES --source SQLITE --yes + printf 'id,name\n1,alpha\n' > alpha.csv + printf 'id,name\n2,beta\n' > beta.csv + printf 'source: LOCAL\ntarget: SQLITE\ndefaults:\n mode: full-refresh\nstreams:\n file://alpha.csv:\n object: main.job_alpha\n file://beta.csv:\n object: main.job_beta\n' > replications/local.yaml + printf 'name: jobdemo\njobs:\n onlyalpha:\n file: replications/local.yaml\n streams: ["file://alpha.csv"]\n mode: truncate\n' > sling_project.yml + sling run -j onlyalpha -d + env: + SLING_PROJECT_TOKEN: '' + output_contains: + - 'job_alpha' + - 'full-refresh => truncate' + - 'file://alpha.csv' + output_does_not_contain: + - 'job_beta' + - 'file://beta.csv' + +- id: 562 + name: 'sling run resolves a bare manifest key' + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + sling init --target POSTGRES --source SQLITE --yes + printf 'id,name\n1,alpha\n' > alpha.csv + printf 'source: LOCAL\ntarget: SQLITE\ndefaults:\n mode: full-refresh\nstreams:\n file://alpha.csv:\n object: main.bare_key\n' > replications/local.yaml + printf 'name: jobdemo\njobs:\n loadit:\n file: replications/local.yaml\n mode: truncate\n' > sling_project.yml + sling run loadit -d + env: + SLING_PROJECT_TOKEN: '' + output_contains: + - 'bare_key' + - 'full-refresh => truncate' + +- id: 563 + name: 'a file path shadowing a job key wins over the key' + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + sling init --target POSTGRES --source SQLITE --yes + printf 'id,name\n1,alpha\n' > alpha.csv + printf 'source: LOCAL\ntarget: SQLITE\nstreams:\n file://alpha.csv:\n object: main.from_key\n' > replications/local.yaml + printf 'source: LOCAL\ntarget: SQLITE\nstreams:\n file://alpha.csv:\n object: main.from_path\n' > loadit + printf 'name: jobdemo\njobs:\n loadit:\n file: replications/local.yaml\n mode: truncate\n' > sling_project.yml + sling run loadit -d + env: + SLING_PROJECT_TOKEN: '' + output_contains: + - 'from_path' + output_does_not_contain: + - 'from_key' + - 'full-refresh => truncate' + +- id: 564 + name: 'sling run -j with a missing key lists the available keys' + err: true + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + sling init --target POSTGRES --source SQLITE --yes + printf 'jobs:\n daily:\n file: pipelines/daily.yaml\n hourly:\n file: pipelines/daily.yaml\n' >> sling_project.yml + sling run -j nope + env: + SLING_PROJECT_TOKEN: '' + output_contains: + - 'is not in the manifest' + - 'daily' + - 'hourly' + +- id: 565 + name: 'sling run -j applies job variables to a pipeline' + run: | + DIR=$(mktemp -d) + cd "$DIR" + unset SLING_PROJECT_TOKEN + sling init --target POSTGRES --source SQLITE --yes + printf 'steps:\n - type: log\n message: "who={env.WHO} where={env.WHERE}"\n' > pipelines/daily.yaml + printf 'name: jobdemo\njobs:\n greet:\n file: pipelines/daily.yaml\n variables:\n WHO: sling\n WHERE: here\n' > sling_project.yml + sling run -j greet -d + env: + SLING_PROJECT_TOKEN: '' + output_contains: + - 'who=sling where=here' From 9c982e0e814875073b7f98fd33fbba729080df48 Mon Sep 17 00:00:00 2001 From: Fritz Larco Date: Thu, 27 Aug 2026 10:57:16 -0300 Subject: [PATCH 09/30] Add sling assist --- cmd/sling/resource/llm_API_SPEC.md | 10 +- cmd/sling/resource/llm_CONNECTION.md | 22 +- cmd/sling/resource/llm_CONNECTION_DATABASE.md | 3 +- cmd/sling/resource/llm_CONNECTION_FILE.md | 3 +- cmd/sling/resource/llm_PIPELINE.md | 46 +- cmd/sling/resource/llm_PLATFORM.md | 8 +- cmd/sling/resource/llm_REPLICATION.md | 112 +- cmd/sling/resource/mcp.yaml | 115 +- cmd/sling/sling_assist.go | 536 ++ cmd/sling/sling_assist_test.go | 243 + cmd/sling/sling_cli.go | 119 +- cmd/sling/sling_conns.go | 81 +- cmd/sling/sling_init.go | 155 + cmd/sling/sling_prompt.go | 122 - cmd/sling/sling_run.go | 94 +- core/sling/assist/assist.go | 279 + core/sling/assist/assist_test.go | 776 +++ core/sling/assist/browser.go | 319 ++ core/sling/assist/browser_test.go | 182 + core/sling/assist/clients.go | 1404 +++++ core/sling/assist/clients_test.go | 1052 ++++ core/sling/assist/doctor.go | 514 ++ core/sling/assist/history.go | 430 ++ core/sling/assist/install.go | 746 +++ core/sling/assist/investigate.go | 1169 ++++ core/sling/assist/investigate_test.go | 1276 +++++ core/sling/assist/jsonedit.go | 255 + core/sling/assist/opencode.go | 426 ++ core/sling/assist/opencode_network_test.go | 35 + core/sling/assist/opencode_test.go | 432 ++ core/sling/assist/prompt.go | 945 ++++ core/sling/assist/prompt_test.go | 1036 ++++ core/sling/assist/prompts.yaml | 44 + core/sling/assist/report.go | 574 ++ core/sling/assist/report_test.go | 358 ++ core/sling/assist/session.go | 820 +++ .../sling/assist/skills/agent-browser/CORE.md | 516 ++ .../assist/skills/agent-browser/SKILL.md | 70 + .../skills/sling-api-specs/AUTHENTICATION.md | 242 + .../assist/skills/sling-api-specs/DYNAMIC.md | 236 + .../skills/sling-api-specs/ENDPOINTS.md | 189 + .../skills/sling-api-specs/FUNCTIONS.md | 211 + .../skills/sling-api-specs/INCREMENTAL.md | 235 + .../skills/sling-api-specs/PAGINATION.md | 216 + .../skills/sling-api-specs/PROCESSORS.md | 239 + .../assist/skills/sling-api-specs/QUEUES.md | 279 + .../assist/skills/sling-api-specs/REQUEST.md | 159 + .../assist/skills/sling-api-specs/RESPONSE.md | 187 + .../assist/skills/sling-api-specs/RULES.md | 279 + .../assist/skills/sling-api-specs/SKILL.md | 173 + .../skills/sling-api-specs/VARIABLES.md | 196 + core/sling/assist/skills/sling-build/SKILL.md | 156 + .../assist/skills/sling-build/STRUCTURE.md | 61 + .../assist/skills/sling-connections/SKILL.md | 227 + .../assist/skills/sling-pipelines/SKILL.md | 317 ++ .../assist/skills/sling-pipelines/STEPS.md | 368 ++ .../assist/skills/sling-platform/SKILL.md | 156 + .../assist/skills/sling-project/SKILL.md | 140 + .../assist/skills/sling-project/STRUCTURE.md | 76 + .../sling/assist/skills/sling-python/SKILL.md | 188 + .../assist/skills/sling-replications/CDC.md | 97 + .../assist/skills/sling-replications/SKILL.md | 348 ++ .../skills/sling-replications/TRANSFORMS.md | 184 + core/sling/assist/skills/sling/SKILL.md | 108 + .../assist/skills/sling/TROUBLESHOOTING.md | 296 + core/sling/assist/testdata/print_ask.golden | 33 + .../assist/testdata/print_default.golden | 34 + .../assist/testdata/print_failed_run.golden | 38 + .../assist/testdata/print_no_project.golden | 34 + .../testdata/print_zero_connections.golden | 34 + tests/assist/assist_harness.go | 509 ++ tests/assist/assist_test.go | 333 ++ tests/assist/cases/01.install_first_run.yaml | 29 + tests/assist/cases/02.install_idempotent.yaml | 13 + tests/assist/cases/03.doctor_green.yaml | 16 + tests/assist/cases/04.history_table.yaml | 12 + tests/assist/cases/05.history_json.yaml | 11 + .../cases/06.investigate_file_print.yaml | 11 + tests/assist/cases/07.uninstall_all.yaml | 14 + tests/assist/cases/08.refusal_no_agent.yaml | 8 + tests/assist/cases/09.agent_override.yaml | 11 + .../cases/11.new_interactive_picker.yaml | 14 + .../cases/12.new_interactive_replication.yaml | 14 + .../cases/13.new_picker_lists_all_tasks.yaml | 16 + .../cases/14.new_edit_existing_with_path.yaml | 16 + .../15.new_interactive_build_create.yaml | 15 + .../cases/16.skills_hooks_refs_and_prune.yaml | 41 + .../17.skills_project_layout_and_secrets.yaml | 51 + tests/assist/cases/18.bare_sling_landing.yaml | 12 + tests/evals/README.md | 73 + .../e.01.repl_pg_duckdb_incremental/case.yaml | 38 + .../expected.yaml | 8 + .../mutants/m1.yaml | 8 + .../cases/e.02.repl_mysql_pg_select/case.yaml | 30 + .../e.02.repl_mysql_pg_select/expected.yaml | 7 + .../e.02.repl_mysql_pg_select/mutants/m1.yaml | 7 + .../e.03.repl_pg_clickhouse_keys/case.yaml | 30 + .../expected.yaml | 11 + .../mutants/m1.yaml | 8 + .../e.04.repl_ch_duckdb_backfill/case.yaml | 26 + .../expected.yaml | 8 + .../mutants/m1.yaml | 6 + .../cases/e.05.repl_pg_local_csv/case.yaml | 26 + .../e.05.repl_pg_local_csv/expected.yaml | 8 + .../e.05.repl_pg_local_csv/mutants/m1.yaml | 6 + .../e.06.repl_local_mysql_glob/case.yaml | 28 + .../e.06.repl_local_mysql_glob/expected.yaml | 7 + .../mutants/m1.yaml | 7 + .../cases/e.07.repl_pg_custom_sql/case.yaml | 27 + .../e.07.repl_pg_custom_sql/expected.sql | 4 + .../e.07.repl_pg_custom_sql/expected.yaml | 11 + .../e.07.repl_pg_custom_sql/mutants/m1.yaml | 7 + .../e.09.repl_pg_ch_multistream/case.yaml | 27 + .../e.09.repl_pg_ch_multistream/expected.yaml | 15 + .../mutants/m1.yaml | 6 + .../e.10.repl_update_incremental/case.yaml | 27 + .../expected.yaml | 8 + .../mutants/m1.yaml | 10 + .../seed/orders_repl.yaml | 7 + .../cases/e.11.repl_unknown_conn/case.yaml | 18 + .../e.11.repl_unknown_conn/mutants/m1.yaml | 6 + .../cases/e.12.repl_no_extra_hooks/case.yaml | 29 + .../e.12.repl_no_extra_hooks/expected.yaml | 6 + .../e.12.repl_no_extra_hooks/mutants/m1.yaml | 15 + .../cases/e.13.pipeline_http_duckdb/case.yaml | 28 + .../e.13.pipeline_http_duckdb/expected.yaml | 10 + .../e.13.pipeline_http_duckdb/mutants/m1.yaml | 5 + .../cases/e.14.pipeline_query_check/case.yaml | 26 + .../e.14.pipeline_query_check/expected.yaml | 9 + .../e.14.pipeline_query_check/mutants/m1.yaml | 3 + .../e.15.pipeline_list_copy_s3/case.yaml | 24 + .../e.15.pipeline_list_copy_s3/expected.yaml | 8 + .../mutants/m1.yaml | 3 + .../seed/data/a.csv | 3 + .../seed/data/b.csv | 2 + .../e.16.pipeline_update_add_check/case.yaml | 25 + .../expected.yaml | 7 + .../mutants/m1.yaml | 3 + .../seed/pipe.yaml | 5 + .../cases/e.17.pipeline_no_delete/case.yaml | 21 + .../e.17.pipeline_no_delete/expected.yaml | 7 + .../e.17.pipeline_no_delete/mutants/m1.yaml | 6 + .../cases/e.18.build_staging_view/case.yaml | 30 + .../e.18.build_staging_view/expected.yaml | 3 + .../e.18.build_staging_view/mutants/m1.yaml | 3 + .../seed/seeds/orders.csv | 2 + .../seed/staging/stg_orders.sql | 2 + .../cases/e.22.spec_simple_rest/case.yaml | 28 + .../cases/e.22.spec_simple_rest/expected.yaml | 27 + .../e.22.spec_simple_rest/mutants/m1.yaml | 11 + .../cases/e.23.spec_incremental/case.yaml | 29 + .../cases/e.23.spec_incremental/expected.yaml | 23 + .../e.23.spec_incremental/mutants/m1.yaml | 11 + .../e.24.spec_update_add_endpoint/case.yaml | 29 + .../expected.yaml | 28 + .../mutants/m1.yaml | 22 + .../seed/spec.yaml | 14 + .../cases/e.25.debug_bad_column/case.yaml | 21 + .../e.25.debug_bad_column/mutants/m1.yaml | 2 + .../evals/cases/e.26.debug_no_rerun/case.yaml | 17 + .../cases/e.26.debug_no_rerun/mutants/m1.yaml | 2 + .../cases/e.27.build_tpch_layers/case.yaml | 39 + .../e.27.build_tpch_layers/expected.yaml | 8 + .../e.27.build_tpch_layers/mutants/m1.yaml | 3 + .../e.27.build_tpch_layers/reference/ltv.sql | 8 + .../seed/intermediate/int_order_items.sql | 8 + .../seed/marts/dim_customers.sql | 10 + .../seed/marts/fct_orders.sql | 8 + .../seed/sling_build.yml | 3 + .../seed/staging/stg_customers.sql | 7 + .../seed/staging/stg_lineitem.sql | 8 + .../seed/staging/stg_orders.sql | 7 + .../e.28.build_tpch_incremental_ch/case.yaml | 31 + .../expected.yaml | 3 + .../mutants/m1.yaml | 3 + .../seed/marts/fct_lineitem_daily.sql | 11 + .../seed/sling_build.yml | 3 + .../cases/e.29.build_dirty_staging/case.yaml | 32 + .../e.29.build_dirty_staging/expected.yaml | 3 + .../e.29.build_dirty_staging/mutants/m1.yaml | 3 + .../seed/sling_build.yml | 3 + .../seed/staging/stg_orders.sql | 18 + .../cases/e.30.build_fix_broken_dag/case.yaml | 24 + .../expected/marts/mart_ok.sql | 1 + .../expected/seeds/orders.csv | 2 + .../expected/sling_build.yml | 1 + .../expected/staging/model_a.sql | 1 + .../expected/staging/model_b.sql | 1 + .../e.30.build_fix_broken_dag/mutants/m1.yaml | 3 + .../seed/marts/mart_ok.sql | 1 + .../seed/seeds/orders.csv | 2 + .../seed/sling_build.yml | 1 + .../seed/staging/model_a.sql | 1 + .../seed/staging/model_b.sql | 3 + .../cases/e.31.build_range_backfill/case.yaml | 28 + .../e.31.build_range_backfill/expected.yaml | 3 + .../e.31.build_range_backfill/mutants/m1.yaml | 3 + .../seed/marts/fct_orders.sql | 16 + .../seed/sling_build.yml | 3 + .../cases/e.32.build_dbt_migrate/case.yaml | 27 + .../e.32.build_dbt_migrate/expected.yaml | 2 + .../e.32.build_dbt_migrate/mutants/m1.yaml | 2 + .../seed/models/staging/stg_orders.sql | 1 + .../seed/seeds/staging/country_codes.csv | 3 + .../seed/sling_build.yml | 2 + .../e.33.repl_incremental_outcome/case.yaml | 54 + .../expected.yaml | 8 + .../mutants/m1.yaml | 6 + .../e.34.repl_transforms_parquet/case.yaml | 43 + .../expected.yaml | 7 + .../mutants/m1.yaml | 6 + .../cases/e.35.repl_api_source/case.yaml | 32 + .../cases/e.35.repl_api_source/expected.yaml | 6 + .../e.35.repl_api_source/mutants/m1.yaml | 6 + .../cases/e.36.pipeline_group_loop/case.yaml | 23 + .../e.36.pipeline_group_loop/expected.yaml | 10 + .../e.36.pipeline_group_loop/mutants/m1.yaml | 4 + .../e.37.pipeline_replication_step/case.yaml | 28 + .../expected.yaml | 9 + .../mutants/m1.yaml | 4 + .../seed/orders.yaml | 6 + .../cases/e.38.spec_cursor_stop/case.yaml | 27 + .../cases/e.38.spec_cursor_stop/expected.yaml | 20 + .../e.38.spec_cursor_stop/mutants/m1.yaml | 19 + tests/evals/cases/e.39.spec_oauth2/case.yaml | 24 + .../cases/e.39.spec_oauth2/expected.yaml | 16 + .../cases/e.39.spec_oauth2/mutants/m1.yaml | 13 + .../cases/e.40.spec_parent_child/case.yaml | 24 + .../e.40.spec_parent_child/expected.yaml | 29 + .../e.40.spec_parent_child/mutants/m1.yaml | 14 + .../cases/e.41.spec_from_openapi/case.yaml | 23 + .../e.41.spec_from_openapi/expected.yaml | 14 + .../e.41.spec_from_openapi/mutants/m1.yaml | 9 + .../e.41.spec_from_openapi/seed/openapi.json | 44 + .../e.42.spec_secrets_negative/case.yaml | 24 + .../e.42.spec_secrets_negative/expected.yaml | 14 + .../mutants/m1.yaml | 13 + .../cases/e.43.spec_real_dummyjson/case.yaml | 22 + .../e.43.spec_real_dummyjson/expected.yaml | 10 + .../e.43.spec_real_dummyjson/mutants/m1.yaml | 9 + .../cases/e.44.spec_real_github/case.yaml | 23 + .../cases/e.44.spec_real_github/expected.yaml | 14 + .../e.44.spec_real_github/mutants/m1.yaml | 11 + .../evals/cases/e.45.spec_real_omdb/case.yaml | 21 + .../cases/e.45.spec_real_omdb/expected.yaml | 10 + .../cases/e.45.spec_real_omdb/mutants/m1.yaml | 9 + .../cases/e.46.repl_cdc_create/case.yaml | 23 + .../cases/e.46.repl_cdc_create/expected.yaml | 15 + .../e.46.repl_cdc_create/mutants/m1.yaml | 7 + tests/evals/cases/e.47.cdc_debug/case.yaml | 23 + .../cases/e.47.cdc_debug/mutants/m1.yaml | 6 + .../evals/cases/e.49.debug_real_run/case.yaml | 36 + .../cases/e.49.debug_real_run/mutants/m1.yaml | 6 + tests/evals/eval.go | 1430 +++++ tests/evals/eval_test.go | 3533 ++++++++++++ tests/evals/fixtures.go | 458 ++ tests/evals/fixtures/data/e.03.expected.sql | 4 + .../fixtures/data/ecom/raw_customers.csv | 801 +++ .../fixtures/data/ecom/raw_customers.parquet | Bin 0 -> 11413 bytes tests/evals/fixtures/data/ecom/raw_events.csv | 3001 ++++++++++ .../fixtures/data/ecom/raw_events.parquet | Bin 0 -> 47367 bytes tests/evals/fixtures/data/ecom/raw_orders.csv | 5001 +++++++++++++++++ .../fixtures/data/ecom/raw_orders.parquet | Bin 0 -> 63104 bytes tests/evals/fixtures/data/mock_api.yaml | 27 + tests/evals/fixtures/data/orders.sql | 9 + tests/evals/fixtures/data/sample.csv | 3 + tests/evals/fixtures/home_claude/.claude.json | 8 + .../evals/fixtures/home_claude/.claude/.keep | 1 + .../fixtures/home_claude/.sling/env.yaml | 38 + .../fixtures/home_codex/.codex/config.toml | 16 + .../evals/fixtures/home_codex/.sling/env.yaml | 38 + .../fixtures/home_grok/.grok/config.toml | 3 + .../evals/fixtures/home_grok/.sling/env.yaml | 38 + .../.config/opencode/opencode.json | 37 + .../fixtures/home_opencode2/.sling/env.yaml | 38 + tests/evals/fixtures/registry.yaml | 37 + tests/evals/fixtures/seed/ecom_to_pg.yaml | 13 + tests/evals/fixtures/seed/tpch_to_ch.yaml | 16 + tests/evals/fixtures/seed/tpch_to_pg.yaml | 16 + tests/evals/fixtures/var/.gitignore | 2 + tests/evals/graders.go | 2517 +++++++++ tests/evals/results/.gitignore | 4 + tests/evals/results/round3-live.log | 27 + tests/evals/runner.go | 2390 ++++++++ tests/evals/server.go | 227 + tests/evals/testdata/baseline_a.jsonl | 2 + tests/evals/testdata/baseline_b.jsonl | 2 + .../evals/testdata/judge_response_sonnet.json | 1 + tests/evals/testdata/judge_shadow.md | 10 + tests/evals/timed_unix.go | 21 + tests/evals/timed_windows.go | 24 + tests/evals/validate.go | 280 + tests/suite.cli.assist.yaml | 308 + 293 files changed, 44485 insertions(+), 343 deletions(-) create mode 100644 cmd/sling/sling_assist.go create mode 100644 cmd/sling/sling_assist_test.go create mode 100644 cmd/sling/sling_init.go delete mode 100644 cmd/sling/sling_prompt.go create mode 100644 core/sling/assist/assist.go create mode 100644 core/sling/assist/assist_test.go create mode 100644 core/sling/assist/browser.go create mode 100644 core/sling/assist/browser_test.go create mode 100644 core/sling/assist/clients.go create mode 100644 core/sling/assist/clients_test.go create mode 100644 core/sling/assist/doctor.go create mode 100644 core/sling/assist/history.go create mode 100644 core/sling/assist/install.go create mode 100644 core/sling/assist/investigate.go create mode 100644 core/sling/assist/investigate_test.go create mode 100644 core/sling/assist/jsonedit.go create mode 100644 core/sling/assist/opencode.go create mode 100644 core/sling/assist/opencode_network_test.go create mode 100644 core/sling/assist/opencode_test.go create mode 100644 core/sling/assist/prompt.go create mode 100644 core/sling/assist/prompt_test.go create mode 100644 core/sling/assist/prompts.yaml create mode 100644 core/sling/assist/report.go create mode 100644 core/sling/assist/report_test.go create mode 100644 core/sling/assist/session.go create mode 100644 core/sling/assist/skills/agent-browser/CORE.md create mode 100644 core/sling/assist/skills/agent-browser/SKILL.md create mode 100644 core/sling/assist/skills/sling-api-specs/AUTHENTICATION.md create mode 100644 core/sling/assist/skills/sling-api-specs/DYNAMIC.md create mode 100644 core/sling/assist/skills/sling-api-specs/ENDPOINTS.md create mode 100644 core/sling/assist/skills/sling-api-specs/FUNCTIONS.md create mode 100644 core/sling/assist/skills/sling-api-specs/INCREMENTAL.md create mode 100644 core/sling/assist/skills/sling-api-specs/PAGINATION.md create mode 100644 core/sling/assist/skills/sling-api-specs/PROCESSORS.md create mode 100644 core/sling/assist/skills/sling-api-specs/QUEUES.md create mode 100644 core/sling/assist/skills/sling-api-specs/REQUEST.md create mode 100644 core/sling/assist/skills/sling-api-specs/RESPONSE.md create mode 100644 core/sling/assist/skills/sling-api-specs/RULES.md create mode 100644 core/sling/assist/skills/sling-api-specs/SKILL.md create mode 100644 core/sling/assist/skills/sling-api-specs/VARIABLES.md create mode 100644 core/sling/assist/skills/sling-build/SKILL.md create mode 100644 core/sling/assist/skills/sling-build/STRUCTURE.md create mode 100644 core/sling/assist/skills/sling-connections/SKILL.md create mode 100644 core/sling/assist/skills/sling-pipelines/SKILL.md create mode 100644 core/sling/assist/skills/sling-pipelines/STEPS.md create mode 100644 core/sling/assist/skills/sling-platform/SKILL.md create mode 100644 core/sling/assist/skills/sling-project/SKILL.md create mode 100644 core/sling/assist/skills/sling-project/STRUCTURE.md create mode 100644 core/sling/assist/skills/sling-python/SKILL.md create mode 100644 core/sling/assist/skills/sling-replications/CDC.md create mode 100644 core/sling/assist/skills/sling-replications/SKILL.md create mode 100644 core/sling/assist/skills/sling-replications/TRANSFORMS.md create mode 100644 core/sling/assist/skills/sling/SKILL.md create mode 100644 core/sling/assist/skills/sling/TROUBLESHOOTING.md create mode 100644 core/sling/assist/testdata/print_ask.golden create mode 100644 core/sling/assist/testdata/print_default.golden create mode 100644 core/sling/assist/testdata/print_failed_run.golden create mode 100644 core/sling/assist/testdata/print_no_project.golden create mode 100644 core/sling/assist/testdata/print_zero_connections.golden create mode 100644 tests/assist/assist_harness.go create mode 100644 tests/assist/assist_test.go create mode 100644 tests/assist/cases/01.install_first_run.yaml create mode 100644 tests/assist/cases/02.install_idempotent.yaml create mode 100644 tests/assist/cases/03.doctor_green.yaml create mode 100644 tests/assist/cases/04.history_table.yaml create mode 100644 tests/assist/cases/05.history_json.yaml create mode 100644 tests/assist/cases/06.investigate_file_print.yaml create mode 100644 tests/assist/cases/07.uninstall_all.yaml create mode 100644 tests/assist/cases/08.refusal_no_agent.yaml create mode 100644 tests/assist/cases/09.agent_override.yaml create mode 100644 tests/assist/cases/11.new_interactive_picker.yaml create mode 100644 tests/assist/cases/12.new_interactive_replication.yaml create mode 100644 tests/assist/cases/13.new_picker_lists_all_tasks.yaml create mode 100644 tests/assist/cases/14.new_edit_existing_with_path.yaml create mode 100644 tests/assist/cases/15.new_interactive_build_create.yaml create mode 100644 tests/assist/cases/16.skills_hooks_refs_and_prune.yaml create mode 100644 tests/assist/cases/17.skills_project_layout_and_secrets.yaml create mode 100644 tests/assist/cases/18.bare_sling_landing.yaml create mode 100644 tests/evals/README.md create mode 100644 tests/evals/cases/e.01.repl_pg_duckdb_incremental/case.yaml create mode 100644 tests/evals/cases/e.01.repl_pg_duckdb_incremental/expected.yaml create mode 100644 tests/evals/cases/e.01.repl_pg_duckdb_incremental/mutants/m1.yaml create mode 100644 tests/evals/cases/e.02.repl_mysql_pg_select/case.yaml create mode 100644 tests/evals/cases/e.02.repl_mysql_pg_select/expected.yaml create mode 100644 tests/evals/cases/e.02.repl_mysql_pg_select/mutants/m1.yaml create mode 100644 tests/evals/cases/e.03.repl_pg_clickhouse_keys/case.yaml create mode 100644 tests/evals/cases/e.03.repl_pg_clickhouse_keys/expected.yaml create mode 100644 tests/evals/cases/e.03.repl_pg_clickhouse_keys/mutants/m1.yaml create mode 100644 tests/evals/cases/e.04.repl_ch_duckdb_backfill/case.yaml create mode 100644 tests/evals/cases/e.04.repl_ch_duckdb_backfill/expected.yaml create mode 100644 tests/evals/cases/e.04.repl_ch_duckdb_backfill/mutants/m1.yaml create mode 100644 tests/evals/cases/e.05.repl_pg_local_csv/case.yaml create mode 100644 tests/evals/cases/e.05.repl_pg_local_csv/expected.yaml create mode 100644 tests/evals/cases/e.05.repl_pg_local_csv/mutants/m1.yaml create mode 100644 tests/evals/cases/e.06.repl_local_mysql_glob/case.yaml create mode 100644 tests/evals/cases/e.06.repl_local_mysql_glob/expected.yaml create mode 100644 tests/evals/cases/e.06.repl_local_mysql_glob/mutants/m1.yaml create mode 100644 tests/evals/cases/e.07.repl_pg_custom_sql/case.yaml create mode 100644 tests/evals/cases/e.07.repl_pg_custom_sql/expected.sql create mode 100644 tests/evals/cases/e.07.repl_pg_custom_sql/expected.yaml create mode 100644 tests/evals/cases/e.07.repl_pg_custom_sql/mutants/m1.yaml create mode 100644 tests/evals/cases/e.09.repl_pg_ch_multistream/case.yaml create mode 100644 tests/evals/cases/e.09.repl_pg_ch_multistream/expected.yaml create mode 100644 tests/evals/cases/e.09.repl_pg_ch_multistream/mutants/m1.yaml create mode 100644 tests/evals/cases/e.10.repl_update_incremental/case.yaml create mode 100644 tests/evals/cases/e.10.repl_update_incremental/expected.yaml create mode 100644 tests/evals/cases/e.10.repl_update_incremental/mutants/m1.yaml create mode 100644 tests/evals/cases/e.10.repl_update_incremental/seed/orders_repl.yaml create mode 100644 tests/evals/cases/e.11.repl_unknown_conn/case.yaml create mode 100644 tests/evals/cases/e.11.repl_unknown_conn/mutants/m1.yaml create mode 100644 tests/evals/cases/e.12.repl_no_extra_hooks/case.yaml create mode 100644 tests/evals/cases/e.12.repl_no_extra_hooks/expected.yaml create mode 100644 tests/evals/cases/e.12.repl_no_extra_hooks/mutants/m1.yaml create mode 100644 tests/evals/cases/e.13.pipeline_http_duckdb/case.yaml create mode 100644 tests/evals/cases/e.13.pipeline_http_duckdb/expected.yaml create mode 100644 tests/evals/cases/e.13.pipeline_http_duckdb/mutants/m1.yaml create mode 100644 tests/evals/cases/e.14.pipeline_query_check/case.yaml create mode 100644 tests/evals/cases/e.14.pipeline_query_check/expected.yaml create mode 100644 tests/evals/cases/e.14.pipeline_query_check/mutants/m1.yaml create mode 100644 tests/evals/cases/e.15.pipeline_list_copy_s3/case.yaml create mode 100644 tests/evals/cases/e.15.pipeline_list_copy_s3/expected.yaml create mode 100644 tests/evals/cases/e.15.pipeline_list_copy_s3/mutants/m1.yaml create mode 100644 tests/evals/cases/e.15.pipeline_list_copy_s3/seed/data/a.csv create mode 100644 tests/evals/cases/e.15.pipeline_list_copy_s3/seed/data/b.csv create mode 100644 tests/evals/cases/e.16.pipeline_update_add_check/case.yaml create mode 100644 tests/evals/cases/e.16.pipeline_update_add_check/expected.yaml create mode 100644 tests/evals/cases/e.16.pipeline_update_add_check/mutants/m1.yaml create mode 100644 tests/evals/cases/e.16.pipeline_update_add_check/seed/pipe.yaml create mode 100644 tests/evals/cases/e.17.pipeline_no_delete/case.yaml create mode 100644 tests/evals/cases/e.17.pipeline_no_delete/expected.yaml create mode 100644 tests/evals/cases/e.17.pipeline_no_delete/mutants/m1.yaml create mode 100644 tests/evals/cases/e.18.build_staging_view/case.yaml create mode 100644 tests/evals/cases/e.18.build_staging_view/expected.yaml create mode 100644 tests/evals/cases/e.18.build_staging_view/mutants/m1.yaml create mode 100644 tests/evals/cases/e.18.build_staging_view/seed/seeds/orders.csv create mode 100644 tests/evals/cases/e.18.build_staging_view/seed/staging/stg_orders.sql create mode 100644 tests/evals/cases/e.22.spec_simple_rest/case.yaml create mode 100644 tests/evals/cases/e.22.spec_simple_rest/expected.yaml create mode 100644 tests/evals/cases/e.22.spec_simple_rest/mutants/m1.yaml create mode 100644 tests/evals/cases/e.23.spec_incremental/case.yaml create mode 100644 tests/evals/cases/e.23.spec_incremental/expected.yaml create mode 100644 tests/evals/cases/e.23.spec_incremental/mutants/m1.yaml create mode 100644 tests/evals/cases/e.24.spec_update_add_endpoint/case.yaml create mode 100644 tests/evals/cases/e.24.spec_update_add_endpoint/expected.yaml create mode 100644 tests/evals/cases/e.24.spec_update_add_endpoint/mutants/m1.yaml create mode 100644 tests/evals/cases/e.24.spec_update_add_endpoint/seed/spec.yaml create mode 100644 tests/evals/cases/e.25.debug_bad_column/case.yaml create mode 100644 tests/evals/cases/e.25.debug_bad_column/mutants/m1.yaml create mode 100644 tests/evals/cases/e.26.debug_no_rerun/case.yaml create mode 100644 tests/evals/cases/e.26.debug_no_rerun/mutants/m1.yaml create mode 100644 tests/evals/cases/e.27.build_tpch_layers/case.yaml create mode 100644 tests/evals/cases/e.27.build_tpch_layers/expected.yaml create mode 100644 tests/evals/cases/e.27.build_tpch_layers/mutants/m1.yaml create mode 100644 tests/evals/cases/e.27.build_tpch_layers/reference/ltv.sql create mode 100644 tests/evals/cases/e.27.build_tpch_layers/seed/intermediate/int_order_items.sql create mode 100644 tests/evals/cases/e.27.build_tpch_layers/seed/marts/dim_customers.sql create mode 100644 tests/evals/cases/e.27.build_tpch_layers/seed/marts/fct_orders.sql create mode 100644 tests/evals/cases/e.27.build_tpch_layers/seed/sling_build.yml create mode 100644 tests/evals/cases/e.27.build_tpch_layers/seed/staging/stg_customers.sql create mode 100644 tests/evals/cases/e.27.build_tpch_layers/seed/staging/stg_lineitem.sql create mode 100644 tests/evals/cases/e.27.build_tpch_layers/seed/staging/stg_orders.sql create mode 100644 tests/evals/cases/e.28.build_tpch_incremental_ch/case.yaml create mode 100644 tests/evals/cases/e.28.build_tpch_incremental_ch/expected.yaml create mode 100644 tests/evals/cases/e.28.build_tpch_incremental_ch/mutants/m1.yaml create mode 100644 tests/evals/cases/e.28.build_tpch_incremental_ch/seed/marts/fct_lineitem_daily.sql create mode 100644 tests/evals/cases/e.28.build_tpch_incremental_ch/seed/sling_build.yml create mode 100644 tests/evals/cases/e.29.build_dirty_staging/case.yaml create mode 100644 tests/evals/cases/e.29.build_dirty_staging/expected.yaml create mode 100644 tests/evals/cases/e.29.build_dirty_staging/mutants/m1.yaml create mode 100644 tests/evals/cases/e.29.build_dirty_staging/seed/sling_build.yml create mode 100644 tests/evals/cases/e.29.build_dirty_staging/seed/staging/stg_orders.sql create mode 100644 tests/evals/cases/e.30.build_fix_broken_dag/case.yaml create mode 100644 tests/evals/cases/e.30.build_fix_broken_dag/expected/marts/mart_ok.sql create mode 100644 tests/evals/cases/e.30.build_fix_broken_dag/expected/seeds/orders.csv create mode 100644 tests/evals/cases/e.30.build_fix_broken_dag/expected/sling_build.yml create mode 100644 tests/evals/cases/e.30.build_fix_broken_dag/expected/staging/model_a.sql create mode 100644 tests/evals/cases/e.30.build_fix_broken_dag/expected/staging/model_b.sql create mode 100644 tests/evals/cases/e.30.build_fix_broken_dag/mutants/m1.yaml create mode 100644 tests/evals/cases/e.30.build_fix_broken_dag/seed/marts/mart_ok.sql create mode 100644 tests/evals/cases/e.30.build_fix_broken_dag/seed/seeds/orders.csv create mode 100644 tests/evals/cases/e.30.build_fix_broken_dag/seed/sling_build.yml create mode 100644 tests/evals/cases/e.30.build_fix_broken_dag/seed/staging/model_a.sql create mode 100644 tests/evals/cases/e.30.build_fix_broken_dag/seed/staging/model_b.sql create mode 100644 tests/evals/cases/e.31.build_range_backfill/case.yaml create mode 100644 tests/evals/cases/e.31.build_range_backfill/expected.yaml create mode 100644 tests/evals/cases/e.31.build_range_backfill/mutants/m1.yaml create mode 100644 tests/evals/cases/e.31.build_range_backfill/seed/marts/fct_orders.sql create mode 100644 tests/evals/cases/e.31.build_range_backfill/seed/sling_build.yml create mode 100644 tests/evals/cases/e.32.build_dbt_migrate/case.yaml create mode 100644 tests/evals/cases/e.32.build_dbt_migrate/expected.yaml create mode 100644 tests/evals/cases/e.32.build_dbt_migrate/mutants/m1.yaml create mode 100644 tests/evals/cases/e.32.build_dbt_migrate/seed/models/staging/stg_orders.sql create mode 100644 tests/evals/cases/e.32.build_dbt_migrate/seed/seeds/staging/country_codes.csv create mode 100644 tests/evals/cases/e.32.build_dbt_migrate/seed/sling_build.yml create mode 100644 tests/evals/cases/e.33.repl_incremental_outcome/case.yaml create mode 100644 tests/evals/cases/e.33.repl_incremental_outcome/expected.yaml create mode 100644 tests/evals/cases/e.33.repl_incremental_outcome/mutants/m1.yaml create mode 100644 tests/evals/cases/e.34.repl_transforms_parquet/case.yaml create mode 100644 tests/evals/cases/e.34.repl_transforms_parquet/expected.yaml create mode 100644 tests/evals/cases/e.34.repl_transforms_parquet/mutants/m1.yaml create mode 100644 tests/evals/cases/e.35.repl_api_source/case.yaml create mode 100644 tests/evals/cases/e.35.repl_api_source/expected.yaml create mode 100644 tests/evals/cases/e.35.repl_api_source/mutants/m1.yaml create mode 100644 tests/evals/cases/e.36.pipeline_group_loop/case.yaml create mode 100644 tests/evals/cases/e.36.pipeline_group_loop/expected.yaml create mode 100644 tests/evals/cases/e.36.pipeline_group_loop/mutants/m1.yaml create mode 100644 tests/evals/cases/e.37.pipeline_replication_step/case.yaml create mode 100644 tests/evals/cases/e.37.pipeline_replication_step/expected.yaml create mode 100644 tests/evals/cases/e.37.pipeline_replication_step/mutants/m1.yaml create mode 100644 tests/evals/cases/e.37.pipeline_replication_step/seed/orders.yaml create mode 100644 tests/evals/cases/e.38.spec_cursor_stop/case.yaml create mode 100644 tests/evals/cases/e.38.spec_cursor_stop/expected.yaml create mode 100644 tests/evals/cases/e.38.spec_cursor_stop/mutants/m1.yaml create mode 100644 tests/evals/cases/e.39.spec_oauth2/case.yaml create mode 100644 tests/evals/cases/e.39.spec_oauth2/expected.yaml create mode 100644 tests/evals/cases/e.39.spec_oauth2/mutants/m1.yaml create mode 100644 tests/evals/cases/e.40.spec_parent_child/case.yaml create mode 100644 tests/evals/cases/e.40.spec_parent_child/expected.yaml create mode 100644 tests/evals/cases/e.40.spec_parent_child/mutants/m1.yaml create mode 100644 tests/evals/cases/e.41.spec_from_openapi/case.yaml create mode 100644 tests/evals/cases/e.41.spec_from_openapi/expected.yaml create mode 100644 tests/evals/cases/e.41.spec_from_openapi/mutants/m1.yaml create mode 100644 tests/evals/cases/e.41.spec_from_openapi/seed/openapi.json create mode 100644 tests/evals/cases/e.42.spec_secrets_negative/case.yaml create mode 100644 tests/evals/cases/e.42.spec_secrets_negative/expected.yaml create mode 100644 tests/evals/cases/e.42.spec_secrets_negative/mutants/m1.yaml create mode 100644 tests/evals/cases/e.43.spec_real_dummyjson/case.yaml create mode 100644 tests/evals/cases/e.43.spec_real_dummyjson/expected.yaml create mode 100644 tests/evals/cases/e.43.spec_real_dummyjson/mutants/m1.yaml create mode 100644 tests/evals/cases/e.44.spec_real_github/case.yaml create mode 100644 tests/evals/cases/e.44.spec_real_github/expected.yaml create mode 100644 tests/evals/cases/e.44.spec_real_github/mutants/m1.yaml create mode 100644 tests/evals/cases/e.45.spec_real_omdb/case.yaml create mode 100644 tests/evals/cases/e.45.spec_real_omdb/expected.yaml create mode 100644 tests/evals/cases/e.45.spec_real_omdb/mutants/m1.yaml create mode 100644 tests/evals/cases/e.46.repl_cdc_create/case.yaml create mode 100644 tests/evals/cases/e.46.repl_cdc_create/expected.yaml create mode 100644 tests/evals/cases/e.46.repl_cdc_create/mutants/m1.yaml create mode 100644 tests/evals/cases/e.47.cdc_debug/case.yaml create mode 100644 tests/evals/cases/e.47.cdc_debug/mutants/m1.yaml create mode 100644 tests/evals/cases/e.49.debug_real_run/case.yaml create mode 100644 tests/evals/cases/e.49.debug_real_run/mutants/m1.yaml create mode 100644 tests/evals/eval.go create mode 100644 tests/evals/eval_test.go create mode 100644 tests/evals/fixtures.go create mode 100644 tests/evals/fixtures/data/e.03.expected.sql create mode 100644 tests/evals/fixtures/data/ecom/raw_customers.csv create mode 100644 tests/evals/fixtures/data/ecom/raw_customers.parquet create mode 100644 tests/evals/fixtures/data/ecom/raw_events.csv create mode 100644 tests/evals/fixtures/data/ecom/raw_events.parquet create mode 100644 tests/evals/fixtures/data/ecom/raw_orders.csv create mode 100644 tests/evals/fixtures/data/ecom/raw_orders.parquet create mode 100644 tests/evals/fixtures/data/mock_api.yaml create mode 100644 tests/evals/fixtures/data/orders.sql create mode 100644 tests/evals/fixtures/data/sample.csv create mode 100644 tests/evals/fixtures/home_claude/.claude.json create mode 100644 tests/evals/fixtures/home_claude/.claude/.keep create mode 100644 tests/evals/fixtures/home_claude/.sling/env.yaml create mode 100644 tests/evals/fixtures/home_codex/.codex/config.toml create mode 100644 tests/evals/fixtures/home_codex/.sling/env.yaml create mode 100644 tests/evals/fixtures/home_grok/.grok/config.toml create mode 100644 tests/evals/fixtures/home_grok/.sling/env.yaml create mode 100644 tests/evals/fixtures/home_opencode2/.config/opencode/opencode.json create mode 100644 tests/evals/fixtures/home_opencode2/.sling/env.yaml create mode 100644 tests/evals/fixtures/registry.yaml create mode 100644 tests/evals/fixtures/seed/ecom_to_pg.yaml create mode 100644 tests/evals/fixtures/seed/tpch_to_ch.yaml create mode 100644 tests/evals/fixtures/seed/tpch_to_pg.yaml create mode 100644 tests/evals/fixtures/var/.gitignore create mode 100644 tests/evals/graders.go create mode 100644 tests/evals/results/.gitignore create mode 100644 tests/evals/results/round3-live.log create mode 100644 tests/evals/runner.go create mode 100644 tests/evals/server.go create mode 100644 tests/evals/testdata/baseline_a.jsonl create mode 100644 tests/evals/testdata/baseline_b.jsonl create mode 100644 tests/evals/testdata/judge_response_sonnet.json create mode 100644 tests/evals/testdata/judge_shadow.md create mode 100644 tests/evals/timed_unix.go create mode 100644 tests/evals/timed_windows.go create mode 100644 tests/evals/validate.go create mode 100644 tests/suite.cli.assist.yaml diff --git a/cmd/sling/resource/llm_API_SPEC.md b/cmd/sling/resource/llm_API_SPEC.md index 800680122..56378561c 100644 --- a/cmd/sling/resource/llm_API_SPEC.md +++ b/cmd/sling/resource/llm_API_SPEC.md @@ -159,7 +159,7 @@ endpoints: { coalesce( env.LAST_UPDATED, sync.last_updated, - date_format(date_add(now(), -30, 'day'), '%Y-%m-%dT%H:%M:%SZ') + date_format(date_add(now(), -30, "day"), "%Y-%m-%dT%H:%M:%SZ") ) } page: 1 # Example for page-based pagination @@ -976,7 +976,7 @@ endpoints: start_timestamp: > { coalesce( sync.last_sync_ts, - date_format(date_add(now(), -7, 'day'), '%Y-%m-%dT%H:%M:%SZ') + date_format(date_add(now(), -7, "day"), "%Y-%m-%dT%H:%M:%SZ") ) } @@ -1162,7 +1162,7 @@ endpoints: You can use the following functions within `{...}` expressions in your API spec. Functions provide capabilities for data manipulation, type casting, date operations, control flow, and more. -**IMPORTANT:** Always use double quotes (`"`) for string literals in expressions, never single quotes (`'`). This is required by the [goval](https://github.com/maja42/goval) expression library that Sling uses. +**String literals:** double quotes (`"`) are preferred. Single-quoted SQL-style literals (`'%Y-%m-%d'`) are also accepted. ### String Functions @@ -1207,7 +1207,7 @@ Uses Go's `time` package and `strftime` conventions via [timefmt-go](https://git | `date_extract(date, part)` | Extracts part from date | `date`, `part` ("year", "month", "day", "hour", etc.) | Number | `date_extract(now(), "year")` → 2023 | | `date_last(date[, period])` | Gets last day of period | `date`, `period` ("month", "year", default "month") | Time object | `date_last(now())` → Last day of current month | | `date_first(date[, period])` | Gets first day of period | `date`, `period` ("month", "year", default "month") | Time object | `date_first(now())` → First day of current month | -| `range(start, end[, step])` | Creates array of time objects | `start`, `end` (time obj), `step` (string duration) | Array of Time objects | `range(date_add(now(),-2,'day'), now(), '1d')` → `[t-2d, t-1d, t]` | +| `range(start, end[, step])` | Creates array of time objects | `start`, `end` (time obj), `step` (string duration) | Array of Time objects | `range(date_add(now(),-2,"day"), now(), "1d")` → `[t-2d, t-1d, t]` | *Date function `unit`/`part`/`period` parameters often accept: "year", "month", "week", "day", "hour", "minute", "second".* *`range` function with dates requires time objects as start/end.* @@ -1230,7 +1230,7 @@ Uses Go's `time` package and `strftime` conventions via [timefmt-go](https://git # Format for API parameter (ISO 8601 with timezone) request: parameters: - updated_since: "{date_format(date_add(now(), -1, 'hour'), '%Y-%m-%dT%H:%M:%SZ')}" + updated_since: "{date_format(date_add(now(), -1, \"hour\"), \"%Y-%m-%dT%H:%M:%SZ\")}" ``` ### Value Handling Functions diff --git a/cmd/sling/resource/llm_CONNECTION.md b/cmd/sling/resource/llm_CONNECTION.md index 820595632..89ac04abc 100644 --- a/cmd/sling/resource/llm_CONNECTION.md +++ b/cmd/sling/resource/llm_CONNECTION.md @@ -255,9 +255,9 @@ API specifications define: "type": "api", "spec": "file:///path/to/my_api_spec.yaml", "secrets": { - "api_key": "your-api-key", - "client_id": "your-client-id", - "client_secret": "your-client-secret" + "api_key": "${MY_API_API_KEY}", + "client_id": "${MY_API_CLIENT_ID}", + "client_secret": "${MY_API_CLIENT_SECRET}" } } } @@ -295,7 +295,7 @@ Create or update a connection in the Sling environment file: "host": "localhost", "user": "myuser", "database": "mydb", - "password": "mypass", + "password": "${MY_POSTGRES_PASSWORD}", "port": 5432 } } @@ -304,8 +304,8 @@ Create or update a connection in the Sling environment file: **Important Notes:** - Check existing connections with `list` before overwriting -- Sensitive credentials should be manually set in `~/.sling/env.yaml` -- The tool will provide the env.yaml file path after setting +- Never pass secret values. Use a `${VAR}` ref, or omit the secret field to scaffold a ref +- The tool returns `{path, line, missing}` so the user can replace refs in `env.yaml` ### Testing Connections @@ -430,7 +430,7 @@ my_project: type: postgres host: localhost user: myuser - password: mypass + password: ${PG_PASSWORD} port: 5432 dbname: mydb schema: public @@ -605,10 +605,10 @@ streams: "type": "api", "spec": "file:///configs/salesforce_spec.yaml", "secrets": { - "client_id": "your-client-id", - "client_secret": "your-client-secret", - "username": "api-user@company.com", - "password": "password-with-token" + "client_id": "${SALESFORCE_API_CLIENT_ID}", + "client_secret": "${SALESFORCE_API_CLIENT_SECRET}", + "username": "${SALESFORCE_API_USERNAME}", + "password": "${SALESFORCE_API_PASSWORD}" } } } diff --git a/cmd/sling/resource/llm_CONNECTION_DATABASE.md b/cmd/sling/resource/llm_CONNECTION_DATABASE.md index 7893e198a..0687abac8 100644 --- a/cmd/sling/resource/llm_CONNECTION_DATABASE.md +++ b/cmd/sling/resource/llm_CONNECTION_DATABASE.md @@ -72,7 +72,7 @@ The `database` tool provides database-specific operations through various action } ``` -**Important**: Database operations require a Pro token and are rate-limited. +**Important**: Database operations are read-oriented and run locally through your configured connections. --- @@ -85,7 +85,6 @@ Before using database operations: 1. **Connection Setup**: Database connection must be configured using the `connection` tool 2. **Connection Testing**: Verify connection works with `connection` tool `test` action 3. **Permissions**: Database user must have SELECT permissions on target objects -4. **Token**: Operations require a valid Sling CLI Pro token ### Operation Categories diff --git a/cmd/sling/resource/llm_CONNECTION_FILE.md b/cmd/sling/resource/llm_CONNECTION_FILE.md index 6dc7e47d1..5e43e3216 100644 --- a/cmd/sling/resource/llm_CONNECTION_FILE.md +++ b/cmd/sling/resource/llm_CONNECTION_FILE.md @@ -72,7 +72,7 @@ The `file_system` tool provides file system-specific operations through various } ``` -**Important**: File system operations require a Pro token and are rate-limited. +**Important**: File system operations are read-oriented and run locally through your configured connections. --- @@ -85,7 +85,6 @@ Before using file system operations: 1. **Connection Setup**: File system connection must be configured using the `connection` tool 2. **Connection Testing**: Verify connection works with `connection` tool `test` action 3. **Permissions**: Connection must have appropriate read/write permissions -4. **Token**: Operations require a valid Sling CLI Pro token ### Operation Categories diff --git a/cmd/sling/resource/llm_PIPELINE.md b/cmd/sling/resource/llm_PIPELINE.md index 746316f00..c0b02d589 100644 --- a/cmd/sling/resource/llm_PIPELINE.md +++ b/cmd/sling/resource/llm_PIPELINE.md @@ -48,14 +48,16 @@ Sling pipelines are powerful YAML-based workflow definitions that allow you to o The Sling MCP tool provides these pipeline commands: - `pipeline/docs` - Get documentation. -- `pipeline/parse` - Parse and validate the pipeline configuration. -- `pipeline/run` - Execute the pipeline. +- `pipeline/validate` - Parse and validate the pipeline configuration. + +There is no MCP `run` action. Execute pipelines with the CLI: `sling run -p /path/to/pipeline.yaml`. +There is no MCP `build` action. Execute SQL models with the CLI: `sling build`. --- ## 2. Quick Start Guide -### Essential MCP Commands +### Essential Commands ```json // Get pipeline documentation @@ -66,21 +68,18 @@ The Sling MCP tool provides these pipeline commands: // Parse a pipeline file { - "action": "parse", + "action": "validate", "input": { "file_path": "/path/to/pipeline.yaml", "working_dir": "/optional/work/dir" } } +``` -// Run a pipeline -{ - "action": "run", - "input": { - "file_path": "/path/to/pipeline.yaml", - "working_dir": "/optional/work/dir" - } -} +```bash +# Run a pipeline (CLI only — no MCP run action) +sling run -p /path/to/pipeline.yaml +sling run -p /path/to/pipeline.yaml --debug ``` ### Basic Pipeline Structure @@ -603,7 +602,7 @@ steps: #### Parse Configuration ```json { - "action": "parse", + "action": "validate", "input": { "file_path": "/path/to/pipeline.yaml", "working_dir": "/optional/work/dir" @@ -612,25 +611,20 @@ steps: ``` #### Execute Pipeline -```json -{ - "action": "run", - "input": { - "file_path": "/path/to/pipeline.yaml", - "working_dir": "/optional/work/dir", - "env": { - "CUSTOM_VAR": "value" - } - } -} + +There is no MCP `run` action. Use the CLI: + +```bash +sling run -p /path/to/pipeline.yaml +sling run -p /path/to/pipeline.yaml --debug ``` ### Development Workflow 1. **Write** your pipeline YAML file. -2. **Parse** the configuration to validate syntax: `{"action": "parse", "input": {"file_path": "my_pipeline.yaml"}}`. +2. **Validate** the configuration with MCP: `{"action": "validate", "input": {"file_path": "my_pipeline.yaml"}}`. 3. **Test** individual steps if possible (e.g., run `query` or `command` steps manually). -4. **Run** the full pipeline: `{"action": "run", "input": {"file_path": "my_pipeline.yaml"}}`. +4. **Run** the full pipeline with the CLI: `sling run -p my_pipeline.yaml`. 5. **Debug** by checking the logs and the output of each step. --- diff --git a/cmd/sling/resource/llm_PLATFORM.md b/cmd/sling/resource/llm_PLATFORM.md index b11bff0c9..a13a8d723 100644 --- a/cmd/sling/resource/llm_PLATFORM.md +++ b/cmd/sling/resource/llm_PLATFORM.md @@ -4,7 +4,7 @@ 1. [Introduction and Overview](#1-introduction-and-overview) 2. [Authentication](#2-authentication) -3. [`sling project init`](#3-sling-project-init) +3. [`sling init`](#3-sling-init) 4. [`sling project status`](#4-sling-project-status) 5. [`sling project sync`](#5-sling-project-sync) 6. [`sling project jobs list`](#6-sling-project-jobs-list) @@ -72,12 +72,12 @@ By default the CLI talks to `https://api.slingdata.io`. Override with: --- -## 3. `sling project init` +## 3. `sling init` Create a `.sling.json` marker in the current directory so the CLI knows it is inside a project. Subsequent commands will be scoped to this directory tree. ```bash -sling project init +sling init ``` Effects: @@ -550,7 +550,7 @@ The full `Job` JSON shape accepted by `save` and returned by `get`: ```bash export SLING_PROJECT_TOKEN=... cd ~/my-project -sling project init +sling init # ... write replications/nightly.yaml ... sling project sync -f sling project jobs save --payload '{"name":"nightly","type":"replication","file_name":"replications/nightly.yaml","active":true,"schedules":["0 2 * * *"],"timezone":"UTC","config":{"mode":"incremental","threads":4}}' diff --git a/cmd/sling/resource/llm_REPLICATION.md b/cmd/sling/resource/llm_REPLICATION.md index d5114df73..456fc442f 100644 --- a/cmd/sling/resource/llm_REPLICATION.md +++ b/cmd/sling/resource/llm_REPLICATION.md @@ -57,15 +57,15 @@ Sling replications are YAML or JSON configuration files that define data movemen The Sling MCP tool provides these replication commands: - `replication/docs` - Get documentation -- `replication/parse` - Parse and validate configuration -- `replication/compile` - Compile configuration with validation -- `replication/run` - Execute the replication +- `replication/validate` - Parse and compile configuration (default compile is true). `compile: false` is parse-only and does not mean the file is ready to run. + +There is no MCP `run` action. Execute replications with the CLI: `sling run -r /path/to/replication.yaml`. --- ## 2. Quick Start Guide -### Essential MCP Commands +### Essential Commands ```json // Get replication documentation @@ -76,20 +76,17 @@ The Sling MCP tool provides these replication commands: // Parse a replication file { - "action": "parse", + "action": "validate", "input": { "file_path": "/path/to/replication.yaml" } } +``` -// Run a replication -{ - "action": "run", - "input": { - "file_path": "/path/to/replication.yaml", - "mode": "incremental" - } -} +```bash +# Run a replication (CLI only — no MCP run action) +sling run -r /path/to/replication.yaml -m incremental +sling run -r /path/to/replication.yaml --streams table1 --debug ``` ### Basic Replication Structure @@ -1443,7 +1440,7 @@ streams: #### Parse Configuration ```json { - "action": "parse", + "action": "validate", "input": { "file_path": "/path/to/replication.yaml", "working_dir": "/optional/work/dir" @@ -1452,9 +1449,10 @@ streams: ``` #### Validate Configuration +Parsing compiles by default, which checks connections and streams. Prefer this when connections exist. `compile: false` is parse-only and does not mean the file is ready to run. ```json { - "action": "compile", + "action": "validate", "input": { "file_path": "/path/to/replication.yaml", "select_streams": ["table1", "table2"], // Optional @@ -1464,21 +1462,14 @@ streams: ``` #### Execute Replication -```json -{ - "action": "run", - "input": { - "file_path": "/path/to/replication.yaml", - "select_streams": ["specific_table"], // Optional: run specific streams - "working_dir": "/project/directory", // Optional: change directory - "range": "2024-01-01,2024-01-31", // Optional: backfill range - "mode": "incremental", // Optional: override mode - "env": { // Optional: environment variables - "CUSTOM_VAR": "value", - "SLING_THREADS": "5" - } - } -} + +There is no MCP `run` action. Use the CLI: + +```bash +sling run -r /path/to/replication.yaml +sling run -r /path/to/replication.yaml --streams specific_table -m incremental +sling run -r /path/to/replication.yaml --range 2024-01-01,2024-01-31 +sling run -r /path/to/replication.yaml --env '{CUSTOM_VAR: value, SLING_THREADS: "5"}' ``` ### Combining with Connection Tools @@ -1502,23 +1493,18 @@ Then test connections: } ``` -Finally run replication: -```json -{ - "action": "run", - "input": { - "file_path": "/path/to/replication.yaml" - } -} +Finally run the replication with the CLI: + +```bash +sling run -r /path/to/replication.yaml ``` ### Development Workflow -1. **Parse** configuration for syntax validation -2. **Compile** to check connections and streams -3. **Test** on subset of streams first -4. **Run** full replication -5. **Monitor** with debug/trace options +1. **Validate** configuration with MCP `replication/validate` (compile is the default) +2. **Test** on a subset of streams first: `sling run -r file.yaml --streams small_table` +3. **Run** the full replication: `sling run -r file.yaml` +4. **Monitor** with `--debug` / `--trace` --- @@ -1648,14 +1634,8 @@ hooks: ### Testing Strategies #### Test with Subsets First -```json -{ - "action": "run", - "input": { - "file_path": "/path/to/replication.yaml", - "select_streams": ["small_test_table"] - } -} +```bash +sling run -r /path/to/replication.yaml --streams small_test_table ``` #### Use Development Targets @@ -1780,23 +1760,15 @@ target_options: ### Debug Options #### Enable Debug Logging -```json -{ - "action": "run", - "input": { - "file_path": "/path/to/replication.yaml", - "env": { - "DEBUG": "true" - } - } -} +```bash +sling run -r /path/to/replication.yaml --debug ``` #### Parse Configuration Always validate before running: ```json { - "action": "parse", + "action": "validate", "input": { "file_path": "/path/to/replication.yaml" } @@ -1804,10 +1776,10 @@ Always validate before running: ``` #### Compile Configuration -Check connections and streams: +Parsing compiles by default, which checks connections and streams. Prefer this when connections exist. ```json { - "action": "compile", + "action": "validate", "input": { "file_path": "/path/to/replication.yaml" } @@ -1889,14 +1861,8 @@ streams: Sling automatically tracks progress, but you can: 1. **Run specific streams:** -```json -{ - "action": "run", - "input": { - "file_path": "/path/to/replication.yaml", - "select_streams": ["failed_stream"] - } -} +```bash +sling run -r /path/to/replication.yaml --streams failed_stream ``` 2. **Use retry configuration:** @@ -1906,4 +1872,4 @@ env: SLING_RETRY_DELAY: 60s ``` -This comprehensive guide provides everything needed to effectively use Sling replications with the MCP tool, from basic concepts to advanced troubleshooting techniques. \ No newline at end of file +This comprehensive guide provides everything needed to effectively use Sling replications: validate with MCP, then execute with `sling run`. \ No newline at end of file diff --git a/cmd/sling/resource/mcp.yaml b/cmd/sling/resource/mcp.yaml index 46bde6044..2e4ad876c 100644 --- a/cmd/sling/resource/mcp.yaml +++ b/cmd/sling/resource/mcp.yaml @@ -322,12 +322,12 @@ tools: **Parameters**: - `action` (string, required): The API spec action to perform. Valid values: - - `"parse"` - Load and parse the API specification file, will return various details + - `"validate"` - Load and validate the API specification file, will return various details - `"docs"` - Fetch the Sling API specification documentation - `"test"` - Test an existing API Spec connection - `input` (object, required): The input parameters for the specific action. The structure varies based on the action: - **For `action: "parse"`**: + **For `action: "validate"`**: - `file_path` (string, required): The file path of the API specification. - `working_dir` (string, optional): Working directory to change to before running @@ -367,7 +367,7 @@ tools: # Parse API specification content { - "action": "parse", + "action": "validate", "input": { "file_path": "path/to/github_api.yaml", "working_dir": "/path/to/dir" @@ -383,31 +383,28 @@ tools: **Parameters**: - `action` (string, required): The replication action to perform. Valid values: - `"docs"` - Fetch documentation on how to use replications - - `"parse"` - Parse the content of a replication configuration file (non-compile validation) - - `"compile"` - Compile/validate a replication configuration - - `"run"` - Execute a replication configuration + - `"validate"` - Validate and compile a replication configuration file - `input` (object, required): The input parameters for the specific action. The structure varies based on the action: **For `action: "docs"`**: - No parameters required (empty object) - **For `action: "parse"`**: + **For `action: "validate"`**: - `file_path` (string, required): Path to the replication configuration file to read - - `working_dir` (string, optional): Working directory to change to before reading - - **For `action: "compile"`**: - - `file_path` (string, required): Path to the replication configuration file to compile + - `compile` (boolean, optional): Compile the configuration (default: true). Compile resolves + connections, interpolates `${VAR}` and `{var}` references, and returns the task list. + Set to false only for a parse-only syntax check. Parse-only does not mean the file is ready to run. + Prefer compile: true when connections exist. - `select_streams` (array, optional): List of specific streams to compile (default: all streams) - - `working_dir` (string, optional): Working directory to change to before compiling + - `working_dir` (string, optional): Working directory to change to before reading - **For `action: "run"`**: - - `file_path` (string, required): Path to the replication configuration file to execute - - `select_streams` (array, optional): List of specific streams to run (default: all streams) - - `working_dir` (string, optional): Working directory to change to before running - - `range` (string, optional): Backfill range for source options (e.g., "2024-01-01:2024-01-31") - - `mode` (string, optional): Mode override for replication (e.g., "full-refresh", "incremental") - - `env` (object, optional): Environment variables to set for the replication run + **To execute a replication, use the CLI** (there is no MCP `run` action): + ``` + sling run -r path/to/my_replication.yaml + sling run -r path/to/my_replication.yaml --streams table1,table2 -m incremental + sling run -r path/to/my_replication.yaml --debug + ``` **Output**: The output format depends on the action and audience: - **For User**: Human-readable messages, CSV tables, or status updates @@ -421,29 +418,11 @@ tools: "input": {} } - # Parse a replication file + # Parse and compile a replication file (default; prefer this when connections exist) { - "action": "parse", + "action": "validate", "input": { - "file_path": "path/to/my_replication.yaml", - } - } - - # Compile a replication file - { - "action": "compile", - "input": { - "file_path": "path/to/my_replication.yaml", - } - } - - # Run a replication with specific streams - { - "action": "run", - "input": { - "file_path": "path/to/my_replication.yaml", - "select_streams": ["table1", "table2"], - "mode": "incremental" + "file_path": "path/to/my_replication.yaml" } } ``` @@ -456,21 +435,28 @@ tools: **Parameters**: - `action` (string, required): The pipeline action to perform. Valid values: - `"docs"` - Fetch documentation on how to use pipelines - - `"parse"` - Parse and validate the content of a pipeline configuration file - - `"run"` - Execute a pipeline configuration + - `"validate"` - Validate the content of a pipeline configuration file - `input` (object, required): The input parameters for the specific action. The structure varies based on the action: **For `action: "docs"`**: - No parameters required (empty object) - **For `action: "parse"`**: + **For `action: "validate"`**: - `file_path` (string, required): Path to the pipeline configuration file to read. - `working_dir` (string, optional): Working directory to change to before reading - **For `action: "run"`**: - - `file_path` (string, required): Path to the pipeline configuration file to execute. - - `working_dir` (string, optional): Working directory to change to before running - - `env` (object, optional): Environment variables to set for the pipeline run. + **To execute a pipeline, use the CLI** (there is no MCP `run` action): + ``` + sling run -p path/to/my_pipeline.yaml + sling run -p path/to/my_pipeline.yaml --debug + ``` + + **To execute SQL models, use the CLI** (there is no MCP `build` action): + ``` + sling build + sling build --compile + sling build -s stg_users,fct_orders + ``` **Output**: The output format depends on the action and audience: - **For User**: Human-readable messages, CSV tables, or status updates @@ -486,30 +472,11 @@ tools: # Parse a pipeline file { - "action": "parse", - "input": { - "file_path": "path/to/my_pipeline.yaml" - } - } - - # Run a pipeline - { - "action": "run", + "action": "validate", "input": { "file_path": "path/to/my_pipeline.yaml" } } - - # Run a pipeline with environment variables - { - "action": "run", - "input": { - "file_path": "path/to/my_pipeline.yaml", - "env": { - "MY_VAR": "my_value" - } - } - } ``` @@ -547,7 +514,7 @@ prompts: The assistant should first attempt to use a browser mcp tool to fetch the documentation, as that will yield better content. If unable to use the browser mcp tool, the assistant should next attempt to directly fetch from the URL links (without a browser). - If the assistant is unable to obtain or fetch meaningful documentation from the URL links provided, stop and alert the user for further instruction. Recommend the user to download and activate the `browsermcp` (https://browsermcp.io/) which will provide tooling to access the browser. If the user wishes the assistant to proceed without having read the documentation, that is an option as well, although results are likely to be poor. + If the assistant is unable to obtain or fetch meaningful documentation from the URL links provided, stop and alert the user. Recommend `sling assist setup` so the agent-browser MCP server is wired. If the user wishes the assistant to proceed without having read the documentation, that is an option as well, although results are likely to be poor. 3. **Create Specification**: Now that the API is greatly understood, formulate and create a fully functional Sling API Spec in file path {spec_file_path}. If the assistant sees obstacles or issues in developing the spec, or does not have sufficient information, this is the time to alert the user with any concerns or questions for clarification. @@ -600,9 +567,9 @@ prompts: 1. **Get Documentation**: Call the tool `api_spec/docs` to obtain the latest Sling API specification guide - 2. **Load Existing Spec**: Use `api_spec/parse` to load the current specification content. + 2. **Load Existing Spec**: Use `api_spec/validate` to load the current specification content. - 3. **Read Raw Spec**: Examine the existing spec structure and patterns. If there were any parsing errors from the previous step, attempt to fix the structure of the file, and continue using `api_spec/parse` until there are no errors or determining that no progress can be made. + 3. **Read Raw Spec**: Examine the existing spec structure and patterns. If there were any parsing errors from the previous step, attempt to fix the structure of the file, and continue using `api_spec/validate` until there are no errors or determining that no progress can be made. 4. **Analyze Endpoint Documentation**: Navigate to the endpoint docs URL (if provided) or the main API docs, using the browser to fully understand: - Endpoint URL structure and parameters @@ -614,7 +581,7 @@ prompts: The assistant should first attempt to use a browser mcp tool to fetch the documentation, as that will yield better content. If unable to use the browser mcp tool, the assistant should next attempt to directly fetch from the URL links (without a browser). - If the assistant is unable to obtain or fetch meaningful documentation from the URL links provided, stop and alert the user for further instruction. Recommend the user to download and activate the `browsermcp` (https://browsermcp.io/) which will provide tooling to access the browser. If the user wishes the assistant to proceed without having read the documentation, that is an option as well, although results are likely to be poor. + If the assistant is unable to obtain or fetch meaningful documentation from the URL links provided, stop and alert the user. Recommend `sling assist setup` so the agent-browser MCP server is wired. If the user wishes the assistant to proceed without having read the documentation, that is an option as well, although results are likely to be poor. 5. **Implement Endpoint**: Add the new endpoint configuration to the spec file following existing patterns. @@ -654,9 +621,9 @@ prompts: ## Workflow Steps: 1. **Get Documentation**: Use `api_spec/docs` to access the Sling API specification guide - 2. **Load Existing Spec**: Use `api_spec/parse` to load the current specification content. + 2. **Load Existing Spec**: Use `api_spec/validate` to load the current specification content. - 3. **Read Raw Spec**: Examine the existing spec structure and patterns. If there were any parsing errors from the previous step, attempt to fix the structure of the file, and continue using `api_spec/parse` until there are no errors or determining that no progress can be made. + 3. **Read Raw Spec**: Examine the existing spec structure and patterns. If there were any parsing errors from the previous step, attempt to fix the structure of the file, and continue using `api_spec/validate` until there are no errors or determining that no progress can be made. 4. **Analyze API Documentation**: Navigate to the API docs using the browser to fully understand and verify: - Correct endpoint URL and parameters @@ -667,7 +634,7 @@ prompts: The assistant should first attempt to use a browser mcp tool to fetch the documentation, as that will yield better content. If unable to use the browser mcp tool, the assistant should next attempt to directly fetch from the URL links (without a browser). - If the assistant is unable to obtain or fetch meaningful documentation from the URL links provided, stop and alert the user for further instruction. Recommend the user to download and activate the `browsermcp` (https://browsermcp.io/) which will provide tooling to access the browser. If the user wishes the assistant to proceed without having read the documentation, that is an option as well, although results are likely to be poor. + If the assistant is unable to obtain or fetch meaningful documentation from the URL links provided, stop and alert the user. Recommend `sling assist setup` so the agent-browser MCP server is wired. If the user wishes the assistant to proceed without having read the documentation, that is an option as well, although results are likely to be poor. 5. **Test and Diagnose**: Use `api_spec/test` with debug enabled, specifying the endpoint name - Examine error messages and response details diff --git a/cmd/sling/sling_assist.go b/cmd/sling/sling_assist.go new file mode 100644 index 000000000..dd5b61ce5 --- /dev/null +++ b/cmd/sling/sling_assist.go @@ -0,0 +1,536 @@ +package main + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + + "github.com/flarco/g" + "github.com/integrii/flaggy" + "github.com/slingdata-io/sling-cli/core/env" + "github.com/slingdata-io/sling-cli/core/sling/assist" + "github.com/spf13/cast" +) + +var cliAssistFlags = []g.Flag{ + {Name: "id", Type: "string", Description: "investigate a failure by its id (an id prefix works)"}, + {Name: "out", ShortName: "o", Type: "string", Description: "emit prompt instead of launching: FILE, or - for stdout"}, + {Name: "agent", Type: "string", Description: "override profile.agent for this run only"}, + {Name: "model", Type: "string", Description: "pass --model to the harness for this run only"}, + {Name: "name", Type: "string", Description: "history slug override"}, + {Name: "non-interactive", Type: "bool", Description: "skip interactive prompts (use defaults / launch headless)"}, + {Name: "resume", Type: "string", Description: "resume a past session (omit id to pick)"}, +} + +var cliAssistSetupFlags = []g.Flag{ + {Name: "doctor", Type: "bool", Description: "print the health/install report and exit"}, + {Name: "agent", Type: "string", Description: "preferred agent for this setup"}, + {Name: "non-interactive", Type: "bool", Description: "skip interactive prompts (use defaults)"}, + {Name: "install", Type: "string", Description: "comma-separated components (e.g. mcp,skills)"}, + {Name: "uninstall", Type: "bool", Description: "remove all Sling skills + MCP wiring"}, + {Name: "reconfigure", Type: "bool", Description: "re-prompt the profile form before installing"}, + {Name: "scope", Type: "string", Description: "user (default) or project"}, + {Name: "clients", Type: "string", Description: "with --uninstall: comma-separated client names"}, +} + +// cliAssist is the `sling assist` top-level command. +var cliAssist = &g.CliSC{ + Name: "assist", + Description: "Get AI help to build, run and debug replications, pipelines, API Specs, etc.", + AdditionalHelpPrepend: "\n" + + " sling assist # first run: setup; after: greet with choices\n" + + " sling assist \"backfill orders\" # positional ask\n" + + " sling assist --id # investigate a failure\n sling assist --resume # pick a past session\n" + + " sling assist --resume # resume that session\n" + + " sling assist setup # re-run setup / change harness\n" + + " sling assist setup --doctor # report only\n" + + " sling assist error # look up an error signature\n" + + " sling assist report --id # review a redacted failure report\n" + + " sling assist report --id --github # open a prefilled GitHub issue\n" + + " sling assist report --id --email # send to support (confirm first)\n" + + " sling assist --out - | --out F # emit prompt, no launch\n" + + " sling assist --agent claude # one-run override\n" + + " sling assist --model sonnet # pass --model to the harness\n" + + "See https://docs.slingdata.io/sling-cli/assist", + ExecuteWithoutFlags: true, + Flags: cliAssistFlags, + SubComs: []*g.CliSC{ + { + Name: "setup", + Description: "install skills + MCP, or reconfigure / uninstall", + ExecuteWithoutFlags: true, + Flags: cliAssistSetupFlags, + }, + { + Name: "error", + Description: "look up guidance for an error signature", + Flags: []g.Flag{ + {Name: "json", Type: "bool", Description: "emit machine-readable JSON"}, + }, + PosFlags: []g.Flag{ + {Name: "signature", Type: "string", Description: "8-char error signature from a failed run"}, + }, + }, + { + Name: "report", + Description: "compose a redacted issue report from a failed run", + Flags: []g.Flag{ + {Name: "id", Type: "string", Description: "failure id (a unique prefix works)"}, + {Name: "title", Type: "string", Description: "override the report title"}, + {Name: "description", Type: "string", Description: "custom context shown above the error in the report"}, + {Name: "github", Type: "bool", Description: "open a prefilled GitHub issue after confirm"}, + {Name: "email", Type: "bool", Description: "send to support after confirm"}, + {Name: "submit", Type: "bool", Description: "skip the confirm prompt (for agents)"}, + }, + }, + }, + ExecProcess: processAssist, +} + +func init() { + cliAssist.Make().Add() +} + +// processAssist is the dispatcher. Auto-refresh runs once at the top so every +// subcommand below sees a fresh canonical bundle. If skills are already +// installed, AutoRefresh updates and prunes them. If none are installed, +// the user must run `sling assist setup`. +func processAssist(c *g.CliSC) (ok bool, err error) { + ok = true + if notice, refreshErr := assist.AutoRefresh(context.Background()); refreshErr == nil && notice != "" { + fmt.Fprintln(os.Stderr, notice) + } + + switch c.UsedSC() { + case "setup": + return ok, runAssistSetup(c) + case "error": + return ok, runAssistError(c) + case "report": + return ok, runAssistReport(c) + } + return ok, runAssistFlags(c) +} + +// runAssistFlags is the flags-only path: --resume, else first-run +// setup or probe+launch / --out. +func runAssistFlags(c *g.CliSC) error { + vals := flatVals(c) + + resumeSet, resumeID := resumeFromArgs(os.Args) + if resumeSet { + return runAssistResume(c, resumeID) + } + + _, profileExists, err := assist.LoadProfile() + if err != nil { + return err + } + if !profileExists && strings.TrimSpace(cast.ToString(vals["out"])) == "" { + return runAssistSetup(c) + } + return runAssistSession(c) +} + +func runAssistSession(c *g.CliSC) error { + vals := flatVals(c) + // Trailing args are the ask (flaggy cannot share PosFlags with SubComs). + ask := "" + if len(flaggy.TrailingArguments) > 0 { + ask = strings.TrimSpace(strings.Join(flaggy.TrailingArguments, " ")) + } + opts := assist.SessionOptions{ + Ask: ask, + ExecID: strings.TrimSpace(cast.ToString(vals["id"])), + Name: cast.ToString(vals["name"]), + Agent: cast.ToString(vals["agent"]), + Model: cast.ToString(vals["model"]), + Headless: cast.ToBool(vals["non-interactive"]), + } + applyAssistOut(&opts, cast.ToString(vals["out"])) + _, err := assist.Session(opts) + if code, ok := assist.ExitCodeOf(err); ok { + os.Exit(code) + } + return err +} + +// applyAssistOut maps --out onto the session emit options. +// "-" means stdout; any other value is a file path. +func applyAssistOut(opts *assist.SessionOptions, out string) { + out = strings.TrimSpace(out) + switch out { + case "": + case "-": + opts.Print = true + default: + opts.OutputFile = out + } +} + +func runAssistResume(c *g.CliSC, id string) error { + vals := flatVals(c) + if id == "" { + e, err := assist.PickHistoryEntry() + if err != nil { + if errors.Is(err, assist.ErrUserAborted) { + return nil + } + return err + } + id = e.ID + } + opts := assist.SessionOptions{ + ResumeID: id, + ResumeSet: true, + Agent: cast.ToString(vals["agent"]), + Model: cast.ToString(vals["model"]), + Headless: cast.ToBool(vals["non-interactive"]), + } + applyAssistOut(&opts, cast.ToString(vals["out"])) + _, err := assist.Session(opts) + if code, ok := assist.ExitCodeOf(err); ok { + os.Exit(code) + } + return err +} + +// padAssistResumeFlag lets flaggy accept a bare `--resume` (picker) as `--resume=`. +func padAssistResumeFlag(args []string) []string { + assist := false + for _, a := range args[1:] { + if a == "assist" { + assist = true + break + } + if strings.HasPrefix(a, "-") { + continue + } + break + } + if !assist { + return args + } + out := make([]string, 0, len(args)+1) + for i := 0; i < len(args); i++ { + a := args[i] + if a == "--resume" { + if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") { + out = append(out, "--resume=") + continue + } + } + out = append(out, a) + } + return out +} + +func resumeFromArgs(args []string) (present bool, id string) { + for i := 1; i < len(args); i++ { + a := args[i] + if a == "--resume" { + if i+1 < len(args) && args[i+1] != "" && !strings.HasPrefix(args[i+1], "-") { + return true, args[i+1] + } + return true, "" + } + if strings.HasPrefix(a, "--resume=") { + return true, strings.TrimPrefix(a, "--resume=") + } + } + return false, "" +} + +func parseScope(v string) assist.Scope { + if strings.EqualFold(v, "project") { + return assist.ScopeProject + } + return assist.ScopeUser +} + +// runAssistSetup is the unified setup entry point. One command does everything +// install/doctor/uninstall used to do — doctor runs implicitly each call to +// detect current state, then we branch by flag intent or interactive choice. +func runAssistSetup(c *g.CliSC) error { + vals := flatVals(c) + + // --uninstall is the only path that skips doctor (we're tearing it all + // down anyway; running doctor first would just be noise). + if cast.ToBool(vals["uninstall"]) { + return runSetupUninstallAll(vals) + } + + // Doctor always runs first; we use its result both to render state for + // the user and to decide what the implicit "next action" should be. + // Scope must match install scope so project installs aren't reported broken. + report, err := assist.Doctor(context.Background(), assist.DoctorOptions{ + Scope: parseScope(cast.ToString(vals["scope"])), + }) + if err != nil { + return err + } + _, profileExists, _ := assist.LoadProfile() + + // --doctor: print and exit. + if cast.ToBool(vals["doctor"]) { + fmt.Fprint(os.Stdout, report.Render()) + if !report.OK { + return g.Error("doctor reported failures") + } + return nil + } + + // --install : skip the picker, install whatever was named. + if comps := cast.ToString(vals["install"]); comps != "" { + // Split "mcp,skills" into lower-case names. + var parts []string + for _, p := range strings.Split(comps, ",") { + p = strings.TrimSpace(strings.ToLower(p)) + if p != "" { + parts = append(parts, p) + } + } + return runSetupInstall(vals, parts, profileExists, report) + } + + // Non-interactive without an explicit verb → behave like first-run install + // of all components (back-compat with how the old `install --non-interactive` + // behaved). + if cast.ToBool(vals["non-interactive"]) { + return runSetupInstall(vals, allComponents(), profileExists, report) + } + + // Subsequent run (profile exists): show doctor output, then drop into the + // interactive "what do you want to do?" form. + if profileExists { + fmt.Fprint(os.Stdout, report.Render()) + fmt.Fprintln(os.Stdout, "") + action, err := assist.RunSetupActionForm(report) + if err != nil { + return err + } + switch action { + case assist.SetupActionRefresh: + return runSetupInstall(vals, allComponents(), profileExists, report) + case assist.SetupActionInstallMissing: + comps := report.MissingComponents() + if len(comps) == 0 { + comps = allComponents() + } + return runSetupInstall(vals, comps, profileExists, report) + case assist.SetupActionUninstall: + return runSetupUninstallAll(vals) + case assist.SetupActionReconfigure: + vals["reconfigure"] = true + return runSetupInstall(vals, allComponents(), profileExists, report) + case assist.SetupActionExit: + return nil + } + return nil + } + + // First run (no profile): harness confirm (agents + bundled opencode fallback). + if len(assist.DetectedClients()) == 0 && len(assist.RankedCLIAgents()) == 0 { + return g.Error("no AI agent on $PATH; install one of: claude, codex, gemini, cursor, opencode, pi, grok") + } + prefill := assist.DefaultProfile() + prefill.Agent = assist.RecommendedAgent() + result, err := assist.RunHarnessConfirmForm(prefill) + if err != nil { + if errors.Is(err, assist.ErrUserAborted) { + return nil + } + return err + } + prefill.Agent = result.Agent + prefill.HintInErrors = result.HintInErrors + prefill.DefaultInstallScope = result.Scope + if err := assist.SaveProfile(prefill); err != nil { + return err + } + vals["agent"] = result.Agent + vals["scope"] = result.Scope + return runSetupInstall(vals, result.Components, true, report) +} + +// runSetupInstall runs the profile form (if needed) then installs the +// requested components. components is the canonical set; we translate it +// to SkillsOnly/MCPOnly for the existing Install API. +func runSetupInstall(vals map[string]any, components []string, _ bool, _ *assist.DoctorReport) error { + opts := assist.InstallOptions{ + Reconfigure: cast.ToBool(vals["reconfigure"]), + Scope: parseScope(cast.ToString(vals["scope"])), + NonInteractive: cast.ToBool(vals["non-interactive"]), + DefaultAgent: cast.ToString(vals["agent"]), + } + hasSkills := containsString(components, "skills") + hasMCP := containsString(components, "mcp") + if !hasSkills && !hasMCP { + return g.Error("no components selected; pass --install mcp,skills or pick at least one in the form") + } + opts.SkillsOnly = hasSkills && !hasMCP + opts.MCPOnly = hasMCP && !hasSkills + + // Interactive first-run (or --reconfigure): show the profile form before + // falling through to Install(). + if !opts.NonInteractive { + prof, exists, _ := assist.LoadProfile() + needForm := !exists || opts.Reconfigure + if needForm { + if len(assist.DetectedClients()) == 0 && len(assist.RankedCLIAgents()) == 0 { + return g.Error("no AI agent on $PATH; install one of: claude, codex, gemini, cursor, opencode, pi, grok") + } + prefill := prof + if !exists { + prefill = assist.DefaultProfile() + prefill.Agent = assist.RecommendedAgent() + } + result, err := assist.RunInstallForm(prefill) + if err != nil { + if errors.Is(err, assist.ErrUserAborted) { + return nil + } + return err + } + prefill.Agent = result.Agent + prefill.HintInErrors = result.HintInErrors + prefill.DefaultInstallScope = result.Scope + if err := assist.SaveProfile(prefill); err != nil { + return err + } + opts.Reconfigure = false + opts.DefaultAgent = "" + opts.Scope = parseScope(result.Scope) + } + } + + res, err := assist.Install(context.Background(), opts) + if err != nil { + return err + } + // Install summary: profile, canonical skills, then each wired client. + if res.ProfileWritten { + fmt.Fprintf(os.Stdout, "%s Wrote AI profile to %s\n", + env.GreenString("✓"), env.CyanString(env.HomeDirEnvFile)) + } + fmt.Fprintf(os.Stdout, "%s Wrote canonical skills to %s\n", + env.GreenString("✓"), env.CyanString(res.CanonicalSkillsDir)) + fmt.Fprintln(os.Stdout, "") + fmt.Fprintln(os.Stdout, env.BlueString("Wired clients:")) + yesNo := func(v bool) string { + if v { + return env.GreenString("yes") + } + return env.DarkGrayString("no") + } + for _, row := range res.WiredClients { + mark := env.GreenString("✓") + if !row.WroteSkills && !row.WroteMCP { + mark = env.YellowString("⊘") + } + auth := row.Authed.YesNo() + authOut := env.DarkGrayString(auth) + if row.Authed == assist.AuthOK { + authOut = env.GreenString(auth) + } + fmt.Fprintf(os.Stdout, " %s %-8s skills=%s mcp=%s authed=%s %s\n", + mark, row.Name, yesNo(row.WroteSkills), yesNo(row.WroteMCP), authOut, + env.DarkGrayString(row.Notes)) + } + fmt.Fprintln(os.Stdout, "") + fmt.Fprintf(os.Stdout, "Run %s to verify.\n", env.CyanString("`sling assist setup --doctor`")) + return nil +} + +// runSetupUninstallAll wipes everything (skills + mcp from every detected +// client + the canonical bundle). No interactive form — `--uninstall` is the +// blunt instrument; per-client/per-component selection isn't worth a separate +// surface. +func runSetupUninstallAll(vals map[string]any) error { + opts := assist.UninstallOptions{ + All: true, + Scope: parseScope(cast.ToString(vals["scope"])), + } + if v := cast.ToString(vals["clients"]); v != "" { + opts.IncludeClients = strings.Split(v, ",") + } + if err := assist.Uninstall(context.Background(), opts); err != nil { + return err + } + fmt.Fprintln(os.Stdout, env.GreenString("✓ Removed Sling skills + MCP wiring from detected clients.")) + return nil +} + +func allComponents() []string { return []string{"skills", "mcp"} } + +func containsString(xs []string, x string) bool { + for _, v := range xs { + if v == x { + return true + } + } + return false +} + +func runAssistReport(c *g.CliSC) error { + vals := flatVals(c) + id := strings.TrimSpace(cast.ToString(vals["id"])) + if id == "" { + return g.Error("usage: sling assist report --id ") + } + return assist.RunReport(assist.ReportCmd{ + ExecID: id, + Title: strings.TrimSpace(cast.ToString(vals["title"])), + Description: strings.TrimSpace(cast.ToString(vals["description"])), + GitHub: cast.ToBool(vals["github"]), + Email: cast.ToBool(vals["email"]), + Submit: cast.ToBool(vals["submit"]), + }) +} + +func runAssistError(c *g.CliSC) error { + vals := flatVals(c) + sig := strings.TrimSpace(cast.ToString(vals["signature"])) + if sig == "" { + return g.Error("usage: sling assist error ") + } + result, err := assist.LookupError(sig) + if err != nil { + return err + } + if cast.ToBool(vals["json"]) { + fmt.Println(g.Marshal(result)) + return nil + } + fmt.Printf("error_signature: %s\n", result.Signature) + fmt.Printf("status: %s\n", result.Status) + if result.Title != "" { + fmt.Printf("title: %s\n", result.Title) + } + if result.Guidance != "" { + fmt.Println() + fmt.Println(result.Guidance) + } + if result.DocsURL != "" { + fmt.Printf("\nDocs: %s\n", result.DocsURL) + } + fmt.Println() + fmt.Println("Next steps:") + fmt.Printf(" sling assist # open assist (offers to investigate failures)\n") + if result.Status == "unknown" || result.Status == "pending" { + fmt.Printf(" sling assist report --id # share a redacted report\n") + } + return nil +} + +// flatVals returns the val map from the active subcommand. CliSC stores per- +// subcommand flag values in c.Vals; we just pass that through. +func flatVals(c *g.CliSC) map[string]any { + out := map[string]any{} + for k, v := range c.Vals { + out[k] = v + } + return out +} diff --git a/cmd/sling/sling_assist_test.go b/cmd/sling/sling_assist_test.go new file mode 100644 index 000000000..5542fa1f3 --- /dev/null +++ b/cmd/sling/sling_assist_test.go @@ -0,0 +1,243 @@ +package main + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/flarco/g" + "github.com/slingdata-io/sling-cli/core/dbio" + "github.com/slingdata-io/sling-cli/core/dbio/connection" +) + +func TestAssistBrowseFlagRemoved(t *testing.T) { + for _, f := range cliAssistFlags { + if f.Name == "browse" || f.Name == "cdp" || f.Name == "setup" || f.Name == "doctor" || f.Name == "task" || f.Name == "path" { + t.Fatalf("removed flag still present: %s", f.Name) + } + } + foundResume, foundModel := false, false + for _, f := range cliAssistFlags { + if f.Name == "resume" { + foundResume = true + } + if f.Name == "model" { + foundModel = true + } + } + if !foundResume || !foundModel { + t.Fatal("missing --resume or --model on sling assist") + } + hasSetup, setupHasDoctor := false, false + for _, sc := range cliAssist.SubComs { + if sc.Name == "setup" { + hasSetup = true + for _, f := range sc.Flags { + if f.Name == "doctor" { + setupHasDoctor = true + } + } + } + if sc.Name == "investigate" || sc.Name == "history" { + t.Fatalf("removed subcommand still present: %s", sc.Name) + } + } + hasReport := false + for _, sc := range cliAssist.SubComs { + if sc.Name == "report" { + hasReport = true + } + } + if !hasSetup { + t.Fatal("missing setup subcommand") + } + if !hasReport { + t.Fatal("missing report subcommand") + } + if !setupHasDoctor { + t.Fatal("missing --doctor on sling assist setup") + } +} + +func TestPadAssistResumeFlag(t *testing.T) { + got := padAssistResumeFlag([]string{"sling", "assist", "--resume"}) + if len(got) < 3 || got[2] != "--resume=" { + t.Fatalf("bare resume: %v", got) + } + got = padAssistResumeFlag([]string{"sling", "assist", "--resume", "--out"}) + if got[2] != "--resume=" { + t.Fatalf("resume before flag: %v", got) + } + got = padAssistResumeFlag([]string{"sling", "assist", "--resume", "sess1"}) + if got[2] != "--resume" || got[3] != "sess1" { + t.Fatalf("resume with id: %v", got) + } +} + +func TestResumeFromArgs(t *testing.T) { + ok, id := resumeFromArgs([]string{"sling", "assist", "--resume="}) + if !ok || id != "" { + t.Fatalf("empty resume: %v %q", ok, id) + } + ok, id = resumeFromArgs([]string{"sling", "assist", "--resume", "abc"}) + if !ok || id != "abc" { + t.Fatalf("id: %v %q", ok, id) + } + ok, _ = resumeFromArgs([]string{"sling", "assist", "--out"}) + if ok { + t.Fatal("resume not set") + } +} + +// parseKVList is kept in this test file so TestCLI still compiles after +// the production helper was removed in the assist redesign. +func parseKVList(s string) map[string]string { + out := map[string]string{} + for _, pair := range splitKVPairs(s) { + kv := strings.SplitN(pair, "=", 2) + if len(kv) != 2 { + continue + } + k := strings.TrimSpace(kv[0]) + v := unquoteKV(strings.TrimSpace(kv[1])) + if strings.HasPrefix(v, "@") && !strings.HasPrefix(v, "@@") { + if b, err := os.ReadFile(v[1:]); err == nil { + v = string(b) + } + } + out[k] = v + } + return out +} + +func splitKVPairs(s string) []string { + var parts []string + var b strings.Builder + var quote byte + for i := 0; i < len(s); i++ { + c := s[i] + if quote != 0 { + if c == '\\' && i+1 < len(s) { + b.WriteByte(c) + i++ + b.WriteByte(s[i]) + continue + } + if c == quote { + quote = 0 + } + b.WriteByte(c) + continue + } + if c == '"' || c == '\'' { + quote = c + b.WriteByte(c) + continue + } + if c == ',' { + parts = append(parts, b.String()) + b.Reset() + continue + } + b.WriteByte(c) + } + if b.Len() > 0 { + parts = append(parts, b.String()) + } + return parts +} + +func unquoteKV(v string) string { + if len(v) < 2 { + return v + } + if (v[0] == '"' && v[len(v)-1] == '"') || (v[0] == '\'' && v[len(v)-1] == '\'') { + inner := v[1 : len(v)-1] + inner = strings.ReplaceAll(inner, `\"`, `"`) + inner = strings.ReplaceAll(inner, `\'`, `'`) + return inner + } + return v +} + +func TestParseKVListQuotedComma(t *testing.T) { + got := parseKVList(`Intention="Select id, email, name. Range 2024-01-01,2024-12-31",Path=./out.yaml`) + if got["Intention"] != "Select id, email, name. Range 2024-01-01,2024-12-31" { + t.Fatalf("Intention=%q", got["Intention"]) + } + if got["Path"] != "./out.yaml" { + t.Fatalf("Path=%q", got["Path"]) + } +} + +func TestParseKVListAtFile(t *testing.T) { + dir := t.TempDir() + p := filepath.Join(dir, "intention.txt") + body := "Load fixtures/data/*.csv and range 2024-01-01,2024-12-31" + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + got := parseKVList("Intention=@" + p + ",Path=./out.yaml") + if got["Intention"] != body { + t.Fatalf("Intention=%q", got["Intention"]) + } + if got["Path"] != "./out.yaml" { + t.Fatalf("Path=%q", got["Path"]) + } +} + +func TestParseKVListUnquotedStillSplits(t *testing.T) { + got := parseKVList("A=1,B=2") + if got["A"] != "1" || got["B"] != "2" { + t.Fatalf("%v", got) + } +} + +func TestOverlaySpecConn(t *testing.T) { + dir := t.TempDir() + specPath := filepath.Join(dir, "draft.yaml") + if err := os.WriteFile(specPath, []byte("name: draft\n"), 0o644); err != nil { + t.Fatal(err) + } + + apiConn, err := connection.NewConnection("MY_API", dbio.TypeApi, g.M("type", "api", "spec", "baseline")) + if err != nil { + t.Fatal(err) + } + otherConn, err := connection.NewConnection("LOCAL", dbio.TypeFileLocal, g.M("type", "file")) + if err != nil { + t.Fatal(err) + } + entries := connection.ConnEntries{ + {Name: "MY_API", Connection: apiConn}, + {Name: "LOCAL", Connection: otherConn}, + } + + out, err := overlaySpecConn(entries, "MY_API", "draft.yaml", dir) + if err != nil { + t.Fatal(err) + } + + got := out.Get("MY_API").Connection.Data["spec"] + want := "file://" + specPath + if got != want { + t.Fatalf("overlay spec=%q want %q", got, want) + } + // original entries stay untouched + if entries.Get("MY_API").Connection.Data["spec"] != "baseline" { + t.Fatalf("original entry was mutated") + } + if out.Get("LOCAL").Connection.Data["spec"] != nil { + t.Fatalf("unrelated entry changed") + } + + // missing file errors + if _, err := overlaySpecConn(entries, "MY_API", "nope.yaml", dir); err == nil { + t.Fatal("expected error for missing spec file") + } + // unknown connection errors + if _, err := overlaySpecConn(entries, "NOPE", specPath, dir); err == nil { + t.Fatal("expected error for unknown connection") + } +} diff --git a/cmd/sling/sling_cli.go b/cmd/sling/sling_cli.go index 720979ad8..6a812dcd2 100755 --- a/cmd/sling/sling_cli.go +++ b/cmd/sling/sling_cli.go @@ -17,6 +17,7 @@ import ( "github.com/slingdata-io/sling-cli/core" "github.com/slingdata-io/sling-cli/core/env" "github.com/slingdata-io/sling-cli/core/sling" + "github.com/slingdata-io/sling-cli/core/sling/assist" "github.com/slingdata-io/sling-cli/core/store" "github.com/flarco/g" @@ -60,6 +61,12 @@ var cliRunFlags = []g.Flag{ Type: "string", Description: "The directory path file to use to run nested replications/pipelines.\n", }, + { + Name: "job", + ShortName: "j", + Type: "string", + Description: "The job key from the project manifest (sling_project.yml) to run locally.\n", + }, { Name: "src-conn", ShortName: "", @@ -216,12 +223,6 @@ var cliRun = &g.CliSC{ ExecProcess: processRun, } -var cliInteractive = &g.CliSC{ - Name: "it", - Description: "launch interactive mode", - ExecProcess: slingPrompt, -} - var cliUpdate = &g.CliSC{ Name: "update", Description: "Update Sling to the latest version", @@ -367,10 +368,26 @@ var cliConns = &g.CliSC{ Name: "key=value properties...", ShortName: "", Type: "string", - Description: "The key=value properties to set. See https://docs.slingdata.io/sling-cli/environment#set-connections", + Description: "The key=value properties to set. Secret fields omitted with --type are written as ${NAME_KEY} refs. See https://docs.slingdata.io/sling-cli/environment#set-connections", }, }, Flags: []g.Flag{ + { + Name: "type", + Type: "string", + Description: "Connection type (postgres, s3, api, ...). Use ${NAME_KEY} refs for secrets.", + }, + { + Name: "output", + ShortName: "o", + Type: "string", + Description: "Output format: text (default), json. Overrides SLING_OUTPUT.", + }, + { + Name: "stdin", + Type: "bool", + Description: "Read a YAML or JSON property map from stdin.", + }, { Name: "home-dir", Type: "string", @@ -559,6 +576,27 @@ func main() { os.Exit(exitCode) } +// startAssistLogCapture buffers the run log tail for the assist failure +// snapshot. Only for commands that write one (see writeFailure below). +// Reads os.Args: g.CliObj is not populated until g.CliProcess runs. +func startAssistLogCapture() { + cmd, sub := "", "" + for _, a := range os.Args[1:] { + if strings.HasPrefix(a, "-") { + continue + } + if cmd == "" { + cmd = a + continue + } + sub = a + break + } + if cmd == "run" || (cmd == "conns" && g.In(sub, "test", "discover")) { + env.StartLogCapture() + } +} + func cliInit(done chan struct{}) int { defer close(done) @@ -592,12 +630,20 @@ func cliInit(done chan struct{}) int { os.Args = []string{os.Args[0], "runner"} case len(os.Args) > 2 && os.Args[1] == "agent": os.Args = append([]string{os.Args[0], "runner"}, os.Args[2:]...) + // 'sling project' into 'sling platform' + // case len(os.Args) == 2 && os.Args[1] == "project": + // os.Args = []string{os.Args[0], "platform"} + // case len(os.Args) > 2 && os.Args[1] == "project": + // os.Args = append([]string{os.Args[0], "platform"}, os.Args[2:]...) } + os.Args = padAssistResumeFlag(os.Args) + flaggy.ShowHelpOnUnexpectedDisable() flaggy.Parse() setSentry() + startAssistLogCapture() ok, err := g.CliProcess() if err != nil || env.TelMap["error"] != nil { @@ -615,9 +661,39 @@ func cliInit(done chan struct{}) int { Track(eventName) } - g.PrintFatal(err) + // print main error + env.PrintFatal(err) + + // append failure hint + if g.CliObj != nil && (g.In(g.CliObj.Name, "run", "build") || g.In(g.CliObj.UsedSC(), "test", "discover")) { + errMsg := getErrString(err) + snap := assist.FailureSnapshot{ + ExecID: env.ExecID, + ErrMsg: errMsg, + RunLog: env.RecentLogs(), + SignMeta: assist.MakeSignMeta(), + } + if g.CliObj != nil && g.CliObj.Name == "conns" { + snap.ConnName = cast.ToString(g.CliObj.Vals["name"]) + } else if g.CliObj != nil { + for _, key := range []string{"replication", "pipeline", "path"} { + if v, ok := g.CliObj.Vals[key]; ok { + snap.ConfigPath = cast.ToString(v) + break + } + } + } + + assist.WriteFailureSnapshot(snap) + assist.PrintFailureFooter(assist.FailureFooterOpts{ + ExecID: env.ExecID, + ErrMsg: errMsg, + SignMeta: snap.SignMeta, + }) + } return 1 } else if !ok { + // Always print classic help. flaggy.ShowHelp("") } @@ -631,15 +707,28 @@ func cliInit(done chan struct{}) int { return 0 } -func getErrString(err error) (errString string) { - if err != nil { - errString = err.Error() - E, ok := err.(*g.ErrType) - if ok && E.Debug() != "" { - errString = E.Debug() +func getErrString(err error) string { + if err == nil { + return "" + } + if eg, ok := err.(*g.ErrorGroup); ok { + parts := make([]string, 0, len(eg.Errors)) + for i, child := range eg.Errors { + s := getErrString(child) + if s == "" { + continue + } + if i < len(eg.Names) && eg.Names[i] != "" { + s = g.F("--------------------------- %s ---------------------------\n%s", eg.Names[i], s) + } + parts = append(parts, s) } + return strings.Join(parts, "\n") + } + if d := g.ErrMsgDebug(err); d != "" { + return d } - return + return err.Error() } func setSentry() { diff --git a/cmd/sling/sling_conns.go b/cmd/sling/sling_conns.go index 6a1d46686..ce580eacb 100644 --- a/cmd/sling/sling_conns.go +++ b/cmd/sling/sling_conns.go @@ -4,6 +4,7 @@ import ( "encoding/csv" "encoding/hex" "fmt" + "io" "os" "strings" "time" @@ -69,24 +70,72 @@ func processConns(c *g.CliSC) (ok bool, err error) { } g.Info("connection `%s` has been removed from %s", name, ec.EnvFile.Path) case "set": - if len(c.Vals) == 0 { + name := strings.ToUpper(cast.ToString(c.Vals["name"])) + if name == "" { flaggy.ShowHelp("") return ok, nil } + setOutput, outErr := ResolveOutputFormat(c, "json") + if outErr != nil { + return ok, outErr + } + asJSON = setOutput == "json" + + kvMap := map[string]any{} + if cast.ToBool(c.Vals["stdin"]) { + stat, _ := os.Stdin.Stat() + if stat != nil && stat.Mode()&os.ModeCharDevice != 0 { + return ok, g.Error("stdin is a terminal; pipe a YAML or JSON property map") + } + raw, readErr := io.ReadAll(os.Stdin) + if readErr != nil { + return ok, g.Error(readErr, "could not read stdin") + } + stdinMap, parseErr := connection.ParsePropsInput(string(raw)) + if parseErr != nil { + return ok, parseErr + } + kvMap = stdinMap + } + kvArr := []string{cast.ToString(c.Vals["value properties..."])} - kvMap := map[string]interface{}{} for k, v := range g.KVArrToMap(append(kvArr, flaggy.TrailingArguments...)...) { k = strings.ToLower(k) + if k == "" { + continue + } kvMap[k] = v } - name := strings.ToUpper(cast.ToString(c.Vals["name"])) + if t := strings.TrimSpace(cast.ToString(c.Vals["type"])); t != "" { + kvMap["type"] = strings.ToLower(t) + } + + if err = connection.RejectLiteralSecrets(name, kvMap); err != nil { + return ok, err + } - err := ec.Set(name, kvMap) + err = ec.Set(name, kvMap) if err != nil { return ok, g.Error(err, "could not set %s (See https://docs.slingdata.io/sling-cli/environment)", name) } - g.Info("connection `%s` has been set in %s. Please test with `sling conns test %s`", name, ec.EnvFile.Path, name) + + loc, locErr := ec.EnvFile.LookupConnection(name) + if locErr != nil { + loc = env.ConnLocation{Path: ec.EnvFile.Path, Connection: name, Missing: []env.MissingRef{}} + } + + if asJSON { + fmt.Println(g.Marshal(loc)) + return ok, nil + } + + g.Info("connection `%s` has been set in %s:%d", name, loc.Path, loc.Line) + if len(loc.Missing) > 0 { + g.Info("set the env var(s), then: sling conns test %s", name) + } else { + g.Info("next: sling conns test %s", name) + } case "exec": env.SetTelVal("task", g.Marshal(g.M("type", sling.ConnExec))) @@ -305,9 +354,29 @@ func processConns(c *g.CliSC) (ok bool, err error) { env.SetTelVal("task", g.Marshal(g.M("type", sling.ConnTest))) name := cast.ToString(c.Vals["name"]) - if conn := entries.Get(name); conn.Name != "" { + conn := entries.Get(name) + if conn.Name != "" { env.SetTelVal("conn_type", conn.Connection.Type.String()) env.SetTelVal("conn_keys", lo.Keys(conn.Connection.Data)) + if g.IsDebugLow() { + g.Debug("connection %s properties: %s", name, g.Marshal(conn.Connection.Data)) + } + } + + refData := conn.Connection.Data + if len(refData) == 0 { + if cdata, ok := ef.Connections[strings.ToUpper(name)]; ok { + refData = cdata + } + } + if refs := connection.FindUnsetEnvRefs(refData); len(refs) > 0 { + loc, _ := ef.LookupConnection(strings.ToUpper(name)) + err = connection.FormatUnsetRefError(refs, loc) + if os.Getenv("SLING_OUTPUT") == "json" { + fmt.Println(g.Marshal(g.M("success", false, "error", g.ErrMsg(err)))) + return + } + return ok, err } // for testing specific endpoints diff --git a/cmd/sling/sling_init.go b/cmd/sling/sling_init.go new file mode 100644 index 000000000..0fa3533af --- /dev/null +++ b/cmd/sling/sling_init.go @@ -0,0 +1,155 @@ +package main + +import ( + "bufio" + "fmt" + "os" + "strings" + + "github.com/flarco/g" + "github.com/slingdata-io/sling-cli/core/dbio/connection" + "github.com/slingdata-io/sling-cli/core/env" + "github.com/slingdata-io/sling-cli/core/sling/project" + "github.com/spf13/cast" +) + +var initFlags = []g.Flag{ + {Name: "source", Type: "string", Description: "Source connection name"}, + {Name: "target", ShortName: "t", Type: "string", Description: "Target connection name"}, + {Name: "name", Type: "string", Description: "Project name (default: folder name)"}, + {Name: "yes", ShortName: "y", Type: "bool", Description: "Overwrite existing files without a prompt"}, + {Name: "force", Type: "bool", Description: "Allow init inside an existing project"}, + {Name: "test", Type: "bool", Description: "Test source and target connections"}, + {Name: "debug", ShortName: "d", Type: "bool", Description: "Set logging level to DEBUG."}, +} + +var cliInitProject = &g.CliSC{ + Name: "init", + Description: "Create a Sling project in the current folder", + AdditionalHelpPrepend: "\nSee more details at https://docs.slingdata.io/", + Flags: initFlags, + ExecProcess: processInit, +} + +func init() { + cliInitProject.Make().Add() +} + +func processInit(c *g.CliSC) (ok bool, err error) { + if cast.ToBool(c.Vals["debug"]) { + os.Setenv("DEBUG", "LOW") + env.InitLogger() + } + ok = true + return ok, runInit(initOpts(c), cast.ToBool(c.Vals["test"])) +} + +func initOpts(c *g.CliSC) project.Options { + return project.Options{ + Source: cast.ToString(c.Vals["source"]), + Target: cast.ToString(c.Vals["target"]), + Name: cast.ToString(c.Vals["name"]), + Yes: cast.ToBool(c.Vals["yes"]), + Force: cast.ToBool(c.Vals["force"]), + } +} + +func runInit(opts project.Options, testConns bool) error { + if err := resolveInitConns(&opts, testConns); err != nil { + return err + } + + res, err := project.Init(opts) + if err != nil { + return err + } + + for _, f := range res.Files { + g.Info("wrote `%s`", f) + } + fmt.Println() + fmt.Println(" Next: Add .sql models to schema folders and run 'sling build'") + fmt.Println(" Run a job locally with 'sling run -j '. Schedules fire on the platform after 'sling project deploy'.") + fmt.Println() + + if os.Getenv("SLING_PROJECT_TOKEN") != "" { + fmt.Println("To preview the platform jobs for this folder, run `sling project deploy --check`.") + } + return nil +} + +// resolveInitConns fills in missing source/target via interactive prompts and +// optionally tests both connections. +func resolveInitConns(opts *project.Options, testConns bool) error { + opts.Source = strings.TrimSpace(opts.Source) + opts.Target = strings.TrimSpace(opts.Target) + + if opts.Source == "" || opts.Target == "" { + if err := promptMissingConns(opts); err != nil { + return err + } + } + + if testConns { + entries := connection.GetLocalConns() + for _, name := range []string{opts.Source, opts.Target} { + if err := testNamedConn(entries, name); err != nil { + return err + } + } + } + return nil +} + +// promptMissingConns lists local connections and asks for any of source/target +// not already provided via flags. +func promptMissingConns(opts *project.Options) error { + entries := connection.GetLocalConns() + if len(entries) == 0 { + return g.Error("no connections found. Run `sling assist` or `sling conns set --type` to add a connection.") + } + if !isInteractive() { + return g.Error("source and target are required; pass --source and --target with --yes") + } + + fmt.Println("\n Available connections:") + for i, conn := range entries { + fmt.Printf(" %d. %s (%s)\n", i+1, conn.Name, conn.Connection.Type.String()) + } + fmt.Println() + + reader := bufio.NewReader(os.Stdin) + var err error + if opts.Source == "" { + if opts.Source, err = askPrompt(reader, " ? Source connection: "); err != nil { + return err + } + } + if opts.Target == "" { + if opts.Target, err = askPrompt(reader, " ? Target connection: "); err != nil { + return err + } + } + + if opts.Source == "" || opts.Target == "" { + return g.Error("source and target are required") + } + return nil +} + +// testNamedConn tests one connection looked up by name in the given entries. +func testNamedConn(entries connection.ConnEntries, name string) error { + conn := entries.Get(name) + if conn.Name == "" { + return g.Error("connection %s not found", name) + } + ok, err := conn.Connection.Test() + conn.Connection.Close() + if err != nil { + return g.Error(err, "connection %s failed", name) + } + if !ok { + return g.Error("connection %s failed", name) + } + return nil +} diff --git a/cmd/sling/sling_prompt.go b/cmd/sling/sling_prompt.go deleted file mode 100644 index 90d117a34..000000000 --- a/cmd/sling/sling_prompt.go +++ /dev/null @@ -1,122 +0,0 @@ -package main - -import ( - "fmt" - "os" - "strings" - - "github.com/c-bata/go-prompt" - "github.com/flarco/g" - "github.com/slingdata-io/sling-cli/core" - "github.com/slingdata-io/sling-cli/core/dbio/connection" - "github.com/spf13/cast" -) - -var suggestions = []prompt.Suggest{} - -func init() { - suggList := [][]string{ - {"exit", "Exit interactive mode"}, - {cliRun.Name, cliRun.Description}, - {cliConns.Name, cliConns.Description}, - } - for _, sl := range suggList { - suggestions = append(suggestions, prompt.Suggest{Text: sl[0], Description: sl[1]}) - } -} - -func completer(in prompt.Document) []prompt.Suggest { - localSuggestions := []prompt.Suggest{} - - w := in.GetWordBeforeCursor() - blocks := strings.Split(in.Text, " ") - // trimmedBlocks := strings.Split(strings.TrimSpace(in.Text), " ") - - lastWord := blocks[len(blocks)-1] - prevWord := lastWord - - if len(blocks) > 1 { - prevWord = blocks[len(blocks)-2] - } - - switch blocks[0] { - case cliRun.Name: - // collect strings flags - stringFlags := []string{} - for _, f := range cliRun.Flags { - if f.Type == "string" { - stringFlags = append(stringFlags, f.Name) - } - } - - // suggestions based on previous word - switch { - case g.In(prevWord, "src-conn", "tgt-conn"): - for _, conn := range connection.GetLocalConns() { - localSuggestions = append(localSuggestions, prompt.Suggest{Text: conn.Name, Description: conn.Description}) - } - return prompt.FilterHasPrefix(localSuggestions, w, true) - case g.In(prevWord, stringFlags...): - return []prompt.Suggest{} - } - - // suggest normal flags - for _, f := range cliRun.Flags { - localSuggestions = append(localSuggestions, prompt.Suggest{Text: f.Name, Description: f.Description}) - } - return prompt.FilterHasPrefix(localSuggestions, w, true) - - case cliConns.Name: - for _, f := range cliConns.Flags { - localSuggestions = append(localSuggestions, prompt.Suggest{Text: f.Name, Description: f.Description}) - } - return prompt.FilterHasPrefix(localSuggestions, w, true) - case "": - return []prompt.Suggest{} - } - return prompt.FilterHasPrefix(suggestions, w, true) -} - -func executor(in string) { - in = strings.TrimSpace(in) - - blocks := strings.Split(in, " ") - switch blocks[0] { - case "exit": - fmt.Println("exiting") - os.Exit(0) - case cliRun.Name: - cliRun.Vals = g.M(cast.ToSlice(blocks[1:])...) - _, err := cliRun.ExecProcess(cliRun) - g.LogError(err) - case cliConns.Name: - if len(blocks) == 1 { - return - } - for _, subCom := range cliConns.SubComs { - if subCom.Name == blocks[1] { - subCom.Vals = g.M(cast.ToSlice(blocks[2:])...) - _, err := subCom.ExecProcess(subCom) - g.LogError(err) - } - } - } - println(in) -} - -func slingPrompt(c *g.CliSC) (ok bool, err error) { - fmt.Println("sling - An Extract-Load tool") - fmt.Println("Slings data from a data source to a data target.\nVersion " + core.Version) - - p := prompt.New( - executor, - completer, - prompt.OptionPrefix("sling > "), - // prompt.OptionLivePrefix(livePrefix), - prompt.OptionTitle("sling"), - ) - - p.Run() - - return -} diff --git a/cmd/sling/sling_run.go b/cmd/sling/sling_run.go index beb81cc72..6307b93bf 100755 --- a/cmd/sling/sling_run.go +++ b/cmd/sling/sling_run.go @@ -23,6 +23,8 @@ import ( "github.com/slingdata-io/sling-cli/core/dbio/iop" "github.com/slingdata-io/sling-cli/core/env" "github.com/slingdata-io/sling-cli/core/sling" + "github.com/slingdata-io/sling-cli/core/sling/project" + "github.com/slingdata-io/sling-cli/core/sling/validate" "github.com/flarco/g" "github.com/spf13/cast" @@ -50,6 +52,7 @@ func processRun(c *g.CliSC) (ok bool, err error) { } var replicationCfgPath, pipelineCfgPath, directoryPath string + var jobKey, barePath string showExamples := false selectStreams := []string{} @@ -88,10 +91,17 @@ func processRun(c *g.CliSC) (ok bool, err error) { case "directory": env.SetTelVal("run_mode", "directory") directoryPath = cast.ToString(v) + case "job": + jobKey = cast.ToString(v) case "path": filePath := cast.ToString(v) fileInfo, err := os.Stat(filePath) if err != nil { + // Missing path: try it as a job key after the flag loop. + if jobKey == "" && strings.TrimSpace(filePath) != "" { + barePath = filePath + continue + } return true, g.Error(err, "error accessing path: %s", filePath) } @@ -252,6 +262,23 @@ func processRun(c *g.CliSC) (ok bool, err error) { env.InitLogger() } + if key := lo.Ternary(jobKey != "", jobKey, barePath); key != "" { + path, isPipeline, jobErr := resolveJob(key, cfg, &selectStreams) + if jobErr != nil { + if jobKey == "" { + return true, g.Error(jobErr, "error accessing path: %s", barePath) + } + return true, jobErr + } + if isPipeline { + env.SetTelVal("run_mode", "pipeline") + pipelineCfgPath = path + } else { + env.SetTelVal("run_mode", "replication") + replicationCfgPath = path + } + } + if showExamples { println(examples) return ok, nil @@ -297,7 +324,7 @@ runReplication: return ok, g.Error(err, "failure running directory (see docs @ https://docs.slingdata.io)") } } else if pipelineCfgPath != "" { - err = runPipeline(pipelineCfgPath) + err = runPipeline(pipelineCfgPath, cfg.Env) if err != nil { return ok, g.Error(err, "failure running pipeline (see docs @ https://docs.slingdata.io)") } @@ -346,6 +373,57 @@ runReplication: return ok, err } +// resolveJob turns a manifest key into a replication/pipeline path. +// Streams, mode, and variables from the spec fill cfg only when the CLI left them empty. +func resolveJob(key string, cfg *sling.Config, selectStreams *[]string) (path string, isPipeline bool, err error) { + wd, err := os.Getwd() + if err != nil { + return "", false, g.Error(err, "could not get working directory") + } + root, spec, err := project.ResolveJob(wd, key) + if err != nil { + return "", false, err + } + + file := strings.TrimSpace(spec.File) + if file == "" { + return "", false, g.Error("job %s has no file", key) + } + if !filepath.IsAbs(file) { + file = filepath.Join(root, file) + } + body, err := os.ReadFile(file) + if err != nil { + return "", false, g.Error(err, "could not read the file for job %s", key) + } + + kind := validate.DetectFileKind(body, file) + if kind != validate.KindReplication && kind != validate.KindPipeline { + return "", false, g.Error("job %s file %s is a %s; expected a replication or pipeline", key, spec.File, kind) + } + + if len(spec.Streams) > 0 && len(*selectStreams) == 0 { + *selectStreams = spec.Streams + } + if spec.Mode != "" && strings.TrimSpace(string(cfg.Mode)) == "" { + cfg.Mode = sling.Mode(spec.Mode) + } + if len(spec.Variables) > 0 { + if cfg.Env == nil { + cfg.Env = map[string]string{} + } + for k, v := range spec.Variables { + if _, set := cfg.Env[k]; !set { + cfg.Env[k] = v + } + } + } + if len(spec.Schedules) > 0 { + g.Debug("schedules fire on the platform after deploy; this run is manual") + } + return file, kind == validate.KindPipeline, nil +} + func runTask(cfg *sling.Config, replication *sling.ReplicationConfig) (err error) { var task *sling.TaskExecution @@ -698,7 +776,7 @@ func replicationRun(cfgPath string, cfgOverwrite *sling.Config, selectStreams .. return eG.Err() } -func runPipeline(pipelineCfgPath string) (err error) { +func runPipeline(pipelineCfgPath string, overlay map[string]string) (err error) { g.Debug("Sling version: %s (%s %s)", core.Version, runtime.GOOS, runtime.GOARCH) pipeline, err := sling.LoadPipelineConfigFromFile(pipelineCfgPath) @@ -706,6 +784,16 @@ func runPipeline(pipelineCfgPath string) (err error) { return g.Error(err, "could not load pipeline: %s", pipelineCfgPath) } + // Job / --env values overlay the file env. CLI already won over the job spec in resolveJob. + if len(overlay) > 0 { + if pipeline.Env == nil { + pipeline.Env = map[string]any{} + } + for k, v := range overlay { + pipeline.Env[k] = v + } + } + // load SLING_TIMEOUT if specified in pipeline env timeoutR := pipeline.Env["SLING_TIMEOUT"] timeoutE := os.Getenv("SLING_TIMEOUT") @@ -789,7 +877,7 @@ func runDirectory(directoryPath string) (err error) { switch runFile.Type { case sling.RunFilePipeline: - err = runPipeline(runFile.File.RelPath) + err = runPipeline(runFile.File.RelPath, nil) case sling.RunFileReplication: err = runReplication(runFile.File.RelPath, nil) } diff --git a/core/sling/assist/assist.go b/core/sling/assist/assist.go new file mode 100644 index 000000000..abb53a7b6 --- /dev/null +++ b/core/sling/assist/assist.go @@ -0,0 +1,279 @@ +// Package assist implements `sling assist`: profile, skills/MCP install, prompt, resume. +package assist + +import ( + "embed" + "os" + "path/filepath" + "sync" + + "github.com/flarco/g" + "github.com/slingdata-io/sling-cli/core/env" + "gopkg.in/yaml.v3" +) + +//go:embed all:skills +var SkillsFS embed.FS + +//go:embed prompts.yaml +var PromptsYAML []byte + +// SchemaVersion is the on-disk layout version. +const SchemaVersion = 1 + +// HistoryMaxEntries caps entries under ~/.sling/assist/history/. +const HistoryMaxEntries = 100 + +// ExecsMaxEntries caps failure snapshots under ~/.sling/assist/errors/. +const ExecsMaxEntries = 100 + +// Paths is the injectable path seam (tests use SetPaths). +type Paths struct { + SlingHome string + UserHome string + CWD string +} + +var ( + pathsMu sync.RWMutex + pathsOverride *Paths // nil = live OS +) + +// CurrentPaths returns the active path set (override or live). +func CurrentPaths() Paths { + pathsMu.RLock() + ov := pathsOverride + pathsMu.RUnlock() + if ov != nil { + return *ov + } + cwd, _ := os.Getwd() + return Paths{ + SlingHome: env.HomeDir, + UserHome: g.UserHomeDir(), + CWD: cwd, + } +} + +// SetPaths installs a path override; restore undoes it. +func SetPaths(p Paths) (restore func()) { + pathsMu.Lock() + prev := pathsOverride + cp := p + pathsOverride = &cp + pathsMu.Unlock() + return func() { + pathsMu.Lock() + pathsOverride = prev + pathsMu.Unlock() + } +} + +func slingHome() string { + return CurrentPaths().SlingHome +} + +func userHome() string { + return CurrentPaths().UserHome +} + +func workDir() string { + p := CurrentPaths() + if p.CWD != "" { + return p.CWD + } + cwd, _ := os.Getwd() + return cwd +} + +// projectRootMarkers identify a project root when walking up for --scope project. +var projectRootMarkers = []string{ + ".git", + "go.mod", + "package.json", + "pyproject.toml", + "Cargo.toml", + "sling_build.yml", + ".sling", +} + +// projectRoot walks up from workDir for a marker; falls back to workDir. +func projectRoot() string { + dir := workDir() + if dir == "" { + return dir + } + start := dir + for { + for _, m := range projectRootMarkers { + if g.PathExists(filepath.Join(dir, m)) { + return dir + } + } + parent := filepath.Dir(dir) + if parent == dir { + return start + } + dir = parent + } +} + +func projectPath(elem ...string) string { + return filepath.Join(append([]string{projectRoot()}, elem...)...) +} + +const assistEnvKey = "SLING_ASSIST" + +// Profile is stored under env.SLING_ASSIST in ~/.sling/env.yaml. +type Profile struct { + Agent string `yaml:"agent" json:"agent"` // claude | codex | … | auto + HintInErrors bool `yaml:"hint_in_errors" json:"hint_in_errors"` // run-error footer + DefaultInstallScope string `yaml:"default_install_scope,omitempty" json:"default_install_scope"` // user | project +} + +// DefaultProfile returns sane first-run defaults. +func DefaultProfile() Profile { + return Profile{ + Agent: "auto", + HintInErrors: true, + DefaultInstallScope: "user", + } +} + +func envFilePath() string { + return env.GetEnvFilePath(slingHome()) +} + +// LoadProfile reads env.SLING_ASSIST. Missing key → (Profile{}, false, nil). +func LoadProfile() (p Profile, exists bool, err error) { + path := envFilePath() + if _, statErr := os.Stat(path); statErr != nil { + if os.IsNotExist(statErr) { + return Profile{}, false, nil + } + return Profile{}, false, g.Error(statErr, "could not stat %s", path) + } + ef := env.LoadEnvFile(path) + raw, ok := ef.Env[assistEnvKey] + if !ok || raw == nil { + return Profile{}, false, nil + } + m, err := castToStringMap(raw) + if err != nil { + return Profile{}, false, g.Error(err, "env.%s is not a mapping", assistEnvKey) + } + if len(m) == 0 { + return Profile{}, false, nil + } + p, err = profileFromMap(m) + if err != nil { + return Profile{}, false, err + } + return p, true, nil +} + +// castToStringMap normalizes YAML maps (map[string]any or map[any]any). +func castToStringMap(v any) (map[string]any, error) { + switch m := v.(type) { + case map[string]any: + return m, nil + case map[any]any: + out := make(map[string]any, len(m)) + for k, vv := range m { + ks, ok := k.(string) + if !ok { + return nil, g.Error("non-string key %v", k) + } + out[ks] = vv + } + return out, nil + case nil: + return map[string]any{}, nil + default: + return nil, g.Error("unexpected type %T", v) + } +} + +// SaveProfile writes env.SLING_ASSIST via EnvFile (preserves other keys/comments). +func SaveProfile(p Profile) error { + path := envFilePath() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return g.Error(err, "could not create %s", filepath.Dir(path)) + } + ef := env.LoadEnvFile(path) + ef.Path = path + m, err := profileToMap(p) + if err != nil { + return err + } + if ef.Env == nil { + ef.Env = map[string]any{} + } + ef.Env[assistEnvKey] = m + return ef.WriteEnvFile() +} + +func profileToMap(p Profile) (map[string]any, error) { + b, err := yaml.Marshal(p) + if err != nil { + return nil, g.Error(err, "could not marshal profile") + } + m := map[string]any{} + if uerr := yaml.Unmarshal(b, &m); uerr != nil { + return nil, g.Error(uerr, "could not re-parse profile") + } + return m, nil +} + +func profileFromMap(m map[string]any) (Profile, error) { + b, err := yaml.Marshal(m) + if err != nil { + return Profile{}, g.Error(err, "could not marshal SLING_ASSIST block") + } + var p Profile + if uerr := yaml.Unmarshal(b, &p); uerr != nil { + return Profile{}, g.Error(uerr, "could not parse SLING_ASSIST block") + } + return p, nil +} + +// AssistDir returns ~/.sling/assist/. +func AssistDir() string { + d := filepath.Join(slingHome(), "assist") + _ = os.MkdirAll(d, 0o755) + return d +} + +// HistoryDir returns ~/.sling/assist/history/. +func HistoryDir() string { + d := filepath.Join(AssistDir(), "history") + _ = os.MkdirAll(d, 0o755) + return d +} + +// ErrorsDir returns ~/.sling/assist/errors/ (legacy snapshot root). +// New snapshots live under ExecutionsDir(); readers scan both. +func ErrorsDir() string { + d := filepath.Join(AssistDir(), "errors") + _ = os.MkdirAll(d, 0o755) + return d +} + +// ExecutionsDir returns ~/.sling/assist/errors/executions/ (failure snapshots). +func ExecutionsDir() string { + d := filepath.Join(ErrorsDir(), "executions") + _ = os.MkdirAll(d, 0o755) + return d +} + +// VersionFilePath returns ~/.sling/assist/version. +func VersionFilePath() string { + return filepath.Join(AssistDir(), "version") +} + +// CanonicalSkillsDir returns ~/.agents/skills/ (shared skill source of truth). +func CanonicalSkillsDir() string { + d := filepath.Join(userHome(), ".agents", "skills") + _ = os.MkdirAll(d, 0o755) + return d +} diff --git a/core/sling/assist/assist_test.go b/core/sling/assist/assist_test.go new file mode 100644 index 000000000..6a0c6376c --- /dev/null +++ b/core/sling/assist/assist_test.go @@ -0,0 +1,776 @@ +// Package tests: bugs, harness gaps, and extensibility seams. + +package assist + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/slingdata-io/sling-cli/core" +) + +func TestTryRefreshLockExclusiveAndTokenUnlock(t *testing.T) { + dir := t.TempDir() + lockPath := filepath.Join(dir, ".refresh-lock") + + unlock1, ok := tryRefreshLock(lockPath) + if !ok { + t.Fatal("first lock should succeed") + } + if _, ok := tryRefreshLock(lockPath); ok { + t.Fatal("second lock should fail while first is held") + } + unlock1() + unlock2, ok := tryRefreshLock(lockPath) + if !ok { + t.Fatal("lock after unlock should succeed") + } + unlock2() + if _, err := os.Stat(lockPath); !os.IsNotExist(err) { + t.Fatalf("lock file should be removed after unlock, err=%v", err) + } +} + +func TestTryRefreshLockReclaimsStale(t *testing.T) { + dir := t.TempDir() + lockPath := filepath.Join(dir, ".refresh-lock") + if err := os.WriteFile(lockPath, []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + stale := time.Now().Add(-refreshLockStale - time.Minute) + if err := os.Chtimes(lockPath, stale, stale); err != nil { + t.Fatal(err) + } + unlock, ok := tryRefreshLock(lockPath) + if !ok { + t.Fatal("stale lock should be reclaimable") + } + unlock() +} + +func TestPruneRetiredSkillsRemovesStaleDirs(t *testing.T) { + withTempHomeDir(t) + root := CanonicalSkillsDir() + stale := filepath.Join(root, "sling-hooks") + if err := os.MkdirAll(stale, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(stale, "SKILL.md"), []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + keep := filepath.Join(root, "sling-pipelines") + if err := os.MkdirAll(keep, 0o755); err != nil { + t.Fatal(err) + } + + pruneRetiredSkills(context.Background(), ScopeUser) + + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Fatalf("retired skill dir %s should be removed", stale) + } + if _, err := os.Stat(keep); err != nil { + t.Fatalf("current skill dir %s should remain: %v", keep, err) + } +} + +func TestUninstallMarksStampSoAutoRefreshSkips(t *testing.T) { + withTempHomeDir(t) + // Simulate a prior install stamp, then uninstall skills. + if err := os.MkdirAll(AssistDir(), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(VersionFilePath(), []byte("0.0.0-old"), 0o644); err != nil { + t.Fatal(err) + } + if err := Uninstall(context.Background(), UninstallOptions{NonInteractive: true}); err != nil { + t.Fatal(err) + } + got, err := os.ReadFile(VersionFilePath()) + if err != nil { + t.Fatal(err) + } + if strings.TrimSpace(string(got)) != versionUninstalled { + t.Fatalf("stamp = %q, want %q", got, versionUninstalled) + } + notice, err := AutoRefresh(context.Background()) + if err != nil { + t.Fatal(err) + } + if notice != "" { + t.Fatalf("AutoRefresh after uninstall should no-op, got notice %q", notice) + } +} + +func TestAutoRefreshNoopsWhenNeverInstalled(t *testing.T) { + withTempHomeDir(t) + // No skills on disk → never installed; must not write the bundle. + notice, err := AutoRefresh(context.Background()) + if err != nil { + t.Fatal(err) + } + if notice != "" { + t.Fatalf("AutoRefresh with no skills should no-op, got %q", notice) + } + for _, name := range listSkillNames() { + if _, err := os.Stat(canonicalSkillPath(name)); !os.IsNotExist(err) { + t.Fatalf("should not install %s when no skills exist", name) + } + } +} + +func TestAutoRefreshNoopsWhenStampStaleButNoSkills(t *testing.T) { + withTempHomeDir(t) + if err := os.MkdirAll(AssistDir(), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(VersionFilePath(), []byte("0.0.0-old"), 0o644); err != nil { + t.Fatal(err) + } + notice, err := AutoRefresh(context.Background()) + if err != nil { + t.Fatal(err) + } + if notice != "" { + t.Fatalf("AutoRefresh with no skills should no-op, got %q", notice) + } + for _, name := range listSkillNames() { + if _, err := os.Stat(canonicalSkillPath(name)); !os.IsNotExist(err) { + t.Fatalf("should not install %s from a stale stamp", name) + } + } +} + +func TestAutoRefreshHealsDriftWhenStampCurrent(t *testing.T) { + withTempHomeDir(t) + names := listSkillNames() + if len(names) == 0 { + t.Fatal("no embedded skills") + } + if err := writeCanonicalBundle(names); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(AssistDir(), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(VersionFilePath(), []byte(core.Version), 0o644); err != nil { + t.Fatal(err) + } + + name := names[0] + skillPath := canonicalSkillPath(name) + if err := os.WriteFile(skillPath, []byte("drifted-content"), 0o644); err != nil { + t.Fatal(err) + } + + notice, err := AutoRefresh(context.Background()) + if err != nil { + t.Fatal(err) + } + if notice == "" { + t.Fatal("expected refresh notice after drift") + } + + got, err := os.ReadFile(skillPath) + if err != nil { + t.Fatal(err) + } + want, err := SkillsFS.ReadFile(filepath.ToSlash(filepath.Join("skills", name, "SKILL.md"))) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, want) { + t.Fatalf("skill %s still drifted after AutoRefresh", name) + } +} + +func TestAutoRefreshPrunesRetiredWhenStampCurrent(t *testing.T) { + withTempHomeDir(t) + names := listSkillNames() + if err := writeCanonicalBundle(names); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(AssistDir(), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(VersionFilePath(), []byte(core.Version), 0o644); err != nil { + t.Fatal(err) + } + + stale := filepath.Join(CanonicalSkillsDir(), "sling-hooks") + if err := os.MkdirAll(stale, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(stale, "SKILL.md"), []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + + notice, err := AutoRefresh(context.Background()) + if err != nil { + t.Fatal(err) + } + if notice == "" { + t.Fatal("expected refresh notice after prune") + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Fatalf("retired skill dir %s should be removed", stale) + } +} + +func TestAgentExitErrorExitCodeOf(t *testing.T) { + err := &AgentExitError{ExitCode: 42, Agent: "claude"} + code, ok := ExitCodeOf(err) + if !ok || code != 42 { + t.Fatalf("ExitCodeOf = %d, %v", code, ok) + } + if _, ok := ExitCodeOf(fmt.Errorf("other")); ok { + t.Fatal("non-agent error should not match") + } +} + +func TestClaudeProjectMCPPath(t *testing.T) { + c := &claudeClient{} + got := c.mcpPath(ScopeProject) + if got != filepath.Join(".", ".mcp.json") && got != ".mcp.json" { + // filepath.Join(".", ".mcp.json") is "./.mcp.json" on Unix + if !strings.HasSuffix(got, ".mcp.json") || strings.Contains(got, ".claude.json") { + t.Fatalf("project mcp path = %q", got) + } + } + if strings.Contains(c.mcpPath(ScopeProject), ".claude.json") { + t.Fatalf("project scope must not use .claude.json: %q", c.mcpPath(ScopeProject)) + } +} + +func TestVSCodeMCPUsesServersKey(t *testing.T) { + // Unit-level: path helper for project scope. + c := &vscodeClient{} + p := c.vscodeMCPPath(ScopeProject) + if !strings.Contains(p, ".vscode") || !strings.HasSuffix(p, "mcp.json") { + t.Fatalf("project vscode mcp path = %q", p) + } +} + +func TestAgentLaunchArgsPerAgent(t *testing.T) { + path := "/tmp/prompt.md" + cases := []struct { + agent string + wantArgs []string + stdin bool + }{ + {"codex", []string{"exec", "-"}, true}, + {"gemini", []string{"-p", "-"}, true}, + {"claude", []string{"Read and execute the task in @" + path}, false}, + {"cursor", []string{path}, false}, + {"grok", []string{"Read and execute the task in @" + path}, false}, + {"pi", []string{"-p"}, true}, + {"opencode", []string{"run", "--file", path, "Read and execute the attached task"}, false}, + {"unknown-cli", nil, true}, + } + for _, tc := range cases { + p := agentLaunchArgs(tc.agent, path, "", "") + if p.UseStdin != tc.stdin { + t.Errorf("%s UseStdin=%v want %v", tc.agent, p.UseStdin, tc.stdin) + } + if len(p.Args) != len(tc.wantArgs) { + t.Errorf("%s args=%v want %v", tc.agent, p.Args, tc.wantArgs) + continue + } + for i := range tc.wantArgs { + if p.Args[i] != tc.wantArgs[i] { + t.Errorf("%s args[%d]=%q want %q", tc.agent, i, p.Args[i], tc.wantArgs[i]) + } + } + } +} + +func TestResolveAgentOverrideWins(t *testing.T) { + // Need a detectable agent dir so override succeeds. + home := t.TempDir() + prevHome := os.Getenv("HOME") + os.Setenv("HOME", home) + t.Cleanup(func() { os.Setenv("HOME", prevHome) }) + // claude Detect() requires the binary on $PATH + if err := os.MkdirAll(filepath.Join(home, ".claude"), 0o755); err != nil { + t.Fatal(err) + } + // Put claude on PATH via a stub. + bin := filepath.Join(home, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + stub := filepath.Join(bin, "claude") + if err := os.WriteFile(stub, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + prevPath := os.Getenv("PATH") + os.Setenv("PATH", bin+string(os.PathListSeparator)+prevPath) + t.Cleanup(func() { os.Setenv("PATH", prevPath) }) + + got, err := ResolveAgent("claude", Profile{Agent: "codex"}) + if err != nil { + t.Fatal(err) + } + if got != "claude" { + t.Fatalf("override should win: got %q", got) + } +} + +func TestResolveAgentUnknownOverride(t *testing.T) { + _, err := ResolveAgent("not-an-agent", Profile{}) + if err == nil || !strings.Contains(err.Error(), "unknown agent") { + t.Fatalf("err = %v", err) + } +} + +func TestResolveAgentProfileNonLaunchable(t *testing.T) { + _, err := ResolveAgent("", Profile{Agent: "vscode"}) + if err == nil || !strings.Contains(err.Error(), "non-launchable") { + t.Fatalf("err = %v", err) + } +} + +func TestSlugify(t *testing.T) { + if got := slugify("Hello World!"); got != "hello-world" { + t.Fatalf("got %q", got) + } + if got := slugify(" "); got != "entry" { + t.Fatalf("empty → entry, got %q", got) + } + if got := slugify("a / b"); got != "a-b" { + t.Fatalf("separator runs must collapse, got %q", got) + } +} + +func TestSlugifyCapsLength(t *testing.T) { + long := "Help me create or update a Sling config (replication, pipeline, model, or API spec). First ask me which one." + got := slugify(long) + if len(got) > maxSlugLen { + t.Fatalf("slug %q is %d chars, want <= %d", got, len(got), maxSlugLen) + } + if strings.HasSuffix(got, "-") || strings.HasPrefix(got, "-") { + t.Fatalf("slug must not have dangling separators: %q", got) + } + // Cut on a word boundary: every kept word is whole. + want := "help-me-create-or-update-a-sling-config" + if got != want { + t.Fatalf("slug = %q, want %q", got, want) + } +} + +func TestSaveEntryIDStaysShort(t *testing.T) { + withTempHomeDir(t) + a := AnswersFile{ + Name: slugify("Help me create or update a Sling config (replication, pipeline, model, or API spec)"), + Task: "open", + } + id, err := SaveEntry(a, "prompt", Meta{Task: "open"}) + if err != nil { + t.Fatal(err) + } + // + "_" + slug + if len(id) > 20+maxSlugLen { + t.Fatalf("id %q is %d chars", id, len(id)) + } + if _, err := LoadEntry(id); err != nil { + t.Fatalf("round-trip: %v", err) + } +} + +func TestFilterEntriesQuery(t *testing.T) { + entries := []Entry{ + {ID: "one", Answers: AnswersFile{Task: "replication.create", Agent: "claude", Name: "pg-to-sf"}}, + {ID: "two", Answers: AnswersFile{Task: "pipeline.create", Agent: "codex", Name: "daily"}}, + } + got := filterEntries(entries, "pg-to") + if len(got) != 1 || got[0].ID != "one" { + t.Fatalf("got %+v", got) + } + got = filterEntries(entries, "codex") + if len(got) != 1 || got[0].ID != "two" { + t.Fatalf("got %+v", got) + } + got = filterEntries(entries, "") + if len(got) != 2 { + t.Fatalf("empty query len=%d", len(got)) + } +} + +func TestSessionPrintWithoutSetup(t *testing.T) { + withTempHomeDir(t) + _, err := Session(SessionOptions{Print: true, Ask: "from PG"}) + if err != nil { + t.Fatal(err) + } +} + +func TestSessionLaunchWithoutSetupFails(t *testing.T) { + withTempHomeDir(t) + _, err := Session(SessionOptions{Headless: true, Ask: "from PG"}) + if err == nil { + t.Fatal("expected setup error") + } + if !strings.Contains(err.Error(), "sling assist setup") { + t.Fatalf("want setup hint, got: %v", err) + } +} + +func TestPickerRendersOldTaskAndNewMode(t *testing.T) { + entries := []Entry{ + {ID: "old", Answers: AnswersFile{Task: "replication.create", Name: "pg-to-sf", Created: time.Now()}}, + {ID: "open1", Answers: AnswersFile{Task: "open", Name: "backfill", Created: time.Now()}}, + {ID: "ask1", Answers: AnswersFile{Task: "ask", Name: "hello", Created: time.Now()}}, + } + got := newPickerModel(entries).View() + for _, w := range []string{"replication", "open", "ask"} { + if !strings.Contains(got, w) { + t.Errorf("missing %q\n%s", w, got) + } + } +} + +func TestDoctorReportToJSONTyped(t *testing.T) { + withTempHomeDir(t) + r, err := Doctor(context.Background()) + if err != nil { + t.Fatal(err) + } + if r == nil { + t.Fatal("nil report") + } + if r.SlingVersion == "" { + t.Fatal("missing sling_version") + } + if len(r.Findings) == 0 { + t.Fatal("expected findings") + } + // Findings must not use glyph-prefixed prose as the only structure. + for _, f := range r.Findings { + if f.ID == "" { + t.Fatalf("finding missing id: %+v", f) + } + if strings.HasPrefix(f.Summary, "✓") || strings.HasPrefix(f.Summary, "✗") { + t.Fatalf("summary still has glyph: %q", f.Summary) + } + } + body, err := r.ToJSON() + if err != nil { + t.Fatal(err) + } + var decoded map[string]any + if err := json.Unmarshal(body, &decoded); err != nil { + t.Fatal(err) + } + if _, ok := decoded["findings"]; !ok { + t.Fatalf("json missing findings: %s", body) + } + if _, ok := decoded["sling_version"]; !ok { + t.Fatalf("json missing sling_version: %s", body) + } + // Lines must not appear in JSON (json:"-"). + if _, ok := decoded["Lines"]; ok { + t.Fatal("Lines should not be in JSON") + } + if _, ok := decoded["lines"]; ok { + t.Fatal("lines should not be in JSON") + } +} + +func TestDoctorContextCanceled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := Doctor(ctx) + if err == nil { + t.Fatal("expected context error") + } +} + +func TestCellStateJSON(t *testing.T) { + m := DoctorMatrix{ + Clients: []string{"claude"}, + Rows: []MatrixRow{{ + Label: "MCP", + Cells: map[string]CellState{"claude": CellOK}, + }}, + } + b, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), `"ok"`) { + t.Fatalf("cell state not stringified: %s", b) + } +} + +func TestCheckResultRender(t *testing.T) { + r := checkSkill(CellFail, "sling", "redirect missing") + line := r.Render("claude") + if !strings.HasPrefix(line, "✗") { + t.Fatalf("glyph: %q", line) + } + if !strings.Contains(line, "claude") || !strings.Contains(line, "sling") { + t.Fatalf("line: %q", line) + } +} + +func TestPathsOverrideAffectsSlingHome(t *testing.T) { + dir := t.TempDir() + restore := SetPaths(Paths{SlingHome: dir, UserHome: dir, CWD: dir}) + defer restore() + if !strings.HasPrefix(LogsRoot(), dir) { + t.Fatalf("LogsRoot=%q not under %q", LogsRoot(), dir) + } + if !strings.HasPrefix(AssistDir(), dir) { + t.Fatalf("AssistDir=%q not under %q", AssistDir(), dir) + } + if !strings.HasPrefix(ErrorsDir(), dir) { + t.Fatalf("ErrorsDir=%q not under %q", ErrorsDir(), dir) + } + if userHome() != dir { + t.Fatalf("userHome=%q want %q", userHome(), dir) + } +} + +func TestInstallRespectsCanceledContext(t *testing.T) { + withTempHomeDir(t) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := Install(ctx, InstallOptions{NonInteractive: true, DefaultAgent: "claude"}) + if err == nil { + t.Fatal("expected cancel error") + } +} + +func TestCodexMCPSectionLineAnchored(t *testing.T) { + // Mention in a comment or string must not count. + if hasCodexMCPSection(`# see [mcp_servers.sling] docs +name = "x" +`) { + t.Fatal("comment mention should not count") + } + body := ` +[mcp_servers.other] +command = "x" + +[mcp_servers.sling] +command = "sling" +args = ["serve", "mcp"] + +[mcp_servers.sling.env] +FOO = "bar" +` + if !hasCodexMCPSection(body) { + t.Fatal("expected section present") + } + out := removeCodexMCP(body) + if hasCodexMCPSection(out) { + t.Fatalf("header still present after remove:\n%s", out) + } + if strings.Contains(out, "[mcp_servers.sling.env]") { + t.Fatalf("orphan subtable left behind:\n%s", out) + } + if !strings.Contains(out, "[mcp_servers.other]") { + t.Fatalf("sibling section removed:\n%s", out) + } +} + +func TestBackupPreservesSourceMode(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "secret.json") + if err := os.WriteFile(path, []byte(`{"a":1}`), 0o600); err != nil { + t.Fatal(err) + } + if err := backupBeforeEdit(path); err != nil { + t.Fatal(err) + } + info, err := os.Stat(path + backupSuffix) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o600 { + t.Fatalf("backup mode = %o, want 0600", info.Mode().Perm()) + } +} + +func TestResumePrintsSavedPrompt(t *testing.T) { + withTempHomeDir(t) + a := AnswersFile{ + Name: "edit-me", + Task: "replication.update", + Created: time.Now().UTC(), + Agent: "claude", + Answers: map[string]any{"ask": "add incremental mode"}, + } + id, err := SaveEntry(a, "saved prompt body\n", Meta{Task: a.Task, Agent: "claude", HarnessSessionID: "abc"}) + if err != nil { + t.Fatal(err) + } + old := assistOut + var buf bytes.Buffer + assistOut = &buf + t.Cleanup(func() { assistOut = old }) + _, err = Session(SessionOptions{ResumeID: id, Print: true}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(buf.String(), "saved prompt body") { + t.Fatalf("got %q", buf.String()) + } +} + +func TestProjectRootFindsGit(t *testing.T) { + dir := t.TempDir() + sub := filepath.Join(dir, "a", "b") + if err := os.MkdirAll(sub, 0o755); err != nil { + t.Fatal(err) + } + if err := os.MkdirAll(filepath.Join(dir, ".git"), 0o755); err != nil { + t.Fatal(err) + } + restore := SetPaths(Paths{SlingHome: dir, UserHome: dir, CWD: sub}) + defer restore() + if got := projectRoot(); got != dir { + t.Fatalf("projectRoot = %q, want %q", got, dir) + } +} + +func TestSyncCanonicalSkillPrunesStale(t *testing.T) { + withTempHomeDir(t) + // Write bundle then plant a stale file under a skill dir. + skills := listSkillNames() + if len(skills) == 0 { + t.Fatal("no embedded skills") + } + name := skills[0] + if _, err := syncCanonicalSkill(name); err != nil { + t.Fatal(err) + } + stale := filepath.Join(CanonicalSkillsDir(), name, "STALE_DO_NOT_KEEP.md") + if err := os.WriteFile(stale, []byte("gone"), 0o644); err != nil { + t.Fatal(err) + } + changed, err := syncCanonicalSkill(name) + if err != nil { + t.Fatal(err) + } + if !changed { + t.Fatal("expected change when pruning stale file") + } + if _, err := os.Stat(stale); !os.IsNotExist(err) { + t.Fatalf("stale file still present: %v", err) + } +} + +func TestClaudeLaunchArgsAssignSessionAndModel(t *testing.T) { + path := "/tmp/prompt.md" + sid := "550e8400-e29b-41d4-a716-446655440000" + p := agentLaunchArgs("claude", path, "sonnet", sid) + joined := strings.Join(p.Args, " ") + if !strings.Contains(joined, "--session-id "+sid) { + t.Fatalf("missing session-id: %v", p.Args) + } + if !strings.Contains(joined, "--model sonnet") { + t.Fatalf("missing model: %v", p.Args) + } +} + +func TestCodexLaunchArgsInsertModelAfterSubcommand(t *testing.T) { + p := agentLaunchArgs("codex", "/tmp/p.md", "sonnet", "") + if len(p.Args) < 3 || p.Args[0] != "exec" || p.Args[1] != "-m" || p.Args[2] != "sonnet" { + t.Fatalf("codex args=%v", p.Args) + } +} + +func TestResumeArgsHaveNoPrompt(t *testing.T) { + cases := []struct { + agent string + id string + want []string + }{ + {"claude", "u1", []string{"--resume", "u1"}}, + {"grok", "u2", []string{"--resume", "u2"}}, + {"codex", "u3", []string{"resume", "u3"}}, + {"gemini", "u4", []string{"--resume", "u4"}}, + {"cursor", "u5", []string{"--resume=u5"}}, + {"opencode", "u6", []string{"--session", "u6"}}, + {"pi", "u7", []string{"--session", "u7"}}, + } + for _, tc := range cases { + p := agentResumeArgs(tc.agent, tc.id, "") + if strings.Join(p.Args, " ") != strings.Join(tc.want, " ") { + t.Errorf("%s args=%v want %v", tc.agent, p.Args, tc.want) + } + if p.UseStdin { + t.Errorf("%s resume should not use stdin", tc.agent) + } + } + p := agentResumeArgs("claude", "u1", "sonnet") + if !strings.Contains(strings.Join(p.Args, " "), "--model sonnet") { + t.Fatalf("claude resume missing model: %v", p.Args) + } + p = agentResumeArgs("codex", "u3", "sonnet") + if len(p.Args) < 4 || p.Args[0] != "resume" || p.Args[1] != "-m" || p.Args[2] != "sonnet" { + t.Fatalf("codex resume args=%v", p.Args) + } +} + +func TestDiscoverHarnessSessionIDNewFile(t *testing.T) { + home := t.TempDir() + restore := SetPaths(Paths{SlingHome: home, UserHome: home, CWD: home}) + t.Cleanup(restore) + root := filepath.Join(home, ".codex", "sessions") + if err := os.MkdirAll(root, 0o755); err != nil { + t.Fatal(err) + } + old := filepath.Join(root, "old.jsonl") + if err := os.WriteFile(old, []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + before := snapshotHarnessFiles("codex") + newID := "019dd4bf-0929-7ea0-b227-1f51085e7d71" + if err := os.WriteFile(filepath.Join(root, newID+".jsonl"), []byte("y"), 0o644); err != nil { + t.Fatal(err) + } + got := discoverHarnessSessionID("codex", before) + if got != newID { + t.Fatalf("got %q want %q", got, newID) + } +} + +func TestAgentBinaryCursorIsAgent(t *testing.T) { + if agentBinary("cursor") != "cursor-agent" { + t.Fatal(agentBinary("cursor")) + } + if agentBinary("claude") != "claude" { + t.Fatal(agentBinary("claude")) + } +} + +func TestDoctorHonorsScope(t *testing.T) { + withTempHomeDir(t) + // Doctor with ScopeProject should not panic and should return a report. + r, err := Doctor(context.Background(), DoctorOptions{Scope: ScopeProject}) + if err != nil { + t.Fatal(err) + } + if r == nil { + t.Fatal("nil report") + } +} + +func TestEnsureAssistReadyRequiresProfile(t *testing.T) { + withTempHomeDir(t) + err := EnsureAssistReady() + if err == nil { + t.Fatal("expected error when assist not set up") + } + if !strings.Contains(err.Error(), "sling assist setup") { + t.Fatalf("error should point at setup: %v", err) + } +} diff --git a/core/sling/assist/browser.go b/core/sling/assist/browser.go new file mode 100644 index 000000000..de2d0af2b --- /dev/null +++ b/core/sling/assist/browser.go @@ -0,0 +1,319 @@ +package assist + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + + "github.com/flarco/g" + "github.com/flarco/g/net" + "github.com/slingdata-io/sling-cli/core/env" +) + +// AgentBrowserVersion is the pinned CLI release. Override with AGENT_BROWSER_VERSION. +// Pin checked against https://github.com/vercel-labs/agent-browser/releases (v0.34.0, 2026-08-11). +const AgentBrowserVersion = "0.34.0" + +const agentBrowserGitHubBase = "https://github.com/vercel-labs/agent-browser/releases/download/v{version}/{asset}" + +const agentBrowserMCPName = "agent-browser" + +// agentBrowserTestDownloadURL replaces the GitHub asset URL in tests. +var agentBrowserTestDownloadURL string + +type agentBrowserInstall struct { + version string +} + +func newAgentBrowserInstall() *agentBrowserInstall { + return &agentBrowserInstall{version: agentBrowserVersion()} +} + +func agentBrowserVersion() string { + if val := strings.TrimSpace(os.Getenv("AGENT_BROWSER_VERSION")); val != "" { + return strings.TrimPrefix(val, "v") + } + return AgentBrowserVersion +} + +func agentBrowserBinName() string { + if runtime.GOOS == "windows" { + return "agent-browser.exe" + } + return "agent-browser" +} + +func (a *agentBrowserInstall) dest() string { + return filepath.Join(env.HomeBinDir(), "agent-browser", a.version) +} + +func (a *agentBrowserInstall) bundledPath() string { + return filepath.Join(a.dest(), agentBrowserBinName()) +} + +// BundledAgentBrowserPath is ~/.sling/bin/agent-browser//agent-browser[.exe]. +func BundledAgentBrowserPath() string { + return newAgentBrowserInstall().bundledPath() +} + +func (a *agentBrowserInstall) assetName(goos, goarch string) (string, error) { + var osName, arch string + switch goos { + case "darwin": + osName = "darwin" + case "linux": + osName = "linux" + case "windows": + osName = "win32" + default: + return "", g.Error("agent-browser is not available for %s/%s", goos, goarch) + } + switch goarch { + case "amd64": + arch = "x64" + case "arm64": + arch = "arm64" + default: + return "", g.Error("agent-browser is not available for %s/%s", goos, goarch) + } + if goos == "windows" && goarch == "arm64" { + // Upstream publishes win32-x64 only; x64 binary runs under emulation. + arch = "x64" + } + if goos == "linux" && linuxMuslPresent() { + return fmt.Sprintf("agent-browser-linux-musl-%s", arch), nil + } + name := fmt.Sprintf("agent-browser-%s-%s", osName, arch) + if goos == "windows" { + name += ".exe" + } + return name, nil +} + +// AgentBrowserAssetName is the GitHub asset for goos/goarch (pinned layout, not /latest). +func AgentBrowserAssetName(goos, goarch string) (string, error) { + return newAgentBrowserInstall().assetName(goos, goarch) +} + +func (a *agentBrowserInstall) downloadURL() (string, error) { + if agentBrowserTestDownloadURL != "" { + return agentBrowserTestDownloadURL, nil + } + asset, err := a.assetName(runtime.GOOS, runtime.GOARCH) + if err != nil { + return "", err + } + return g.R(agentBrowserGitHubBase, "version", a.version, "asset", asset), nil +} + +func (a *agentBrowserInstall) versionOK(binPath string) (bool, error) { + out, err := exec.Command(binPath, "--version").CombinedOutput() + if err != nil { + return false, g.Error(err, "could not get version for agent-browser: %s", strings.TrimSpace(string(out))) + } + s := strings.TrimSpace(string(out)) + return strings.Contains(s, a.version), nil +} + +// agentBrowserBin is the command written into MCP configs. +// Order: AGENT_BROWSER_PATH, $PATH, bundled binary, then the bare name. +func agentBrowserBin() string { + if envPath := strings.TrimSpace(os.Getenv("AGENT_BROWSER_PATH")); envPath != "" { + return envPath + } + if p, err := exec.LookPath("agent-browser"); err == nil { + return p + } + bundled := BundledAgentBrowserPath() + if g.PathExists(bundled) { + return bundled + } + return "agent-browser" +} + +func agentBrowserMCPEntry() map[string]any { + return map[string]any{ + "command": agentBrowserBin(), + "args": []any{"mcp", "--tools", "core"}, + } +} + +func opencodeAgentBrowserMCPEntry() map[string]any { + return map[string]any{ + "type": "local", + "command": []any{agentBrowserBin(), "mcp", "--tools", "core"}, + "enabled": true, + } +} + +func skipAgentBrowserDownload() bool { + return os.Getenv("AGENT_BROWSER_SKIP_DOWNLOAD") == "1" +} + +// EnsureBinAgentBrowser returns a usable agent-browser binary. +// Order: AGENT_BROWSER_PATH, $PATH, then a versioned download under ~/.sling/bin/agent-browser//. +func EnsureBinAgentBrowser() (binPath string, err error) { + return newAgentBrowserInstall().ensure() +} + +func (a *agentBrowserInstall) ensure() (binPath string, err error) { + if envPath := strings.TrimSpace(os.Getenv("AGENT_BROWSER_PATH")); envPath != "" { + if !g.PathExists(envPath) { + return "", g.Error("agent-browser binary not found: %s", envPath) + } + if stat, _ := os.Stat(envPath); stat != nil && stat.IsDir() { + return "", g.Error("AGENT_BROWSER_PATH provided is a directory, should be a file: %s", envPath) + } + return envPath, nil + } + + if p, err := exec.LookPath("agent-browser"); err == nil { + return p, nil + } + + if skipAgentBrowserDownload() { + return "agent-browser", nil + } + + folderPath := a.dest() + binPath = a.bundledPath() + found := g.PathExists(binPath) + if found { + ok, verr := a.versionOK(binPath) + if verr != nil { + found = false + } else { + found = ok + } + } + + if !found { + downloadURL, uerr := a.downloadURL() + if uerr != nil { + return "", uerr + } + + if err = os.MkdirAll(folderPath, 0755); err != nil { + return "", g.Error(err, "could not create agent-browser folder") + } + + tmpPath := binPath + ".download" + defer os.Remove(tmpPath) + + g.Info("downloading agent-browser %s for %s/%s", a.version, runtime.GOOS, runtime.GOARCH) + if err = net.DownloadFile(downloadURL, tmpPath); err != nil { + return "", g.Error(err, "unable to download agent-browser binary") + } + if err = os.Rename(tmpPath, binPath); err != nil { + return "", g.Error(err, "could not move agent-browser binary to %s", binPath) + } + if err = os.Chmod(binPath, 0755); err != nil { + return "", g.Error(err, "could not make agent-browser executable") + } + } + + ok, err := a.versionOK(binPath) + if err != nil { + return "", err + } + if !ok { + return "", g.Error("agent-browser at %s does not report version %s", binPath, a.version) + } + return binPath, nil +} + +func maybeEnsureAgentBrowser(opts InstallOptions) error { + return newAgentBrowserInstall().maybeEnsure(opts) +} + +func (a *agentBrowserInstall) maybeEnsure(opts InstallOptions) error { + if skipAgentBrowserDownload() { + return nil + } + bin, err := a.ensure() + if err != nil { + return err + } + return a.maybeInstallChrome(bin, opts) +} + +func (a *agentBrowserInstall) chromeLikelyPresent() bool { + if g.PathExists(filepath.Join(userHome(), ".agent-browser", "browsers")) { + return true + } + for _, name := range []string{"google-chrome", "google-chrome-stable", "chromium", "chromium-browser", "chrome"} { + if _, err := exec.LookPath(name); err == nil { + return true + } + } + switch runtime.GOOS { + case "darwin": + return g.PathExists("/Applications/Google Chrome.app") + case "windows": + for _, p := range []string{ + filepath.Join(os.Getenv("PROGRAMFILES"), "Google", "Chrome", "Application", "chrome.exe"), + filepath.Join(os.Getenv("PROGRAMFILES(X86)"), "Google", "Chrome", "Application", "chrome.exe"), + } { + if p != "" && g.PathExists(p) { + return true + } + } + } + return false +} + +func (a *agentBrowserInstall) maybeInstallChrome(bin string, opts InstallOptions) error { + if os.Getenv("AGENT_BROWSER_SKIP_CHROME") == "1" { + return nil + } + if a.chromeLikelyPresent() { + return nil + } + yes := os.Getenv("SLING_AGENT_BROWSER_YES") == "1" + if opts.NonInteractive && !yes { + g.Info("Chrome for Testing is not installed. After setup, run: %s install", bin) + return nil + } + if !yes && !env.IsInteractiveTerminal() { + g.Info("Chrome for Testing is not installed. After setup, run: %s install", bin) + return nil + } + g.Info("downloading Chrome for Testing via agent-browser install") + cmd := exec.Command(bin, "install") + cmd.Stdout = os.Stderr + cmd.Stderr = os.Stderr + if err := cmd.Run(); err != nil { + g.Warn("agent-browser install failed: %v (MCP is still wired; run `%s install` later)", err, bin) + } + return nil +} + +func tomlQuote(s string) string { + return strconv.Quote(s) +} + +func tomlStringArray(ss []string) string { + parts := make([]string, len(ss)) + for i, s := range ss { + parts[i] = tomlQuote(s) + } + return "[" + strings.Join(parts, ", ") + "]" +} + +func writeJSONMCP(path, serversKey string) error { + if err := setJSONPath(path, serversKey+".sling", slingMCPEntry()); err != nil { + return err + } + return setJSONPath(path, serversKey+".agent-browser", agentBrowserMCPEntry()) +} + +func removeJSONMCP(path, serversKey string) error { + if err := deleteJSONPath(path, serversKey+".sling"); err != nil { + return err + } + return deleteJSONPath(path, serversKey+".agent-browser") +} diff --git a/core/sling/assist/browser_test.go b/core/sling/assist/browser_test.go new file mode 100644 index 000000000..83c410550 --- /dev/null +++ b/core/sling/assist/browser_test.go @@ -0,0 +1,182 @@ +package assist + +import ( + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "testing" +) + +func isolateAgentBrowserEnv(t *testing.T) { + t.Helper() + home := withTempHomeDir(t) + bin := filepath.Join(home, "empty-bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin) + t.Setenv("AGENT_BROWSER_PATH", "") + t.Setenv("AGENT_BROWSER_VERSION", AgentBrowserVersion) + t.Setenv("AGENT_BROWSER_SKIP_DOWNLOAD", "") + t.Setenv("AGENT_BROWSER_SKIP_CHROME", "1") + t.Cleanup(func() { agentBrowserTestDownloadURL = "" }) +} + +func stubAgentBrowserScript(version string) string { + return "#!/bin/sh\n" + + "if [ \"$1\" = \"--version\" ]; then echo \"" + version + "\"; exit 0; fi\n" + + "if [ \"$1\" = \"install\" ]; then echo chrome-stub; exit 0; fi\n" + + "echo stub\n" +} + +func serveAgentBrowserBin(t *testing.T, version string) (*httptest.Server, *atomic.Int32) { + t.Helper() + payload := []byte(stubAgentBrowserScript(version)) + hits := &atomic.Int32{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + _, _ = io.ReadAll(r.Body) + w.Header().Set("Content-Type", "application/octet-stream") + _, _ = w.Write(payload) + })) + t.Cleanup(srv.Close) + return srv, hits +} + +func TestAgentBrowserAssetNamePinnedPlatforms(t *testing.T) { + cases := map[string]string{ + "darwin/arm64": "agent-browser-darwin-arm64", + "darwin/amd64": "agent-browser-darwin-x64", + "linux/arm64": "agent-browser-linux-arm64", + "linux/amd64": "agent-browser-linux-x64", + "windows/amd64": "agent-browser-win32-x64.exe", + "windows/arm64": "agent-browser-win32-x64.exe", + } + for plat, want := range cases { + if strings.HasPrefix(plat, "linux/") && linuxMuslPresent() { + continue + } + parts := strings.Split(plat, "/") + got, err := AgentBrowserAssetName(parts[0], parts[1]) + if err != nil { + t.Fatalf("%s: %v", plat, err) + } + if got != want { + t.Errorf("%s: got %s want %s", plat, got, want) + } + } +} + +func TestAgentBrowserAssetNameUnsupported(t *testing.T) { + if _, err := AgentBrowserAssetName("plan9", "amd64"); err == nil { + t.Fatal("expected error for plan9") + } +} + +func TestEnsureBinAgentBrowserDownloadsOnce(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("stub is a POSIX shell script") + } + isolateAgentBrowserEnv(t) + srv, hits := serveAgentBrowserBin(t, AgentBrowserVersion) + agentBrowserTestDownloadURL = srv.URL + + p1, err := EnsureBinAgentBrowser() + if err != nil { + t.Fatal(err) + } + if !strings.HasSuffix(p1, agentBrowserBinName()) { + t.Fatalf("path=%s", p1) + } + p2, err := EnsureBinAgentBrowser() + if err != nil { + t.Fatal(err) + } + if p1 != p2 { + t.Fatalf("path changed %s -> %s", p1, p2) + } + if hits.Load() != 1 { + t.Fatalf("downloads=%d want 1", hits.Load()) + } +} + +func TestEnsureBinAgentBrowserPathEnv(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("stub is a POSIX shell script") + } + isolateAgentBrowserEnv(t) + dir := t.TempDir() + stub := filepath.Join(dir, "agent-browser") + if err := os.WriteFile(stub, []byte(stubAgentBrowserScript(AgentBrowserVersion)), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("AGENT_BROWSER_PATH", stub) + got, err := EnsureBinAgentBrowser() + if err != nil { + t.Fatal(err) + } + if got != stub { + t.Fatalf("got %s want %s", got, stub) + } +} + +func TestEnsureBinAgentBrowserSkipDownload(t *testing.T) { + isolateAgentBrowserEnv(t) + t.Setenv("AGENT_BROWSER_SKIP_DOWNLOAD", "1") + got, err := EnsureBinAgentBrowser() + if err != nil { + t.Fatal(err) + } + if got != "agent-browser" { + t.Fatalf("got %s", got) + } +} + +func TestAgentBrowserSkillEmbedded(t *testing.T) { + names := listSkillNames() + found := false + for _, n := range names { + if n == "agent-browser" { + found = true + break + } + } + if !found { + t.Fatalf("listSkillNames missing agent-browser: %v", names) + } + stub, err := SkillsFS.ReadFile("skills/agent-browser/SKILL.md") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(stub), "CORE.md") { + t.Fatal("SKILL.md must point at CORE.md") + } + if !strings.Contains(string(stub), "sling assist setup") { + t.Fatal("SKILL.md must tell the agent about sling assist setup") + } + core, err := SkillsFS.ReadFile("skills/agent-browser/CORE.md") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(core), "The core loop") { + t.Fatal("CORE.md missing official core loop") + } +} + +func TestAgentBrowserMCPEntryUsesResolvedBin(t *testing.T) { + isolateAgentBrowserEnv(t) + t.Setenv("AGENT_BROWSER_PATH", "/tmp/custom-agent-browser") + entry := agentBrowserMCPEntry() + if entry["command"] != "/tmp/custom-agent-browser" { + t.Fatalf("command=%v", entry["command"]) + } + args, _ := entry["args"].([]any) + if len(args) != 3 || args[0] != "mcp" || args[1] != "--tools" || args[2] != "core" { + t.Fatalf("args=%v", args) + } +} diff --git a/core/sling/assist/clients.go b/core/sling/assist/clients.go new file mode 100644 index 000000000..6870e2a8d --- /dev/null +++ b/core/sling/assist/clients.go @@ -0,0 +1,1404 @@ +package assist + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync" + "time" + + "github.com/flarco/g" + "gopkg.in/yaml.v3" +) + +// ClientKind distinguishes launchable CLI agents from install-only UI surfaces. +// Only Kind() == CLIAgent clients are picked by the assist launcher. +type ClientKind int + +const ( + KindCLIAgent ClientKind = iota + KindUISurface +) + +// Client is the per-tool install adapter contract. Each adapter takes care of +// its own redirect/translation of the canonical skills bundle and its own MCP +// config shape. Install/uninstall/doctor iterate over Detected() clients. +// +// Methods that touch the filesystem accept context.Context for cancellation +// (submission/network work will share this seam). Implementations should +// honor ctx.Err() at entry when doing I/O. +type Client interface { + Name() string + Kind() ClientKind + Detect() bool + WriteSkills(ctx context.Context, skillNames []string, scope Scope) error + RemoveSkills(ctx context.Context, skillNames []string, scope Scope) error + WriteMCP(ctx context.Context, scope Scope) error + RemoveMCP(ctx context.Context, scope Scope) error + // CheckSkills returns one typed result per skill (no glyph prefixes). + CheckSkills(ctx context.Context, skillNames []string, scope Scope) []CheckResult + // CheckMCP returns a typed MCP wiring status. + CheckMCP(ctx context.Context, scope Scope) CheckResult + // AuthState reports offline credential-file / env-key presence. Never launches a binary. + AuthState() AuthStatus +} + +// Scope is `--scope user` (default) or `--scope project`. +type Scope int + +const ( + ScopeUser Scope = iota + ScopeProject +) + +// AllClients returns the canonical ordered list of supported clients. +// Order matters for `agent: auto` resolution and for stable output. +func AllClients() []Client { + return []Client{ + &claudeClient{}, + &codexClient{}, + &geminiClient{}, + &cursorClient{}, + &opencodeClient{}, + &piClient{}, + &grokClient{}, + &vscodeClient{}, + } +} + +// CLIAgents returns only the launchable CLI-agent clients (excludes vscode). +func CLIAgents() []Client { + out := []Client{} + for _, c := range AllClients() { + if c.Kind() == KindCLIAgent { + out = append(out, c) + } + } + return out +} + +// DetectedClients returns the subset of AllClients() whose Detect() returned true. +func DetectedClients() []Client { + out := []Client{} + for _, c := range AllClients() { + if c.Detect() { + out = append(out, c) + } + } + return out +} + +// LookupClient finds a client by name; returns nil if no match. +func LookupClient(name string) Client { + name = strings.ToLower(strings.TrimSpace(name)) + for _, c := range AllClients() { + if c.Name() == name { + return c + } + } + return nil +} + +// canonicalSkillPath returns the absolute path to a skill's SKILL.md inside +// the canonical bundle. +func canonicalSkillPath(skill string) string { + return filepath.Join(CanonicalSkillsDir(), skill, "SKILL.md") +} + +// writeRedirectFile writes a 1-line `@` redirect file. +// If symlinks are preferred (Unix), we still write a stub file so doctor's +// "resolves to canonical" check is uniform across platforms. +func writeRedirectFile(redirectPath, canonicalPath string) error { + if err := os.MkdirAll(filepath.Dir(redirectPath), 0o755); err != nil { + return g.Error(err, "mkdir %s", filepath.Dir(redirectPath)) + } + body := fmt.Sprintf("@%s\n", canonicalPath) + return os.WriteFile(redirectPath, []byte(body), 0o644) +} + +// readRedirectTarget parses a 1-line `@` redirect; returns "" if the +// file isn't a redirect (or isn't readable). +func readRedirectTarget(redirectPath string) string { + data, err := os.ReadFile(redirectPath) + if err != nil { + return "" + } + line := strings.TrimSpace(string(data)) + if !strings.HasPrefix(line, "@") { + return "" + } + // drop the @, take only the first line if there are several + target := strings.TrimSpace(strings.SplitN(line[1:], "\n", 2)[0]) + return target +} + +// listSkillNames walks the embedded skills FS and returns canonical names +// (one per top-level directory). Skill bundles with multiple files (like +// sling-api-specs) still come back as one name. +func listSkillNames() []string { + entries, err := SkillsFS.ReadDir("skills") + if err != nil { + return nil + } + out := []string{} + for _, e := range entries { + if e.IsDir() { + out = append(out, e.Name()) + } + } + return out +} + +// writeCanonicalBundle copies the embedded skills tree to ~/.agents/skills/. +// Existing files are overwritten — skills are Sling-owned (see design doc). +// Stale files removed from the embed are deleted from disk. +func writeCanonicalBundle(skillNames []string) error { + for _, name := range skillNames { + if _, err := syncCanonicalSkill(name); err != nil { + return err + } + } + return nil +} + +// syncCanonicalSkill writes one skill from the embed FS onto the canonical +// disk tree and removes on-disk files under that skill that are no longer +// embedded. Shared by writeCanonicalBundle and AutoRefresh. +func syncCanonicalSkill(name string) (changed bool, err error) { + root := CanonicalSkillsDir() + embeddedRoot := filepath.ToSlash(filepath.Join("skills", name)) + wantFiles := map[string]bool{} // path relative to root, slash-separated + + err = fs.WalkDir(SkillsFS, embeddedRoot, func(p string, d fs.DirEntry, werr error) error { + if werr != nil { + return werr + } + if d.IsDir() { + return nil + } + rel := strings.TrimPrefix(p, "skills/") + wantFiles[rel] = true + dst := filepath.Join(root, filepath.FromSlash(rel)) + want, rerr := SkillsFS.ReadFile(p) + if rerr != nil { + return g.Error(rerr, "read embedded %s", p) + } + got, _ := os.ReadFile(dst) + if bytes.Equal(want, got) { + return nil + } + if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil { + return g.Error(err, "mkdir %s", filepath.Dir(dst)) + } + if err := os.WriteFile(dst, want, 0o644); err != nil { + return g.Error(err, "write %s", dst) + } + changed = true + return nil + }) + if err != nil { + return changed, err + } + + // Prune files on disk that are no longer in the embed (stale supporting docs). + skillDir := filepath.Join(root, name) + if g.PathExists(skillDir) { + _ = filepath.WalkDir(skillDir, func(p string, d fs.DirEntry, werr error) error { + if werr != nil || d.IsDir() { + return werr + } + rel, rerr := filepath.Rel(root, p) + if rerr != nil { + return nil + } + rel = filepath.ToSlash(rel) + if !wantFiles[rel] { + if rmErr := os.Remove(p); rmErr == nil { + changed = true + } + } + return nil + }) + } + return changed, nil +} + +// skillMatchesEmbedded compares every embedded file for a skill against disk. +// Returns ok=false with a short detail when any file is missing or drifted. +func skillMatchesEmbedded(name string) (ok bool, detail string, err error) { + root := CanonicalSkillsDir() + embeddedRoot := filepath.ToSlash(filepath.Join("skills", name)) + var mismatches []string + err = fs.WalkDir(SkillsFS, embeddedRoot, func(p string, d fs.DirEntry, werr error) error { + if werr != nil { + return werr + } + if d.IsDir() { + return nil + } + rel := strings.TrimPrefix(p, "skills/") + dst := filepath.Join(root, filepath.FromSlash(rel)) + want, rerr := SkillsFS.ReadFile(p) + if rerr != nil { + return rerr + } + got, rerr := os.ReadFile(dst) + if rerr != nil { + mismatches = append(mismatches, rel+": missing") + return nil + } + if !bytes.Equal(want, got) { + mismatches = append(mismatches, rel+": drifted") + } + return nil + }) + if err != nil { + return false, "", err + } + if len(mismatches) > 0 { + return false, strings.Join(mismatches, "; "), nil + } + return true, "", nil +} + +// removeCanonicalBundle removes ~/.agents/skills/sling-* directories — only +// our own skills, never anyone else's. +func removeCanonicalBundle(skillNames []string) error { + root := CanonicalSkillsDir() + var errs []string + for _, name := range skillNames { + p := filepath.Join(root, name) + if err := os.RemoveAll(p); err != nil { + errs = append(errs, fmt.Sprintf("%s: %v", name, err)) + } + } + if len(errs) > 0 { + return g.Error("remove canonical skills: %s", strings.Join(errs, "; ")) + } + return nil +} + +// slingMCPEntry is the canonical Sling MCP server descriptor written into +// every client's MCP config. Centralized so each adapter renders the same +// shape and a future schema bump is one-line. +func slingMCPEntry() map[string]any { + return map[string]any{ + "command": "sling", + "args": []any{"serve", "mcp"}, + } +} + +// checkCanonicalSkills is the CheckSkills implementation shared by every +// client that reads ~/.agents/skills/ natively (codex, gemini, opencode, pi, +// grok) — there's no per-client redirect to verify, only the bundle itself. +func checkCanonicalSkills(skillNames []string) []CheckResult { + out := []CheckResult{} + for _, name := range skillNames { + if g.PathExists(canonicalSkillPath(name)) { + out = append(out, checkSkill(CellOK, name, "canonical")) + } else { + out = append(out, checkSkill(CellFail, name, "missing in canonical bundle")) + } + } + return out +} + +// checkMCPServersKey reports whether has sling and agent-browser +// entries under the given top-level object key ("mcpServers", "servers", "mcp"). +func checkMCPServersKey(path, key string) CheckResult { + doc, err := jsonReadOrEmpty(path) + if err != nil { + return checkFail(err.Error()) + } + servers, _ := doc[key].(map[string]any) + if servers == nil { + return checkFail("no " + key + " block") + } + if _, ok := servers["sling"]; !ok { + return checkFail("sling MCP entry missing") + } + if _, ok := servers[agentBrowserMCPName]; !ok { + return checkFail("agent-browser MCP entry missing") + } + return checkOK("sling and agent-browser MCP present") +} + +// ---- claude ---- + +type claudeClient struct{} + +func (c *claudeClient) Name() string { return "claude" } +func (c *claudeClient) Kind() ClientKind { return KindCLIAgent } +func (c *claudeClient) Detect() bool { return commandOnPath("claude") } + +func (c *claudeClient) skillsRoot(scope Scope) string { + if scope == ScopeProject { + return projectPath(".claude", "skills") + } + return filepath.Join(userHome(), ".claude", "skills") +} + +func (c *claudeClient) mcpPath(scope Scope) string { + // Project-scoped MCP lives in .mcp.json at the project root (Claude Code + // convention). User/local scope stays in ~/.claude.json. + if scope == ScopeProject { + return projectPath(".mcp.json") + } + return filepath.Join(userHome(), ".claude.json") +} + +func (c *claudeClient) WriteSkills(ctx context.Context, skillNames []string, scope Scope) error { + root := c.skillsRoot(scope) + for _, name := range skillNames { + canonical := canonicalSkillPath(name) + redirect := filepath.Join(root, name, "SKILL.md") + if err := writeRedirectFile(redirect, canonical); err != nil { + return err + } + } + return nil +} + +func (c *claudeClient) RemoveSkills(ctx context.Context, skillNames []string, scope Scope) error { + root := c.skillsRoot(scope) + for _, name := range skillNames { + _ = os.RemoveAll(filepath.Join(root, name)) + } + return nil +} + +func (c *claudeClient) WriteMCP(ctx context.Context, scope Scope) error { + return writeJSONMCP(c.mcpPath(scope), "mcpServers") +} + +func (c *claudeClient) RemoveMCP(ctx context.Context, scope Scope) error { + return removeJSONMCP(c.mcpPath(scope), "mcpServers") +} + +func (c *claudeClient) CheckSkills(ctx context.Context, skillNames []string, scope Scope) []CheckResult { + out := []CheckResult{} + root := c.skillsRoot(scope) + for _, name := range skillNames { + redirect := filepath.Join(root, name, "SKILL.md") + canonical := canonicalSkillPath(name) + target := readRedirectTarget(redirect) + switch { + case target == "": + out = append(out, checkSkill(CellFail, name, "redirect missing")) + case target != canonical: + out = append(out, checkSkill(CellFail, name, fmt.Sprintf("points at %s (expected %s)", target, canonical))) + case !g.PathExists(canonical): + out = append(out, checkSkill(CellFail, name, "redirect target not found")) + default: + out = append(out, checkSkill(CellOK, name, "")) + } + } + return out +} + +func (c *claudeClient) CheckMCP(ctx context.Context, scope Scope) CheckResult { + return checkMCPServersKey(c.mcpPath(scope), "mcpServers") +} + +// ---- codex (reads ~/.agents/skills/ natively) ---- + +type codexClient struct{} + +func (c *codexClient) Name() string { return "codex" } +func (c *codexClient) Kind() ClientKind { return KindCLIAgent } +func (c *codexClient) Detect() bool { return commandOnPath("codex") } + +func (c *codexClient) WriteSkills(ctx context.Context, skillNames []string, scope Scope) error { + // Codex reads ~/.agents/skills/ natively — nothing extra to do. + return nil +} +func (c *codexClient) RemoveSkills(ctx context.Context, skillNames []string, scope Scope) error { + return nil +} + +func (c *codexClient) codexConfigPath(scope Scope) string { + if scope == ScopeProject { + return projectPath(".codex", "config.toml") + } + return filepath.Join(userHome(), ".codex", "config.toml") +} + +func (c *codexClient) WriteMCP(ctx context.Context, scope Scope) error { + path := c.codexConfigPath(scope) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return g.Error(err, "mkdir %s", filepath.Dir(path)) + } + if err := backupBeforeEdit(path); err != nil { + return err + } + body, _ := os.ReadFile(path) + merged := upsertCodexMCP(string(body)) + return writeBytesPreserveMode(path, []byte(merged), 0o600) +} + +func (c *codexClient) RemoveMCP(ctx context.Context, scope Scope) error { + path := c.codexConfigPath(scope) + body, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if err := backupBeforeEdit(path); err != nil { + return err + } + out := removeCodexMCP(string(body)) + return writeBytesPreserveMode(path, []byte(out), 0o600) +} + +func (c *codexClient) CheckSkills(ctx context.Context, skillNames []string, scope Scope) []CheckResult { + return checkCanonicalSkills(skillNames) +} + +func (c *codexClient) CheckMCP(ctx context.Context, scope Scope) CheckResult { + body, err := os.ReadFile(c.codexConfigPath(scope)) + if err != nil { + return checkFail("config.toml not present") + } + if hasCodexMCPSection(string(body)) && hasCodexNamedMCPSection(string(body), agentBrowserMCPName) { + return checkOK("sling and agent-browser MCP present") + } + if !hasCodexMCPSection(string(body)) { + return checkFail("sling MCP entry missing") + } + return checkFail("agent-browser MCP entry missing") +} + +const codexMCPHeader = "[mcp_servers.sling]" + +func codexMCPHeaderNamed(name string) string { + return "[mcp_servers." + name + "]" +} + +// isCodexSlingTable reports whether a trimmed TOML header line is our sling +// table or a nested subtable ([mcp_servers.sling.env], etc.). +func isCodexSlingTable(trim string) bool { + return isCodexNamedMCPTable(trim, "sling") +} + +func isCodexNamedMCPTable(trim, name string) bool { + header := codexMCPHeaderNamed(name) + return trim == header || strings.HasPrefix(trim, header[:len(header)-1]+".") +} + +// hasCodexMCPSection reports whether body has a real [mcp_servers.sling] +// table header (line-anchored), not a mention inside a string or comment. +func hasCodexMCPSection(body string) bool { + return hasCodexNamedMCPSection(body, "sling") +} + +func hasCodexNamedMCPSection(body, name string) bool { + header := codexMCPHeaderNamed(name) + for _, line := range strings.Split(body, "\n") { + if strings.TrimSpace(line) == header { + return true + } + } + return false +} + +func upsertCodexNamedMCP(body, name, command string, args []string) string { + header := codexMCPHeaderNamed(name) + block := header + "\ncommand = " + tomlQuote(command) + "\nargs = " + tomlStringArray(args) + "\n" + if !hasCodexNamedMCPSection(body, name) { + if body != "" && !strings.HasSuffix(body, "\n") { + body += "\n" + } + return body + "\n" + block + } + lines := strings.Split(body, "\n") + out := []string{} + skipping := false + injected := false + for _, line := range lines { + trim := strings.TrimSpace(line) + if trim == header { + out = append(out, strings.TrimRight(block, "\n")) + skipping = true + injected = true + continue + } + if skipping { + if strings.HasPrefix(trim, "[") { + if isCodexNamedMCPTable(trim, name) { + continue + } + skipping = false + } else { + continue + } + } + out = append(out, line) + } + if !injected { + out = append(out, "", strings.TrimRight(block, "\n")) + } + return strings.Join(out, "\n") +} + +// upsertCodexMCP injects/replaces sling and agent-browser MCP tables. +func upsertCodexMCP(body string) string { + body = upsertCodexNamedMCP(body, "sling", "sling", []string{"serve", "mcp"}) + return upsertCodexNamedMCP(body, agentBrowserMCPName, agentBrowserBin(), []string{"mcp", "--tools", "core"}) +} + +func removeCodexNamedMCP(body, name string) string { + lines := strings.Split(body, "\n") + out := []string{} + skipping := false + for _, line := range lines { + trim := strings.TrimSpace(line) + if isCodexNamedMCPTable(trim, name) { + skipping = true + continue + } + if skipping { + if strings.HasPrefix(trim, "[") { + if isCodexNamedMCPTable(trim, name) { + continue + } + skipping = false + } else { + continue + } + } + out = append(out, line) + } + return strings.Join(out, "\n") +} + +// removeCodexMCP strips sling and agent-browser MCP tables (and nested subtables). +func removeCodexMCP(body string) string { + body = removeCodexNamedMCP(body, "sling") + return removeCodexNamedMCP(body, agentBrowserMCPName) +} + +// ---- gemini (reads ~/.agents/skills/ as alias) ---- + +type geminiClient struct{} + +func (c *geminiClient) Name() string { return "gemini" } +func (c *geminiClient) Kind() ClientKind { return KindCLIAgent } +func (c *geminiClient) Detect() bool { return commandOnPath("gemini") } +func (c *geminiClient) WriteSkills(ctx context.Context, skillNames []string, scope Scope) error { + return nil +} +func (c *geminiClient) RemoveSkills(ctx context.Context, skillNames []string, scope Scope) error { + return nil +} + +func (c *geminiClient) settingsPath(scope Scope) string { + if scope == ScopeProject { + return projectPath(".gemini", "settings.json") + } + return filepath.Join(userHome(), ".gemini", "settings.json") +} + +func (c *geminiClient) WriteMCP(ctx context.Context, scope Scope) error { + return writeJSONMCP(c.settingsPath(scope), "mcpServers") +} + +func (c *geminiClient) RemoveMCP(ctx context.Context, scope Scope) error { + return removeJSONMCP(c.settingsPath(scope), "mcpServers") +} + +func (c *geminiClient) CheckSkills(ctx context.Context, skillNames []string, scope Scope) []CheckResult { + return checkCanonicalSkills(skillNames) +} + +func (c *geminiClient) CheckMCP(ctx context.Context, scope Scope) CheckResult { + return checkMCPServersKey(c.settingsPath(scope), "mcpServers") +} + +// ---- cursor (translates to .mdc) ---- + +type cursorClient struct{} + +func (c *cursorClient) Name() string { return "cursor" } +func (c *cursorClient) Kind() ClientKind { return KindCLIAgent } +func (c *cursorClient) Detect() bool { return commandOnPath(agentBinary("cursor")) } + +func (c *cursorClient) rulesDir(scope Scope) string { + if scope == ScopeProject { + return projectPath(".cursor", "rules") + } + return filepath.Join(userHome(), ".cursor", "rules") +} + +func (c *cursorClient) mcpPath(scope Scope) string { + if scope == ScopeProject { + return projectPath(".cursor", "mcp.json") + } + return filepath.Join(userHome(), ".cursor", "mcp.json") +} + +// translateSkillToMDC produces a cursor `.mdc` file body from a SKILL.md. +// Cursor's `.mdc` shape is YAML frontmatter (with `description`, +// `globs`, `alwaysApply`) + Markdown body — same shape as SKILL.md, so we +// pass through with a lightly-rewritten frontmatter. +func translateSkillToMDC(skillBody []byte) []byte { + body := string(skillBody) + // SKILL.md frontmatter looks like: + // --- + // name: sling + // description: ... + // --- + // Cursor's .mdc wants: + // --- + // description: ... + // alwaysApply: false + // --- + if !strings.HasPrefix(body, "---") { + return []byte("---\nalwaysApply: false\n---\n\n" + body) + } + parts := strings.SplitN(body, "\n---", 2) + if len(parts) != 2 { + return []byte("---\nalwaysApply: false\n---\n\n" + body) + } + header := strings.TrimPrefix(parts[0], "---\n") + rest := strings.TrimPrefix(parts[1], "\n") + + var fm map[string]any + _ = yaml.Unmarshal([]byte(header), &fm) + if fm == nil { + fm = map[string]any{} + } + out := map[string]any{ + "description": fm["description"], + "alwaysApply": false, + } + yamlBytes, _ := yaml.Marshal(out) + return []byte("---\n" + string(yamlBytes) + "---\n\n" + rest) +} + +func (c *cursorClient) WriteSkills(ctx context.Context, skillNames []string, scope Scope) error { + dir := c.rulesDir(scope) + if err := os.MkdirAll(dir, 0o755); err != nil { + return g.Error(err, "mkdir %s", dir) + } + for _, name := range skillNames { + src := canonicalSkillPath(name) + body, err := os.ReadFile(src) + if err != nil { + return g.Error(err, "read %s", src) + } + mdc := translateSkillToMDC(body) + dst := filepath.Join(dir, name+".mdc") + if err := os.WriteFile(dst, mdc, 0o644); err != nil { + return g.Error(err, "write %s", dst) + } + } + return nil +} + +func (c *cursorClient) RemoveSkills(ctx context.Context, skillNames []string, scope Scope) error { + dir := c.rulesDir(scope) + for _, name := range skillNames { + _ = os.Remove(filepath.Join(dir, name+".mdc")) + } + return nil +} + +func (c *cursorClient) WriteMCP(ctx context.Context, scope Scope) error { + return writeJSONMCP(c.mcpPath(scope), "mcpServers") +} + +func (c *cursorClient) RemoveMCP(ctx context.Context, scope Scope) error { + return removeJSONMCP(c.mcpPath(scope), "mcpServers") +} + +func (c *cursorClient) CheckSkills(ctx context.Context, skillNames []string, scope Scope) []CheckResult { + out := []CheckResult{} + for _, name := range skillNames { + mdc := filepath.Join(c.rulesDir(scope), name+".mdc") + if g.PathExists(mdc) { + out = append(out, checkSkill(CellOK, name, "mdc present")) + } else { + out = append(out, checkSkill(CellFail, name, "mdc missing")) + } + } + return out +} + +func (c *cursorClient) CheckMCP(ctx context.Context, scope Scope) CheckResult { + return checkMCPServersKey(c.mcpPath(scope), "mcpServers") +} + +// ---- opencode (reads ~/.agents/skills/ natively) ---- + +type opencodeClient struct{} + +func (c *opencodeClient) Name() string { return "opencode" } +func (c *opencodeClient) Kind() ClientKind { return KindCLIAgent } +func (c *opencodeClient) Detect() bool { + return commandOnPath("opencode") || g.PathExists(BundledOpenCodePath()) +} + +// opencodeConfigDir is opencode's global config dir: $XDG_CONFIG_HOME/opencode +// falling back to ~/.config/opencode. +func opencodeConfigDir() string { + if xdg := os.Getenv("XDG_CONFIG_HOME"); xdg != "" { + return filepath.Join(xdg, "opencode") + } + return filepath.Join(userHome(), ".config", "opencode") +} + +func (c *opencodeClient) configPath(scope Scope) string { + if scope == ScopeProject { + return projectPath("opencode.json") + } + return filepath.Join(opencodeConfigDir(), "opencode.json") +} + +func (c *opencodeClient) WriteSkills(ctx context.Context, skillNames []string, scope Scope) error { + // opencode globs ~/.agents/skills//SKILL.md natively. + return nil +} +func (c *opencodeClient) RemoveSkills(ctx context.Context, skillNames []string, scope Scope) error { + return nil +} + +// opencodeMCPEntry is opencode's own MCP shape: `type: local` plus a single +// argv array (not command/args like everyone else). +func opencodeMCPEntry() map[string]any { + return map[string]any{ + "type": "local", + "command": []any{"sling", "serve", "mcp"}, + "enabled": true, + } +} + +func (c *opencodeClient) WriteMCP(ctx context.Context, scope Scope) error { + if err := setJSONPath(c.configPath(scope), "mcp.sling", opencodeMCPEntry()); err != nil { + return err + } + return setJSONPath(c.configPath(scope), "mcp.agent-browser", opencodeAgentBrowserMCPEntry()) +} + +func (c *opencodeClient) RemoveMCP(ctx context.Context, scope Scope) error { + if err := deleteJSONPath(c.configPath(scope), "mcp.sling"); err != nil { + return err + } + return deleteJSONPath(c.configPath(scope), "mcp.agent-browser") +} + +func (c *opencodeClient) CheckSkills(ctx context.Context, skillNames []string, scope Scope) []CheckResult { + return checkCanonicalSkills(skillNames) +} + +func (c *opencodeClient) CheckMCP(ctx context.Context, scope Scope) CheckResult { + return checkMCPServersKey(c.configPath(scope), "mcp") +} + +// ---- pi (reads ~/.agents/skills/ natively) ---- + +type piClient struct{} + +func (c *piClient) Name() string { return "pi" } +func (c *piClient) Kind() ClientKind { return KindCLIAgent } +func (c *piClient) Detect() bool { return commandOnPath("pi") } + +// piAgentDir is pi's agent config dir — $PI_CODING_AGENT_DIR when set, +// otherwise ~/.pi/agent. +func piAgentDir() string { + if d := os.Getenv("PI_CODING_AGENT_DIR"); d != "" { + return d + } + return filepath.Join(userHome(), ".pi", "agent") +} + +// mcpPath: pi keeps MCP servers in a dedicated mcp.json, separate from +// settings.json. Project scope is .pi/mcp.json at the repo root. +func (c *piClient) mcpPath(scope Scope) string { + if scope == ScopeProject { + return projectPath(".pi", "mcp.json") + } + return filepath.Join(piAgentDir(), "mcp.json") +} + +func (c *piClient) WriteSkills(ctx context.Context, skillNames []string, scope Scope) error { + // pi discovers ~/.agents/skills//SKILL.md natively. + return nil +} +func (c *piClient) RemoveSkills(ctx context.Context, skillNames []string, scope Scope) error { + return nil +} + +func (c *piClient) WriteMCP(ctx context.Context, scope Scope) error { + return writeJSONMCP(c.mcpPath(scope), "mcpServers") +} + +func (c *piClient) RemoveMCP(ctx context.Context, scope Scope) error { + return removeJSONMCP(c.mcpPath(scope), "mcpServers") +} + +func (c *piClient) CheckSkills(ctx context.Context, skillNames []string, scope Scope) []CheckResult { + return checkCanonicalSkills(skillNames) +} + +func (c *piClient) CheckMCP(ctx context.Context, scope Scope) CheckResult { + return checkMCPServersKey(c.mcpPath(scope), "mcpServers") +} + +// ---- grok (TOML config, reads ~/.agents/skills/ for AGENTS.md compat) ---- + +type grokClient struct{} + +func (c *grokClient) Name() string { return "grok" } +func (c *grokClient) Kind() ClientKind { return KindCLIAgent } +func (c *grokClient) Detect() bool { return commandOnPath("grok") } + +func (c *grokClient) configPath(scope Scope) string { + if scope == ScopeProject { + return projectPath(".grok", "config.toml") + } + return filepath.Join(userHome(), ".grok", "config.toml") +} + +func (c *grokClient) WriteSkills(ctx context.Context, skillNames []string, scope Scope) error { + // grok discovers ~/.agents/skills/ as part of its AGENTS.md compatibility. + return nil +} +func (c *grokClient) RemoveSkills(ctx context.Context, skillNames []string, scope Scope) error { + return nil +} + +func (c *grokClient) WriteMCP(ctx context.Context, scope Scope) error { + path := c.configPath(scope) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return g.Error(err, "mkdir %s", filepath.Dir(path)) + } + if err := backupBeforeEdit(path); err != nil { + return err + } + body, _ := os.ReadFile(path) + // grok's config.toml uses the same [mcp_servers.] shape as codex. + merged := upsertCodexMCP(string(body)) + return writeBytesPreserveMode(path, []byte(merged), 0o600) +} + +func (c *grokClient) RemoveMCP(ctx context.Context, scope Scope) error { + path := c.configPath(scope) + body, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + if err := backupBeforeEdit(path); err != nil { + return err + } + return writeBytesPreserveMode(path, []byte(removeCodexMCP(string(body))), 0o600) +} + +func (c *grokClient) CheckSkills(ctx context.Context, skillNames []string, scope Scope) []CheckResult { + return checkCanonicalSkills(skillNames) +} + +func (c *grokClient) CheckMCP(ctx context.Context, scope Scope) CheckResult { + body, err := os.ReadFile(c.configPath(scope)) + if err != nil { + return checkFail("config.toml not present") + } + if hasCodexMCPSection(string(body)) && hasCodexNamedMCPSection(string(body), agentBrowserMCPName) { + return checkOK("sling and agent-browser MCP present") + } + if !hasCodexMCPSection(string(body)) { + return checkFail("sling MCP entry missing") + } + return checkFail("agent-browser MCP entry missing") +} + +// ---- vscode (UI surface, install-only) ---- + +type vscodeClient struct{} + +func (c *vscodeClient) Name() string { return "vscode" } +func (c *vscodeClient) Kind() ClientKind { return KindUISurface } + +// vscodeUserDir returns the platform-specific VS Code user-config dir, or "". +// We probe Code, Code-Insiders, and VSCodium in that order. +func vscodeUserDir() string { + for _, name := range []string{"Code", "Code - Insiders", "VSCodium"} { + if p := vscodeUserDirNamed(name); p != "" { + return p + } + } + return "" +} + +func vscodeUserDirNamed(productName string) string { + var base string + switch runtime.GOOS { + case "darwin": + base = filepath.Join(userHome(), "Library", "Application Support", productName, "User") + case "windows": + base = filepath.Join(os.Getenv("APPDATA"), productName, "User") + default: + base = filepath.Join(userHome(), ".config", productName, "User") + } + if g.PathExists(base) { + return base + } + return "" +} + +func (c *vscodeClient) Detect() bool { return vscodeUserDir() != "" } + +func (c *vscodeClient) settingsPath() string { + dir := vscodeUserDir() + if dir == "" { + return "" + } + return filepath.Join(dir, "settings.json") +} + +func (c *vscodeClient) WriteSkills(ctx context.Context, skillNames []string, scope Scope) error { + path := c.settingsPath() + if path == "" { + return nil + } + // VS Code uses literal dotted keys, so the path must escape the dots. + const key = `chat\.instructionsFilesLocations` + canonical := CanonicalSkillsDir() + _, body, err := jsonReadRaw(path) + if err != nil { + return err + } + // Already present → no-op (avoid an unnecessary backup churn + write). + for _, e := range gjsonGetArrayStrings(body, key) { + if e == canonical { + return nil + } + } + // Route through setJSONPath so we get the backup + sanity-check guard. + return setJSONPath(path, key+".-1", canonical) +} + +func (c *vscodeClient) RemoveSkills(ctx context.Context, skillNames []string, scope Scope) error { + path := c.settingsPath() + if path == "" { + return nil + } + const key = `chat\.instructionsFilesLocations` + canonical := CanonicalSkillsDir() + _, body, err := jsonReadRaw(path) + if err != nil { + return err + } + cur := gjsonGetArrayStrings(body, key) + idx := -1 + for i, e := range cur { + if e == canonical { + idx = i + break + } + } + if idx < 0 { + return nil + } + return deleteJSONPath(path, fmt.Sprintf("%s.%d", key, idx)) +} + +// vscodeMCPPath returns the VS Code mcp.json for the given scope. +// User: /User/mcp.json; project: ./.vscode/mcp.json. +// VS Code reads MCP from mcp.json (servers.*), not settings.json. +func (c *vscodeClient) vscodeMCPPath(scope Scope) string { + if scope == ScopeProject { + return projectPath(".vscode", "mcp.json") + } + dir := vscodeUserDir() + if dir == "" { + return "" + } + return filepath.Join(dir, "mcp.json") +} + +func (c *vscodeClient) WriteMCP(ctx context.Context, scope Scope) error { + path := c.vscodeMCPPath(scope) + if path == "" { + return nil + } + // VS Code mcp.json uses top-level "servers", not "mcpServers". + return writeJSONMCP(path, "servers") +} + +func (c *vscodeClient) RemoveMCP(ctx context.Context, scope Scope) error { + path := c.vscodeMCPPath(scope) + if path == "" { + return nil + } + _ = deleteJSONPath(path, "servers.sling") + _ = deleteJSONPath(path, "servers.agent-browser") + // Clean the obsolete flat settings.json key from an earlier buggy path + // (github.copilot.chat.mcp.servers.sling) so doctor/settings stay tidy. + if settings := c.settingsPath(); settings != "" { + _ = deleteJSONPath(settings, `github\.copilot\.chat\.mcp\.servers\.sling`) + } + return nil +} + +func (c *vscodeClient) CheckSkills(ctx context.Context, skillNames []string, scope Scope) []CheckResult { + path := c.settingsPath() + if path == "" { + return []CheckResult{checkNA("no user config dir found")} + } + doc, err := jsonReadOrEmpty(path) + if err != nil { + return []CheckResult{checkFail(err.Error())} + } + canonical := CanonicalSkillsDir() + has := false + if locs, ok := doc["chat.instructionsFilesLocations"].([]any); ok { + for _, e := range locs { + if s, _ := e.(string); s == canonical { + has = true + break + } + } + } + // One aggregate result for all skills (vscode wires the canonical dir once). + if has { + return []CheckResult{checkOK("chat.instructionsFilesLocations includes canonical")} + } + return []CheckResult{checkFail("chat.instructionsFilesLocations missing canonical")} +} + +func (c *vscodeClient) CheckMCP(ctx context.Context, scope Scope) CheckResult { + path := c.vscodeMCPPath(scope) + if path == "" { + return checkNA("no user config dir found") + } + doc, err := jsonReadOrEmpty(path) + if err != nil { + return checkFail(err.Error()) + } + servers, _ := doc["servers"].(map[string]any) + if servers == nil { + return checkFail("no servers block in mcp.json") + } + if _, ok := servers["sling"]; !ok { + return checkFail("servers.sling missing in mcp.json") + } + if _, ok := servers[agentBrowserMCPName]; !ok { + return checkFail("servers.agent-browser missing in mcp.json") + } + return checkOK("mcp.json servers.sling and agent-browser present") +} + +// commandOnPath returns true if `name` resolves to an executable on $PATH. +// Uses exec.LookPath so Windows PATHEXT / .exe resolution works. +func commandOnPath(name string) bool { + _, err := exec.LookPath(name) + return err == nil +} + +// AuthStatus is an offline probe of whether a CLI agent looks signed in. +// Values: ok (credential file or env key present), none, unknown. +type AuthStatus string + +const ( + AuthOK AuthStatus = "ok" + AuthNone AuthStatus = "none" + AuthUnknown AuthStatus = "unknown" +) + +func fileNonEmpty(path string) bool { + st, err := os.Stat(path) + return err == nil && st.Size() > 0 +} + +func envNonEmpty(keys ...string) bool { + for _, k := range keys { + if os.Getenv(k) != "" { + return true + } + } + return false +} + +func claudeJSONHasOAuth(path string) bool { + b, err := os.ReadFile(path) + if err != nil { + return false + } + var doc map[string]any + if json.Unmarshal(b, &doc) != nil { + return false + } + v, ok := doc["oauthAccount"] + if !ok || v == nil { + return false + } + m, ok := v.(map[string]any) + return ok && len(m) > 0 +} + +func (c *claudeClient) AuthState() AuthStatus { + if envNonEmpty("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "CLAUDE_CODE_OAUTH_TOKEN") { + return AuthOK + } + if envNonEmpty("CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_USE_VERTEX", "CLAUDE_CODE_USE_FOUNDRY") { + return AuthOK + } + credDir := filepath.Join(userHome(), ".claude") + if d := strings.TrimSpace(os.Getenv("CLAUDE_CONFIG_DIR")); d != "" { + credDir = d + } + if fileNonEmpty(filepath.Join(credDir, ".credentials.json")) { + return AuthOK + } + if claudeJSONHasOAuth(filepath.Join(userHome(), ".claude.json")) { + return AuthOK + } + // macOS stores /login tokens in Keychain, not a file. + if runtime.GOOS == "darwin" { + return AuthUnknown + } + return AuthNone +} + +func (c *codexClient) AuthState() AuthStatus { + if envNonEmpty("OPENAI_API_KEY") { + return AuthOK + } + home := userHome() + if fileNonEmpty(filepath.Join(home, ".codex", "auth.json")) { + return AuthOK + } + if fileNonEmpty(filepath.Join(home, ".codex", "config.toml")) { + return AuthUnknown + } + return AuthNone +} + +func (c *geminiClient) AuthState() AuthStatus { + if envNonEmpty("GEMINI_API_KEY", "GOOGLE_API_KEY") { + return AuthOK + } + if p := os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"); p != "" && fileNonEmpty(p) { + return AuthOK + } + home := userHome() + if fileNonEmpty(filepath.Join(home, ".gemini", "oauth_creds.json")) || + fileNonEmpty(filepath.Join(home, ".gemini", "google_accounts.json")) { + return AuthOK + } + return AuthNone +} + +func (c *cursorClient) AuthState() AuthStatus { + if envNonEmpty("CURSOR_API_KEY") { + return AuthOK + } + // Browser login stores credentials in the OS keychain, not a file we can + // read, so ask the CLI. `cursor-agent status --format json` reports + // isAuthenticated and always exits 0 — trust the field, not the code. + if st, ok := cursorStatusAuth(); ok { + if st { + return AuthOK + } + return AuthNone + } + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + return AuthUnknown + } + return AuthNone +} + +// cursorStatusOnce caches the probe: doctor asks several times per run and +// each call spawns the CLI. +var cursorStatusOnce struct { + sync.Once + authed, answered bool +} + +// cursorStatusAuth runs `cursor-agent status --format json`. +// Returns (authenticated, true) when the CLI answered, (false, false) otherwise. +func cursorStatusAuth() (bool, bool) { + cursorStatusOnce.Do(func() { + cursorStatusOnce.authed, cursorStatusOnce.answered = probeCursorStatus() + }) + return cursorStatusOnce.authed, cursorStatusOnce.answered +} + +// resetCursorStatusCache clears the memoized probe. Tests use it when they +// swap the stub on PATH. +func resetCursorStatusCache() { + cursorStatusOnce.Once = sync.Once{} + cursorStatusOnce.authed, cursorStatusOnce.answered = false, false +} + +func probeCursorStatus() (bool, bool) { + bin, err := exec.LookPath(agentBinary("cursor")) + if err != nil { + return false, false + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, bin, "status", "--format", "json").Output() + if err != nil && len(out) == 0 { + return false, false + } + var doc struct { + IsAuthenticated *bool `json:"isAuthenticated"` + } + if json.Unmarshal(out, &doc) != nil || doc.IsAuthenticated == nil { + return false, false + } + return *doc.IsAuthenticated, true +} + +func (c *opencodeClient) AuthState() AuthStatus { + if envNonEmpty("OPENCODE_API_KEY", "ANTHROPIC_API_KEY", "OPENAI_API_KEY") { + return AuthOK + } + dataHome := os.Getenv("XDG_DATA_HOME") + if dataHome == "" { + dataHome = filepath.Join(userHome(), ".local", "share") + } + if fileNonEmpty(filepath.Join(dataHome, "opencode", "auth.json")) { + return AuthOK + } + if fileNonEmpty(filepath.Join(userHome(), ".opencode", "auth.json")) { + return AuthOK + } + return AuthNone +} + +func (c *piClient) AuthState() AuthStatus { + if envNonEmpty("PI_API_KEY", "OPENROUTER_API_KEY") { + return AuthOK + } + if fileNonEmpty(filepath.Join(piAgentDir(), "auth.json")) { + return AuthOK + } + return AuthNone +} + +func (c *grokClient) AuthState() AuthStatus { + if envNonEmpty("XAI_API_KEY", "GROK_API_KEY", "GROK_CODE_XAI_API_KEY") { + return AuthOK + } + if fileNonEmpty(filepath.Join(userHome(), ".grok", "auth.json")) { + return AuthOK + } + return AuthNone +} + +func (c *vscodeClient) AuthState() AuthStatus { + return AuthUnknown +} + +// YesNo is the install-summary token: yes / no / —. +func (s AuthStatus) YesNo() string { + switch s { + case AuthOK: + return "yes" + case AuthNone: + return "no" + default: + return "—" + } +} + +// cell is the doctor-matrix cell. Missing auth is not a failure. +func (s AuthStatus) cell() CellState { + switch s { + case AuthOK: + return CellOK + case AuthNone: + return CellEmpty + default: + return CellNA + } +} + +// RankedAgent is a detected CLI agent with install + auth state for the confirm form. +type RankedAgent struct { + Name string + Auth AuthStatus + Detect bool + Bundled bool // true when this is the downloadable opencode fallback + Score int // lower is better +} + +// RankedCLIAgents lists CLI agents on $PATH: authenticated first, then unknown, then none. +// Bundled OpenCode is appended only when no CLI agent is on $PATH (and a release asset exists). +func RankedCLIAgents() []RankedAgent { + out := []RankedAgent{} + for _, c := range CLIAgents() { + if !c.Detect() { + continue + } + auth := c.AuthState() + score := 2 + switch auth { + case AuthOK: + score = 0 + case AuthUnknown: + score = 1 + } + out = append(out, RankedAgent{ + Name: c.Name(), + Auth: auth, + Detect: true, + Score: score, + }) + } + if len(out) == 0 { + if _, err := OpenCodeAssetName(runtime.GOOS, runtime.GOARCH); err == nil { + out = append(out, RankedAgent{ + Name: "opencode", + Auth: AuthNone, + Detect: false, + Bundled: true, + Score: 3, + }) + } + } + for i := 0; i < len(out); i++ { + for j := i + 1; j < len(out); j++ { + if out[j].Score < out[i].Score { + out[i], out[j] = out[j], out[i] + } + } + } + return out +} + +// pathRanked splits RankedCLIAgents into on-PATH agents and optional bundled OpenCode. +func pathRanked() (agents []RankedAgent, bundled *RankedAgent) { + for _, a := range RankedCLIAgents() { + if a.Bundled { + cp := a + bundled = &cp + continue + } + agents = append(agents, a) + } + return agents, bundled +} + +// RecommendedAgent is the first ranked CLI agent (authenticated on PATH, else the only one). +func RecommendedAgent() string { + ranked := RankedCLIAgents() + if len(ranked) == 0 { + return "auto" + } + return ranked[0].Name +} diff --git a/core/sling/assist/clients_test.go b/core/sling/assist/clients_test.go new file mode 100644 index 000000000..9c10b1cb3 --- /dev/null +++ b/core/sling/assist/clients_test.go @@ -0,0 +1,1052 @@ +// Client adapter tests: JSONC preserve, profile YAML comments, backups. + +package assist + +import ( + "context" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/flarco/g" + "github.com/slingdata-io/sling-cli/core/env" + "github.com/tidwall/gjson" +) + +// withTempHomeDir overrides env.HomeDir for the duration of the test. The +// package-level var is set during init() from $SLING_HOME_DIR — for tests we +// need both updated so envFilePath() picks up the temp dir. +func withTempHomeDir(t *testing.T) string { + t.Helper() + dir := t.TempDir() + // Clear the nested-launch markers. Without this, a test run from inside + // a CLI agent makes NestedLaunch() true and Session prints instead of launching. + for _, k := range []string{"CLAUDECODE", "CURSOR_TRACE_ID", "OPENCODE", "OPENCODE_SESSION"} { + t.Setenv(k, "") + os.Unsetenv(k) + } + prev := env.HomeDir + env.HomeDir = dir + // Override path seam so client adapters (userHome) and Sling dirs (slingHome) + // both resolve under the temp tree. + restore := SetPaths(Paths{SlingHome: dir, UserHome: dir, CWD: dir}) + t.Cleanup(func() { + env.HomeDir = prev + restore() + }) + return dir +} + +// TestJSONPreservesComments verifies that a JSONC document with `//` and `/* */` +// comments survives a round-trip through setJSONPath/deleteJSONPath. This is +// the bug that bit us with VS Code's settings.json, where the previous map +// rewrite silently stripped every comment in the file. +func TestJSONPreservesComments(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "settings.json") + + original := `{ + // top comment about the user's editor + "editor.fontSize": 14, + /* block comment + spanning two lines */ + "editor.fontFamily": "JetBrains Mono", // trailing comment + "files.exclude": { + "**/.git": true // hide git + } +}` + if err := os.WriteFile(path, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + if err := setJSONPath(path, `chat\.instructionsFilesLocations.-1`, "/Users/me/.agents/skills"); err != nil { + t.Fatalf("setJSONPath: %v", err) + } + + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + out := string(got) + + wantSubstrings := []string{ + "// top comment about the user's editor", + "/* block comment", + "spanning two lines */", + "// trailing comment", + "// hide git", + `"editor.fontSize": 14`, + `"editor.fontFamily": "JetBrains Mono"`, + `"chat.instructionsFilesLocations"`, + `"/Users/me/.agents/skills"`, + } + for _, sub := range wantSubstrings { + if !strings.Contains(out, sub) { + t.Errorf("expected output to contain %q\n--- got ---\n%s", sub, out) + } + } +} + +// TestJSONDeletePreservesComments verifies that deleting a key still keeps +// surrounding comments intact. +func TestJSONDeletePreservesComments(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "settings.json") + + original := `{ + // keep this comment + "editor.fontSize": 14, + "mcpServers": { + "sling": { "command": "sling", "args": ["serve", "mcp"] }, + "other": { "command": "other" } + } +}` + if err := os.WriteFile(path, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + if err := deleteJSONPath(path, "mcpServers.sling"); err != nil { + t.Fatalf("deleteJSONPath: %v", err) + } + + got, _ := os.ReadFile(path) + out := string(got) + + if !strings.Contains(out, "// keep this comment") { + t.Errorf("expected comment to be preserved\n--- got ---\n%s", out) + } + if strings.Contains(out, `"sling"`) { + t.Errorf("expected sling entry to be deleted\n--- got ---\n%s", out) + } + if !strings.Contains(out, `"other"`) { + t.Errorf("expected sibling entry preserved\n--- got ---\n%s", out) + } +} + +// TestSaveProfilePreservesYAMLComments verifies that adding the +// env.SLING_ASSIST entry to an existing env.yaml doesn't blow away the user's +// comments and unrelated keys. This is the analogue of +// TestJSONPreservesComments for YAML. +func TestSaveProfilePreservesYAMLComments(t *testing.T) { + homeDir := withTempHomeDir(t) + + envFile := filepath.Join(homeDir, "env.yaml") + original := `# Sling environment file — managed by you. +# These connections are used by replications and pipelines. + +connections: + # Production warehouse + PG_PROD: + type: postgres + host: db.example.com + user: app + # Staging warehouse + PG_STAGE: + type: postgres + host: stage.db.example.com + +# Variables shared across runs +variables: + region: us-west-2 +` + if err := os.WriteFile(envFile, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + prof := DefaultProfile() + prof.Agent = "claude" + if err := SaveProfile(prof); err != nil { + t.Fatalf("SaveProfile: %v", err) + } + + got, _ := os.ReadFile(envFile) + out := string(got) + + wantSubstrings := []string{ + "# Sling environment file — managed by you.", + "# These connections are used by replications and pipelines.", + "# Production warehouse", + "# Staging warehouse", + "PG_PROD:", + "PG_STAGE:", + "region: us-west-2", + "SLING_ASSIST:", + "agent: claude", + } + for _, sub := range wantSubstrings { + if !strings.Contains(out, sub) { + t.Errorf("expected output to contain %q\n--- got ---\n%s", sub, out) + } + } + // Legacy `variables:` migrates to `env:` on save — the block contents + // (region: us-west-2) survive, but the heading comment attached to the + // renamed key is dropped along with the old key. + if strings.Contains(out, "variables:") { + t.Errorf("expected legacy variables: block to be renamed to env:\n--- got ---\n%s", out) + } + + // Idempotency: a second save should leave comments intact and not duplicate + // the SLING_ASSIST entry. + prof.HintInErrors = false + if err := SaveProfile(prof); err != nil { + t.Fatalf("second SaveProfile: %v", err) + } + got, _ = os.ReadFile(envFile) + out = string(got) + + if strings.Count(out, "SLING_ASSIST:") != 1 { + t.Errorf("expected exactly one SLING_ASSIST entry, got\n%s", out) + } + if !strings.Contains(out, "# Production warehouse") { + t.Errorf("comments lost on second save\n--- got ---\n%s", out) + } + if !strings.Contains(out, "hint_in_errors: false") { + t.Errorf("SLING_ASSIST entry did not update on second save\n--- got ---\n%s", out) + } +} + +// TestJSONLeadingCommentBanner reproduces the real-world bug where VS Code's +// settings.json has a `// Place your settings...` banner *before* the opening +// `{`. sjson can't parse that prefix and rebuilds the document as a single +// compact line, blowing away the user's config. Our fix strips the leading +// non-JSON content before handing the buffer to sjson, so the body survives. +func TestJSONLeadingCommentBanner(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "settings.json") + + // Mimics the real shape of VS Code's user settings file — leading banner, + // many keys, multi-line. + original := `// Place your settings in this file to overwrite the default settings +{ + "editor.tabSize": 2, + "editor.detectIndentation": false, + "editor.guides.indentation": false, + "editor.formatOnSave": true, + "workbench.colorTheme": "Default Dark Modern", + "files.autoSave": "afterDelay", + "git.autofetch": true, + "terminal.integrated.fontSize": 13 +} +` + if err := os.WriteFile(path, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + if err := setJSONPath(path, `chat\.instructionsFilesLocations.-1`, "/Users/me/.agents/skills"); err != nil { + t.Fatalf("setJSONPath: %v", err) + } + + got, _ := os.ReadFile(path) + out := string(got) + + // Must still contain every original key. + wantKeys := []string{ + `"editor.tabSize"`, + `"editor.detectIndentation"`, + `"editor.guides.indentation"`, + `"editor.formatOnSave"`, + `"workbench.colorTheme"`, + `"files.autoSave"`, + `"git.autofetch"`, + `"terminal.integrated.fontSize"`, + `"chat.instructionsFilesLocations"`, + } + for _, k := range wantKeys { + if !strings.Contains(out, k) { + t.Errorf("expected output to contain %q\n--- got ---\n%s", k, out) + } + } + + // And the file should not have collapsed to one line. + if strings.Count(out, "\n") < 5 { + t.Errorf("file collapsed to a single line, only %d newlines\n--- got ---\n%s", + strings.Count(out, "\n"), out) + } + + // Leading banner must survive — it carries useful context the user wrote + // (or VS Code wrote on their behalf). + if !strings.HasPrefix(out, "// Place your settings in this file") { + t.Errorf("leading banner was dropped\n--- got ---\n%s", out) + } +} + +// TestJSONBackupCreatedBeforeEdit verifies setJSONPath writes .backup +// before mutating the original. The user can always recover from .backup if +// something goes wrong on the next install. +func TestJSONBackupCreatedBeforeEdit(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "settings.json") + + original := `{ + "editor.fontSize": 14, + "workbench.colorTheme": "Default Dark Modern" +}` + if err := os.WriteFile(path, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + if err := setJSONPath(path, `chat\.instructionsFilesLocations.-1`, "/x"); err != nil { + t.Fatalf("setJSONPath: %v", err) + } + + bk, err := os.ReadFile(path + ".backup") + if err != nil { + t.Fatalf("expected backup file at %s: %v", path+".backup", err) + } + if string(bk) != original { + t.Errorf("backup didn't match original\n--- backup ---\n%s\n--- want ---\n%s", string(bk), original) + } +} + +// TestJSONNoBackupWhenSourceMissing ensures we don't create an empty +// .backup when the file we're about to edit doesn't exist yet. +func TestJSONNoBackupWhenSourceMissing(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "fresh.json") + + if err := setJSONPath(path, "mcpServers.sling", map[string]any{"command": "sling"}); err != nil { + t.Fatalf("setJSONPath: %v", err) + } + if _, err := os.Stat(path + ".backup"); !os.IsNotExist(err) { + t.Errorf("expected no backup for fresh-file write, got err=%v", err) + } +} + +// TestJSONDestructiveEditRefused simulates an edit that drops top-level keys +// (e.g. a mangled sjson rewrite) and verifies the helper refuses to commit it. +// We trigger this by directly invoking the validator with mocked before/after +// snapshots. +func TestJSONDestructiveEditRefused(t *testing.T) { + before := []byte(`{ + "a": 1, + "b": 2, + "c": 3, + "d": 4 +}`) + // Mimic an sjson-trip-on-banner result: the whole body collapsed into a + // single key, single line. + after := []byte(`{"only":"survivor"}`) + + if err := validateEditNotDestructive(before, after, 0); err == nil { + t.Errorf("expected validateEditNotDestructive to reject a destructive edit") + } + + // And a single-key delete on the same document should be allowed when + // allowKeyDelta=1. + deleted := []byte(`{ + "a": 1, + "b": 2, + "c": 3 +}`) + if err := validateEditNotDestructive(before, deleted, 1); err != nil { + t.Errorf("expected single-key delete to be allowed: %v", err) + } +} + +// TestSaveProfileFreshFile covers the empty-file path: when env.yaml doesn't +// exist yet, SaveProfile should create it with just env.SLING_ASSIST. +func TestSaveProfileFreshFile(t *testing.T) { + homeDir := withTempHomeDir(t) + + prof := DefaultProfile() + prof.Agent = "codex" + if err := SaveProfile(prof); err != nil { + t.Fatalf("SaveProfile: %v", err) + } + + got, _ := os.ReadFile(filepath.Join(homeDir, "env.yaml")) + out := string(got) + + if !strings.Contains(out, "SLING_ASSIST:") { + t.Errorf("missing SLING_ASSIST entry in fresh file\n--- got ---\n%s", out) + } + if !strings.Contains(out, "agent: codex") { + t.Errorf("missing agent: codex\n--- got ---\n%s", out) + } +} + +// ---- opencode / pi / grok adapters ---- + +// TestNewClientsRegistered verifies the three additions are in the canonical +// list, are launchable CLI agents, and resolve by name. +func TestNewClientsRegistered(t *testing.T) { + for _, name := range []string{"opencode", "pi", "grok"} { + c := LookupClient(name) + if c == nil { + t.Fatalf("LookupClient(%q) returned nil", name) + } + if c.Kind() != KindCLIAgent { + t.Errorf("%s: expected KindCLIAgent, got %v", name, c.Kind()) + } + found := false + for _, a := range CLIAgents() { + if a.Name() == name { + found = true + } + } + if !found { + t.Errorf("%s missing from CLIAgents()", name) + } + } +} + +// TestOpencodeMCPRoundTrip covers opencode's distinctive MCP shape: a `mcp` +// (not `mcpServers`) block whose entries carry `type: local` and a single +// argv array rather than command/args. +func TestOpencodeMCPRoundTrip(t *testing.T) { + home := withTempHomeDir(t) + ctx := context.Background() + c := &opencodeClient{} + + path := filepath.Join(home, ".config", "opencode", "opencode.json") + if got := c.configPath(ScopeUser); got != path { + t.Fatalf("configPath = %s, want %s", got, path) + } + + if res := c.CheckMCP(ctx, ScopeUser); res.State != CellFail { + t.Errorf("expected fail before write, got %v", res.State) + } + if err := c.WriteMCP(ctx, ScopeUser); err != nil { + t.Fatalf("WriteMCP: %v", err) + } + + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + out := string(data) + if got := gjson.GetBytes(data, "mcp.sling.type").String(); got != "local" { + t.Errorf("type = %q, want \"local\"\n--- got ---\n%s", got, out) + } + // command must be a flat argv array, not the command/args pair. + if got := gjson.GetBytes(data, "mcp.sling.command").String(); got != `["sling","serve","mcp"]` { + t.Errorf("command = %q, want [\"sling\",\"serve\",\"mcp\"]\n--- got ---\n%s", got, out) + } + if !gjson.GetBytes(data, "mcp.sling.enabled").Bool() { + t.Errorf("expected enabled: true\n--- got ---\n%s", out) + } + if gjson.GetBytes(data, "mcp.sling.args").Exists() { + t.Errorf("opencode entry should not use args key\n--- got ---\n%s", out) + } + if got := gjson.GetBytes(data, "mcp.agent-browser.type").String(); got != "local" { + t.Errorf("agent-browser type = %q, want \"local\"\n--- got ---\n%s", got, out) + } + if !strings.Contains(gjson.GetBytes(data, "mcp.agent-browser.command").Raw, `"mcp"`) { + t.Errorf("agent-browser command missing mcp\n--- got ---\n%s", out) + } + + if res := c.CheckMCP(ctx, ScopeUser); res.State != CellOK { + t.Errorf("expected ok after write, got %v (%s)", res.State, res.Note) + } + if err := c.RemoveMCP(ctx, ScopeUser); err != nil { + t.Fatalf("RemoveMCP: %v", err) + } + if res := c.CheckMCP(ctx, ScopeUser); res.State != CellFail { + t.Errorf("expected fail after remove, got %v", res.State) + } +} + +// TestOpencodeMCPPreservesSiblings ensures we only touch mcp.sling and leave +// the user's other opencode settings (and their comments) alone. +func TestOpencodeMCPPreservesSiblings(t *testing.T) { + home := withTempHomeDir(t) + ctx := context.Background() + path := filepath.Join(home, ".config", "opencode", "opencode.json") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + original := `{ + // my opencode setup + "$schema": "https://opencode.ai/config.json", + "theme": "tokyonight", + "mcp": { + "other": { "type": "local", "command": ["other"] } + } +}` + if err := os.WriteFile(path, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + c := &opencodeClient{} + if err := c.WriteMCP(ctx, ScopeUser); err != nil { + t.Fatalf("WriteMCP: %v", err) + } + out, _ := os.ReadFile(path) + for _, sub := range []string{"// my opencode setup", `"tokyonight"`, `"other"`, `"sling"`, `"agent-browser"`, `"$schema"`} { + if !strings.Contains(string(out), sub) { + t.Errorf("expected %q preserved\n--- got ---\n%s", sub, string(out)) + } + } + + if err := c.RemoveMCP(ctx, ScopeUser); err != nil { + t.Fatalf("RemoveMCP: %v", err) + } + out, _ = os.ReadFile(path) + if strings.Contains(string(out), `"sling"`) { + t.Errorf("sling entry not removed\n--- got ---\n%s", string(out)) + } + if strings.Contains(string(out), `"agent-browser"`) { + t.Errorf("agent-browser entry not removed\n--- got ---\n%s", string(out)) + } + if !strings.Contains(string(out), `"other"`) { + t.Errorf("sibling MCP entry lost\n--- got ---\n%s", string(out)) + } +} + +// TestOpencodeConfigDirXDG verifies XDG_CONFIG_HOME wins over ~/.config. +func TestOpencodeConfigDirXDG(t *testing.T) { + home := withTempHomeDir(t) + if got, want := opencodeConfigDir(), filepath.Join(home, ".config", "opencode"); got != want { + t.Errorf("default configDir = %s, want %s", got, want) + } + t.Setenv("XDG_CONFIG_HOME", filepath.Join(home, "xdg")) + if got, want := opencodeConfigDir(), filepath.Join(home, "xdg", "opencode"); got != want { + t.Errorf("XDG configDir = %s, want %s", got, want) + } +} + +// TestPiMCPRoundTrip covers pi's mcp.json (separate from settings.json) with +// the standard mcpServers command/args shape. +func TestPiMCPRoundTrip(t *testing.T) { + home := withTempHomeDir(t) + ctx := context.Background() + c := &piClient{} + + path := filepath.Join(home, ".pi", "agent", "mcp.json") + if got := c.mcpPath(ScopeUser); got != path { + t.Fatalf("mcpPath = %s, want %s", got, path) + } + + if err := c.WriteMCP(ctx, ScopeUser); err != nil { + t.Fatalf("WriteMCP: %v", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + // pi uses the standard command/args split (not opencode's argv array). + if got := gjson.GetBytes(data, "mcpServers.sling.command").String(); got != "sling" { + t.Errorf("command = %q, want \"sling\"\n--- got ---\n%s", got, string(data)) + } + if got := gjson.GetBytes(data, "mcpServers.sling.args").String(); got != `["serve","mcp"]` { + t.Errorf("args = %q, want [\"serve\",\"mcp\"]\n--- got ---\n%s", got, string(data)) + } + if got := gjson.GetBytes(data, "mcpServers.agent-browser.args").String(); got != `["mcp","--tools","core"]` { + t.Errorf("agent-browser args = %q\n--- got ---\n%s", got, string(data)) + } + if res := c.CheckMCP(ctx, ScopeUser); res.State != CellOK { + t.Errorf("expected ok after write, got %v (%s)", res.State, res.Note) + } + + if err := c.RemoveMCP(ctx, ScopeUser); err != nil { + t.Fatalf("RemoveMCP: %v", err) + } + if res := c.CheckMCP(ctx, ScopeUser); res.State != CellFail { + t.Errorf("expected fail after remove, got %v", res.State) + } +} + +// TestPiAgentDirEnvOverride verifies PI_CODING_AGENT_DIR takes precedence over +// the ~/.pi/agent default. +func TestPiAgentDirEnvOverride(t *testing.T) { + home := withTempHomeDir(t) + if got, want := piAgentDir(), filepath.Join(home, ".pi", "agent"); got != want { + t.Errorf("default agentDir = %s, want %s", got, want) + } + custom := filepath.Join(home, "custom-pi") + t.Setenv("PI_CODING_AGENT_DIR", custom) + if got := piAgentDir(); got != custom { + t.Errorf("env agentDir = %s, want %s", got, custom) + } + if got, want := (&piClient{}).mcpPath(ScopeUser), filepath.Join(custom, "mcp.json"); got != want { + t.Errorf("mcpPath = %s, want %s", got, want) + } +} + +// TestGrokMCPRoundTrip covers grok's TOML config: [mcp_servers.sling] written +// into ~/.grok/config.toml, same table shape as codex. +func TestGrokMCPRoundTrip(t *testing.T) { + home := withTempHomeDir(t) + ctx := context.Background() + c := &grokClient{} + + path := filepath.Join(home, ".grok", "config.toml") + if got := c.configPath(ScopeUser); got != path { + t.Fatalf("configPath = %s, want %s", got, path) + } + + if res := c.CheckMCP(ctx, ScopeUser); res.State != CellFail { + t.Errorf("expected fail before write, got %v", res.State) + } + if err := c.WriteMCP(ctx, ScopeUser); err != nil { + t.Fatalf("WriteMCP: %v", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + for _, sub := range []string{"[mcp_servers.sling]", `command = "sling"`, `args = ["serve", "mcp"]`, "[mcp_servers.agent-browser]"} { + if !strings.Contains(string(data), sub) { + t.Errorf("expected %q in config.toml\n--- got ---\n%s", sub, string(data)) + } + } + if res := c.CheckMCP(ctx, ScopeUser); res.State != CellOK { + t.Errorf("expected ok after write, got %v (%s)", res.State, res.Note) + } + + // Idempotent: a second write must not duplicate the section. + if err := c.WriteMCP(ctx, ScopeUser); err != nil { + t.Fatalf("second WriteMCP: %v", err) + } + data, _ = os.ReadFile(path) + if n := strings.Count(string(data), "[mcp_servers.sling]"); n != 1 { + t.Errorf("expected 1 sling section, got %d\n--- got ---\n%s", n, string(data)) + } + if n := strings.Count(string(data), "[mcp_servers.agent-browser]"); n != 1 { + t.Errorf("expected 1 agent-browser section, got %d\n--- got ---\n%s", n, string(data)) + } + + if err := c.RemoveMCP(ctx, ScopeUser); err != nil { + t.Fatalf("RemoveMCP: %v", err) + } + if res := c.CheckMCP(ctx, ScopeUser); res.State != CellFail { + t.Errorf("expected fail after remove, got %v", res.State) + } +} + +// TestGrokMCPPreservesOtherSections verifies we only own the sling table and +// leave the user's other grok config intact. +func TestGrokMCPPreservesOtherSections(t *testing.T) { + home := withTempHomeDir(t) + ctx := context.Background() + path := filepath.Join(home, ".grok", "config.toml") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + original := `model = "grok-4" + +[mcp_servers.filesystem] +command = "npx" +args = ["-y", "@modelcontextprotocol/server-filesystem"] + +[skills] +paths = ["~/.agents/skills"] +` + if err := os.WriteFile(path, []byte(original), 0o644); err != nil { + t.Fatal(err) + } + + c := &grokClient{} + if err := c.WriteMCP(ctx, ScopeUser); err != nil { + t.Fatalf("WriteMCP: %v", err) + } + out, _ := os.ReadFile(path) + for _, sub := range []string{`model = "grok-4"`, "[mcp_servers.filesystem]", "[skills]", "[mcp_servers.sling]", "[mcp_servers.agent-browser]"} { + if !strings.Contains(string(out), sub) { + t.Errorf("expected %q preserved\n--- got ---\n%s", sub, string(out)) + } + } + + if err := c.RemoveMCP(ctx, ScopeUser); err != nil { + t.Fatalf("RemoveMCP: %v", err) + } + out, _ = os.ReadFile(path) + if strings.Contains(string(out), "[mcp_servers.sling]") { + t.Errorf("sling section not removed\n--- got ---\n%s", string(out)) + } + if strings.Contains(string(out), "[mcp_servers.agent-browser]") { + t.Errorf("agent-browser section not removed\n--- got ---\n%s", string(out)) + } + if !strings.Contains(string(out), "[mcp_servers.filesystem]") { + t.Errorf("sibling MCP section lost\n--- got ---\n%s", string(out)) + } + if !strings.Contains(string(out), "[skills]") { + t.Errorf("skills section lost\n--- got ---\n%s", string(out)) + } +} + +// TestNewClientsSkillsAreCanonical verifies opencode/pi/grok are all no-op on +// WriteSkills (they read ~/.agents/skills/ natively) and that CheckSkills +// tracks the canonical bundle rather than a per-client redirect. +func TestNewClientsSkillsAreCanonical(t *testing.T) { + withTempHomeDir(t) + ctx := context.Background() + skills := []string{"sling", "sling-replications"} + + clients := []Client{&opencodeClient{}, &piClient{}, &grokClient{}} + for _, c := range clients { + if err := c.WriteSkills(ctx, skills, ScopeUser); err != nil { + t.Fatalf("%s WriteSkills: %v", c.Name(), err) + } + // Bundle not written yet → every skill reports missing. + for _, res := range c.CheckSkills(ctx, skills, ScopeUser) { + if res.State != CellFail { + t.Errorf("%s: expected fail before bundle write, got %v", c.Name(), res.State) + } + } + } + + if err := writeCanonicalBundle(skills); err != nil { + t.Fatalf("writeCanonicalBundle: %v", err) + } + for _, c := range clients { + results := c.CheckSkills(ctx, skills, ScopeUser) + if len(results) != len(skills) { + t.Errorf("%s: got %d results, want %d", c.Name(), len(results), len(skills)) + } + for _, res := range results { + if res.State != CellOK { + t.Errorf("%s: expected ok after bundle write, got %v (%s)", c.Name(), res.State, res.Note) + } + } + // RemoveSkills must not touch the canonical bundle — it's shared. + if err := c.RemoveSkills(ctx, skills, ScopeUser); err != nil { + t.Fatalf("%s RemoveSkills: %v", c.Name(), err) + } + if !g.PathExists(canonicalSkillPath("sling")) { + t.Errorf("%s RemoveSkills deleted the shared canonical bundle", c.Name()) + } + } +} + +// TestNewClientsProjectScope verifies project-scope paths resolve under +// projectRoot() (absolute), not the bare relative "./..." form that scattered +// files when CWD was a subdirectory. +func TestNewClientsProjectScope(t *testing.T) { + root := projectRoot() + cases := []struct { + name string + got string + want string + }{ + {"opencode", (&opencodeClient{}).configPath(ScopeProject), filepath.Join(root, "opencode.json")}, + {"pi", (&piClient{}).mcpPath(ScopeProject), filepath.Join(root, ".pi", "mcp.json")}, + {"grok", (&grokClient{}).configPath(ScopeProject), filepath.Join(root, ".grok", "config.toml")}, + {"claude", (&claudeClient{}).mcpPath(ScopeProject), filepath.Join(root, ".mcp.json")}, + {"vscode", (&vscodeClient{}).vscodeMCPPath(ScopeProject), filepath.Join(root, ".vscode", "mcp.json")}, + } + for _, tc := range cases { + if tc.got != tc.want { + t.Errorf("%s project path = %s, want %s", tc.name, tc.got, tc.want) + } + if !filepath.IsAbs(tc.got) { + t.Errorf("%s project path should be absolute, got %s", tc.name, tc.got) + } + } +} + +// TestNewClientsDetect is PATH-only: a config dir without a binary is not enough. +func TestNewClientsDetect(t *testing.T) { + home := withTempHomeDir(t) + bin := filepath.Join(home, "empty-bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin) + + cases := []Client{&opencodeClient{}, &piClient{}, &grokClient{}, &geminiClient{}} + for _, c := range cases { + if c.Detect() { + t.Errorf("%s: detected with empty PATH", c.Name()) + } + } + for _, name := range []string{"opencode", "pi", "grok", "gemini"} { + stub := filepath.Join(bin, name) + if err := os.WriteFile(stub, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + } + for _, c := range cases { + if !c.Detect() { + t.Errorf("%s: not detected after stub on PATH", c.Name()) + } + } +} + +func clearAuthEnv(t *testing.T) { + t.Helper() + for _, k := range []string{ + "ANTHROPIC_API_KEY", "CLAUDE_CODE_OAUTH_TOKEN", + "OPENAI_API_KEY", "GEMINI_API_KEY", "GOOGLE_API_KEY", + "GOOGLE_APPLICATION_CREDENTIALS", "ANTHROPIC_AUTH_TOKEN", + "CURSOR_API_KEY", "AIDER_API_KEY", "OPENCODE_API_KEY", + "PI_API_KEY", "OPENROUTER_API_KEY", "XAI_API_KEY", "GROK_API_KEY", + "GROK_CODE_XAI_API_KEY", "CLAUDE_CONFIG_DIR", + "CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_USE_VERTEX", "CLAUDE_CODE_USE_FOUNDRY", + "XDG_DATA_HOME", "PI_CODING_AGENT_DIR", + } { + t.Setenv(k, "") + } +} + +func TestAuthStateFakeConfigTrees(t *testing.T) { + clearAuthEnv(t) + dir := withTempHomeDir(t) + + wantClaudeEmpty := AuthNone + if runtime.GOOS == "darwin" { + wantClaudeEmpty = AuthUnknown // Keychain may hold /login tokens + } + if st := (&claudeClient{}).AuthState(); st != wantClaudeEmpty { + t.Fatalf("claude empty = %s, want %s", st, wantClaudeEmpty) + } + if st := (&grokClient{}).AuthState(); st != AuthNone { + t.Fatalf("grok empty = %s, want none", st) + } + + cred := filepath.Join(dir, ".claude", ".credentials.json") + if err := os.MkdirAll(filepath.Dir(cred), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(cred, []byte(`{"oauth":true}`), 0o600); err != nil { + t.Fatal(err) + } + if st := (&claudeClient{}).AuthState(); st != AuthOK { + t.Fatalf("claude credentials file = %s, want ok", st) + } + + oauth := filepath.Join(dir, ".claude.json") + _ = os.Remove(cred) + if err := os.WriteFile(oauth, []byte(`{"oauthAccount":{"uuid":"x"}}`), 0o600); err != nil { + t.Fatal(err) + } + if st := (&claudeClient{}).AuthState(); st != AuthOK { + t.Fatalf("claude oauth json = %s, want ok", st) + } + + grokAuth := filepath.Join(dir, ".grok", "auth.json") + if err := os.MkdirAll(filepath.Dir(grokAuth), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(grokAuth, []byte(`{"token":"x"}`), 0o600); err != nil { + t.Fatal(err) + } + if st := (&grokClient{}).AuthState(); st != AuthOK { + t.Fatalf("grok auth.json = %s, want ok", st) + } + + codexAuth := filepath.Join(dir, ".codex", "auth.json") + if err := os.MkdirAll(filepath.Dir(codexAuth), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(codexAuth, []byte(`{}`), 0o600); err != nil { + t.Fatal(err) + } + if st := (&codexClient{}).AuthState(); st != AuthOK { + t.Fatalf("codex auth.json = %s, want ok", st) + } + + if st := (&vscodeClient{}).AuthState(); st != AuthUnknown { + t.Fatalf("vscode = %s, want unknown", st) + } +} + +func TestAuthStateEnvKeys(t *testing.T) { + withTempHomeDir(t) + t.Setenv("ANTHROPIC_API_KEY", "sk-test") + if st := (&claudeClient{}).AuthState(); st != AuthOK { + t.Fatalf("claude env = %s, want ok", st) + } +} + +func TestRankedCLIAgentsAuthFirst(t *testing.T) { + clearAuthEnv(t) + dir := withTempHomeDir(t) + bin := filepath.Join(dir, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + for _, name := range []string{"claude", "codex"} { + if err := os.WriteFile(filepath.Join(bin, name), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + } + t.Setenv("PATH", bin) + t.Setenv("XDG_CONFIG_HOME", "") + if err := os.MkdirAll(filepath.Join(dir, ".claude"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, ".claude", ".credentials.json"), []byte(`{"ok":1}`), 0o600); err != nil { + t.Fatal(err) + } + ranked := RankedCLIAgents() + if len(ranked) < 2 { + t.Fatalf("expected claude+codex, got %+v", ranked) + } + if ranked[0].Name != "claude" || ranked[0].Auth != AuthOK { + t.Fatalf("want claude/ok first, got %+v", ranked[0]) + } + if ranked[1].Name != "codex" { + t.Fatalf("want codex second, got %+v", ranked[1]) + } + for _, a := range ranked { + if a.Bundled { + t.Fatalf("PATH agents present: bundled opencode should not appear: %+v", a) + } + } +} + +func TestRankedCLIAgentsBundledOnlyWhenNoneOnPath(t *testing.T) { + clearAuthEnv(t) + dir := withTempHomeDir(t) + t.Setenv("PATH", filepath.Join(dir, "empty-bin")) + t.Setenv("XDG_CONFIG_HOME", "") + if err := os.MkdirAll(filepath.Join(dir, ".gemini"), 0o755); err != nil { + t.Fatal(err) + } + ranked := RankedCLIAgents() + if len(ranked) == 0 { + t.Fatal("expected bundled opencode when nothing is on PATH") + } + if ranked[0].Name != "opencode" || !ranked[0].Bundled { + t.Fatalf("want bundled opencode, got %+v", ranked[0]) + } + for _, a := range ranked { + if a.Name == "gemini" { + t.Fatalf("gemini config dir without binary must not rank: %+v", a) + } + } +} + +func TestDoctorMatrixOmitsOffPathAgents(t *testing.T) { + clearAuthEnv(t) + dir := withTempHomeDir(t) + bin := filepath.Join(dir, "bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(bin, "claude"), []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin) + if err := os.MkdirAll(filepath.Join(dir, ".gemini"), 0o755); err != nil { + t.Fatal(err) + } + if err := SaveProfile(DefaultProfile()); err != nil { + t.Fatal(err) + } + r, err := Doctor(context.Background()) + if err != nil { + t.Fatal(err) + } + if r.Matrix == nil { + t.Fatal("expected matrix") + } + for _, c := range r.Matrix.Clients { + if c == "gemini" { + t.Fatal("gemini not on PATH must not appear in the matrix") + } + } + var hasAuth bool + for _, row := range r.Matrix.Rows { + if row.Label == "agent on PATH" { + t.Fatal("PATH row should be gone") + } + if row.Label == "auth" { + hasAuth = true + } + } + if !hasAuth { + t.Fatal("expected auth row") + } + out := r.Render() + if strings.Contains(out, "binary not on $PATH") { + t.Fatalf("PATH-missing note should not render:\n%s", out) + } + if strings.Contains(out, "GEMINI") { + t.Fatalf("gemini column should not render:\n%s", out) + } +} + +func TestNestedLaunchEnvMarkers(t *testing.T) { + t.Setenv("CLAUDECODE", "") + t.Setenv("CURSOR_TRACE_ID", "") + t.Setenv("OPENCODE", "") + t.Setenv("OPENCODE_SESSION", "") + if NestedLaunch() { + t.Fatal("empty env should not nest") + } + t.Setenv("CLAUDECODE", "1") + if !NestedLaunch() { + t.Fatal("CLAUDECODE should nest") + } + t.Setenv("CLAUDECODE", "") + t.Setenv("CURSOR_TRACE_ID", "abc") + if !NestedLaunch() { + t.Fatal("CURSOR_TRACE_ID should nest") + } + t.Setenv("CURSOR_TRACE_ID", "") + t.Setenv("OPENCODE_SESSION", "s1") + if !NestedLaunch() { + t.Fatal("OPENCODE_SESSION should nest") + } +} + +func TestCursorAuthEnvKeyWins(t *testing.T) { + clearAuthEnv(t) + resetCursorStatusCache() + t.Cleanup(resetCursorStatusCache) + t.Setenv("CURSOR_API_KEY", "key-123") + if st := (&cursorClient{}).AuthState(); st != AuthOK { + t.Fatalf("CURSOR_API_KEY = %s, want ok", st) + } +} + +// Cursor keeps browser-login tokens in the OS keychain, so AuthState shells +// out to `cursor-agent status --format json`. When that CLI is absent the +// probe must not claim the user is signed out. +func TestCursorAuthWithoutCLI(t *testing.T) { + clearAuthEnv(t) + resetCursorStatusCache() + t.Cleanup(resetCursorStatusCache) + t.Setenv("PATH", t.TempDir()) + st := (&cursorClient{}).AuthState() + if _, ok := cursorStatusAuth(); ok { + t.Fatal("no cursor-agent on PATH, yet the probe answered") + } + want := AuthNone + if runtime.GOOS == "darwin" || runtime.GOOS == "windows" { + want = AuthUnknown + } + if st != want { + t.Fatalf("AuthState = %s, want %s", st, want) + } +} + +func TestCursorStatusAuthParsesCLI(t *testing.T) { + clearAuthEnv(t) + resetCursorStatusCache() + t.Cleanup(resetCursorStatusCache) + dir := t.TempDir() + stub := filepath.Join(dir, "cursor-agent") + // echo is a shell builtin: the stub's PATH holds only this dir. + script := "#!/bin/sh\necho '{\"status\":\"authenticated\",\"isAuthenticated\":true}'\n" + if err := os.WriteFile(stub, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir) + + authed, ok := cursorStatusAuth() + if !ok || !authed { + t.Fatalf("authed=%v answered=%v", authed, ok) + } + if st := (&cursorClient{}).AuthState(); st != AuthOK { + t.Fatalf("AuthState = %s, want ok", st) + } + + // Logged out: the CLI still exits 0, so the field decides. + script = "#!/bin/sh\necho '{\"status\":\"unauthenticated\",\"isAuthenticated\":false}'\n" + if err := os.WriteFile(stub, []byte(script), 0o755); err != nil { + t.Fatal(err) + } + resetCursorStatusCache() + authed, ok = cursorStatusAuth() + if !ok || authed { + t.Fatalf("logged out: authed=%v answered=%v", authed, ok) + } + if st := (&cursorClient{}).AuthState(); st != AuthNone { + t.Fatalf("AuthState = %s, want none", st) + } +} diff --git a/core/sling/assist/doctor.go b/core/sling/assist/doctor.go new file mode 100644 index 000000000..b2c388e6c --- /dev/null +++ b/core/sling/assist/doctor.go @@ -0,0 +1,514 @@ +package assist + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/flarco/g" + "github.com/jedib0t/go-pretty/table" + "github.com/jedib0t/go-pretty/text" + "github.com/slingdata-io/sling-cli/core" + "github.com/slingdata-io/sling-cli/core/env" +) + +// CellState is one cell in the doctor matrix. +type CellState int + +const ( + CellOK CellState = iota // ✓ + CellFail // ✗ + CellNA // ⊘ unsupported + CellEmpty // — not applicable +) + +// CheckResult is a typed status from Client.CheckSkills / CheckMCP. +type CheckResult struct { + State CellState `json:"state"` + Skill string `json:"skill,omitempty"` // set for per-skill checks + Note string `json:"note,omitempty"` // short detail, no glyph +} + +// Render returns a CLI display line (e.g. "✓ claude: sling"). +func (r CheckResult) Render(clientName string) string { + label := clientName + if r.Skill != "" { + if r.Note != "" { + return fmt.Sprintf("%s %s: %s — %s", r.State.Glyph(), label, r.Skill, r.Note) + } + return fmt.Sprintf("%s %s: %s", r.State.Glyph(), label, r.Skill) + } + if r.Note != "" { + return fmt.Sprintf("%s %s: %s", r.State.Glyph(), label, r.Note) + } + return fmt.Sprintf("%s %s", r.State.Glyph(), label) +} + +// Glyph returns the terminal marker for a cell state. +func (s CellState) Glyph() string { + switch s { + case CellOK: + return "✓" + case CellFail: + return "✗" + case CellNA: + return "⊘" + case CellEmpty: + return "—" + default: + return "?" + } +} + +// String is the JSON/API token for a cell state. +func (s CellState) String() string { + switch s { + case CellOK: + return "ok" + case CellFail: + return "fail" + case CellNA: + return "na" + case CellEmpty: + return "empty" + default: + return "unknown" + } +} + +// MarshalJSON encodes CellState as a stable string. +func (s CellState) MarshalJSON() ([]byte, error) { + return json.Marshal(s.String()) +} + +func checkOK(note string) CheckResult { + return CheckResult{State: CellOK, Note: note} +} + +func checkFail(note string) CheckResult { + return CheckResult{State: CellFail, Note: note} +} + +func checkNA(note string) CheckResult { + return CheckResult{State: CellNA, Note: note} +} + +func checkEmpty(note string) CheckResult { + return CheckResult{State: CellEmpty, Note: note} +} + +func checkSkill(state CellState, skill, note string) CheckResult { + return CheckResult{State: state, Skill: skill, Note: note} +} + +// MatrixRow is one row in the agent × capability matrix. +type MatrixRow struct { + Label string `json:"label"` + Cells map[string]CellState `json:"cells"` // client name → state + Notes map[string]string `json:"notes,omitempty"` +} + +// DoctorMatrix is the cross-check table for detected clients. +type DoctorMatrix struct { + Clients []string `json:"clients"` + Rows []MatrixRow `json:"rows"` +} + +// DoctorFinding is one structured global check (profile, skill, env, version). +type DoctorFinding struct { + ID string `json:"id"` + OK bool `json:"ok"` + Summary string `json:"summary"` + Detail string `json:"detail,omitempty"` + Hint string `json:"hint,omitempty"` +} + +// DoctorReport is the `sling assist setup --doctor` result. +type DoctorReport struct { + OK bool `json:"ok"` + SlingVersion string `json:"sling_version"` + Findings []DoctorFinding `json:"findings"` + Matrix *DoctorMatrix `json:"matrix,omitempty"` + Lines []string `json:"-"` // CLI glyph prose +} + +// AddFinding records a structured finding and a CLI display line. +func (r *DoctorReport) AddFinding(f DoctorFinding) { + r.Findings = append(r.Findings, f) + glyph := "✓" + if !f.OK { + glyph = "✗" + r.OK = false + } + line := fmt.Sprintf("%s %s", glyph, f.Summary) + if f.Hint != "" { + line += " → " + f.Hint + } + r.Lines = append(r.Lines, line) +} + +// Add maps glyph prose into a finding. +func (r *DoctorReport) Add(pass bool, line string) { + sum := strings.TrimSpace(line) + for _, pfx := range []string{"✓ ", "✗ ", "⊘ ", "— "} { + sum = strings.TrimPrefix(sum, pfx) + } + hint := "" + if i := strings.Index(sum, " → "); i >= 0 { + hint = strings.TrimSpace(sum[i+4:]) + sum = strings.TrimSpace(sum[:i]) + } + id := "misc" + if parts := strings.SplitN(sum, ":", 2); len(parts) > 0 { + id = strings.TrimSpace(strings.ReplaceAll(parts[0], " ", "_")) + } + r.AddFinding(DoctorFinding{ID: id, OK: pass, Summary: sum, Hint: hint}) +} + +// ToJSON returns a pretty-printed doctor.json payload. +func (r *DoctorReport) ToJSON() ([]byte, error) { + if r == nil { + return []byte("{}"), nil + } + return json.MarshalIndent(r, "", " ") +} + +// DoctorOptions configures Doctor. Zero value uses ScopeUser. +type DoctorOptions struct { + Scope Scope // must match install scope under test +} + +// Doctor probes the install end-to-end. opts[0].Scope defaults to ScopeUser. +func Doctor(ctx context.Context, opts ...DoctorOptions) (*DoctorReport, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + scope := ScopeUser + if len(opts) > 0 { + scope = opts[0].Scope + } + + r := &DoctorReport{OK: true, SlingVersion: core.Version} + + prof, exists, err := LoadProfile() + switch { + case err != nil: + r.AddFinding(DoctorFinding{ + ID: "profile", OK: false, + Summary: fmt.Sprintf("profile: %v", err), + Hint: "run: sling assist setup", + }) + case !exists: + r.AddFinding(DoctorFinding{ + ID: "profile", OK: false, + Summary: "profile: env.SLING_ASSIST missing in env.yaml", + Hint: "run: sling assist setup", + }) + default: + r.AddFinding(DoctorFinding{ + ID: "profile", OK: true, + Summary: fmt.Sprintf("profile: agent=%s, hint_in_errors=%v", prof.Agent, prof.HintInErrors), + }) + } + + if err := ctx.Err(); err != nil { + return r, err + } + + skillNames := listSkillNames() + for _, name := range skillNames { + if err := ctx.Err(); err != nil { + return r, err + } + canonical := canonicalSkillPath(name) + id := "skills." + name + if !g.PathExists(canonical) { + r.AddFinding(DoctorFinding{ + ID: id, OK: false, + Summary: fmt.Sprintf("skills: %s missing in ~/.agents/skills/", name), + Hint: "run: sling assist setup", + }) + continue + } + ok, detail, merr := skillMatchesEmbedded(name) + if merr != nil { + r.AddFinding(DoctorFinding{ + ID: id, OK: false, + Summary: fmt.Sprintf("skills: %s could not be compared", name), + Detail: merr.Error(), + }) + continue + } + if !ok { + r.AddFinding(DoctorFinding{ + ID: id, OK: false, + Summary: fmt.Sprintf("skills: %s drifted from embedded", name), + Hint: "run: sling assist setup", + Detail: detail, + }) + continue + } + r.AddFinding(DoctorFinding{ + ID: id, OK: true, + Summary: fmt.Sprintf("skills: %s matches embedded", name), + }) + } + + if !g.PathExists(envFilePath()) { + r.AddFinding(DoctorFinding{ + ID: "env", OK: false, + Summary: fmt.Sprintf("env: %s missing", envFilePath()), + }) + } else { + r.AddFinding(DoctorFinding{ + ID: "env", OK: true, + Summary: fmt.Sprintf("env: %s present", envFilePath()), + }) + } + + stamp, _ := os.ReadFile(VersionFilePath()) + stampVer := strings.TrimSpace(string(stamp)) + switch { + case stampVer == "": + r.AddFinding(DoctorFinding{ + ID: "version", OK: false, + Summary: "version: ~/.sling/assist/version not stamped", + Hint: "run: sling assist setup", + }) + case stampVer == versionUninstalled: + r.AddFinding(DoctorFinding{ + ID: "version", OK: false, + Summary: "version: assist uninstalled", + Hint: "run: sling assist setup", + }) + case stampVer != core.Version: + r.AddFinding(DoctorFinding{ + ID: "version", OK: false, + Summary: fmt.Sprintf("version: stamped %q but binary is %q (auto-refresh did not run)", stampVer, core.Version), + }) + default: + r.AddFinding(DoctorFinding{ + ID: "version", OK: true, + Summary: fmt.Sprintf("version: %s", core.Version), + }) + } + + if err := ctx.Err(); err != nil { + return r, err + } + + r.addZenFinding() + + detected := DetectedClients() + if len(detected) == 0 { + r.AddFinding(DoctorFinding{ + ID: "clients", OK: false, + Summary: "clients: no CLI agent on $PATH", + Hint: "install claude, codex, gemini, cursor, opencode, pi, or grok — or run sling assist setup to install OpenCode", + }) + return r, nil + } + r.Matrix = buildMatrix(ctx, detected, skillNames, scope) + for _, row := range r.Matrix.Rows { + for _, cl := range r.Matrix.Clients { + if row.Cells[cl] == CellFail { + r.OK = false + } + } + } + return r, nil +} + +func buildMatrix(ctx context.Context, detected []Client, skillNames []string, scope Scope) *DoctorMatrix { + m := &DoctorMatrix{} + for _, c := range detected { + m.Clients = append(m.Clients, c.Name()) + } + + authRow := MatrixRow{Label: "auth", Cells: map[string]CellState{}, Notes: map[string]string{}} + for _, c := range detected { + if c.Kind() != KindCLIAgent { + authRow.Cells[c.Name()] = CellEmpty + continue + } + authRow.Cells[c.Name()] = c.AuthState().cell() + } + m.Rows = append(m.Rows, authRow) + + mcpRow := MatrixRow{Label: "MCP", Cells: map[string]CellState{}, Notes: map[string]string{}} + for _, c := range detected { + res := c.CheckMCP(ctx, scope) + mcpRow.Cells[c.Name()] = res.State + if res.State == CellFail && res.Note != "" { + mcpRow.Notes[c.Name()] = res.Note + } + } + m.Rows = append(m.Rows, mcpRow) + + for _, skill := range skillNames { + row := MatrixRow{Label: skill, Cells: map[string]CellState{}, Notes: map[string]string{}} + for _, c := range detected { + results := c.CheckSkills(ctx, []string{skill}, scope) + if len(results) == 0 { + row.Cells[c.Name()] = CellNA + continue + } + res := results[0] + row.Cells[c.Name()] = res.State + if res.State == CellFail && res.Note != "" { + row.Notes[c.Name()] = res.Note + } + } + m.Rows = append(m.Rows, row) + } + return m +} + +// Render is the CLI doctor report (glyph lines + matrix). +func (r *DoctorReport) Render() string { + if r == nil { + return "" + } + var b strings.Builder + for _, line := range r.Lines { + b.WriteString(colorizeDoctorLine(line)) + b.WriteByte('\n') + } + if r.Matrix != nil { + b.WriteByte('\n') + b.WriteString(env.BlueString("Agent × Capability:")) + b.WriteByte('\n') + b.WriteString(r.Matrix.render()) + } + return b.String() +} + +// MissingComponents lists install pieces that look incomplete on disk. +func (r *DoctorReport) MissingComponents() []string { + if r == nil || r.OK { + return nil + } + skillsBad, mcpBad := false, false + for _, f := range r.Findings { + if !f.OK && strings.HasPrefix(f.ID, "skills.") { + skillsBad = true + } + } + for _, line := range r.Lines { + if strings.HasPrefix(strings.TrimSpace(line), "✗ skills") { + skillsBad = true + } + } + if r.Matrix != nil { + for _, row := range r.Matrix.Rows { + for _, c := range r.Matrix.Clients { + if row.Cells[c] == CellFail { + switch row.Label { + case "MCP": + mcpBad = true + case "auth": + default: + skillsBad = true + } + } + } + } + } + var out []string + if skillsBad { + out = append(out, "skills") + } + if mcpBad { + out = append(out, "mcp") + } + return out +} + +func (m *DoctorMatrix) render() string { + t := table.NewWriter() + t.SetStyle(table.StyleRounded) + + header := table.Row{""} + for _, c := range m.Clients { + header = append(header, c) + } + t.AppendHeader(header) + + var notes []string + for _, row := range m.Rows { + r := table.Row{row.Label} + for _, c := range m.Clients { + r = append(r, renderCell(row.Cells[c])) + } + t.AppendRow(r) + for _, c := range m.Clients { + if note, ok := row.Notes[c]; ok && note != "" { + notes = append(notes, fmt.Sprintf(" %s/%s: %s", + env.YellowString(c), env.YellowString(row.Label), note)) + } + } + } + + var colCfgs []table.ColumnConfig + for i := range m.Clients { + colCfgs = append(colCfgs, table.ColumnConfig{ + Number: i + 2, + Align: text.AlignCenter, + AlignHeader: text.AlignCenter, + }) + } + t.SetColumnConfigs(colCfgs) + + out := t.Render() + "\n" + if len(notes) > 0 { + out += "\n " + env.YellowString("Notes:") + "\n" + for _, n := range notes { + out += n + "\n" + } + } + return out +} + +func renderCell(s CellState) string { + switch s { + case CellOK: + return env.GreenString("✓") + case CellFail: + return env.RedString("✗") + case CellNA: + return env.YellowString("⊘") + default: + return env.DarkGrayString("—") + } +} + +func colorizeDoctorLine(line string) string { + trimmed := strings.TrimLeft(line, " ") + indent := line[:len(line)-len(trimmed)] + var prefix, rest string + switch { + case strings.HasPrefix(trimmed, "✓"): + prefix = env.GreenString("✓") + rest = strings.TrimPrefix(trimmed, "✓") + case strings.HasPrefix(trimmed, "✗"): + prefix = env.RedString("✗") + rest = strings.TrimPrefix(trimmed, "✗") + case strings.HasPrefix(trimmed, "⊘"): + prefix = env.YellowString("⊘") + rest = strings.TrimPrefix(trimmed, "⊘") + case strings.HasPrefix(trimmed, "—"): + prefix = env.DarkGrayString("—") + rest = strings.TrimPrefix(trimmed, "—") + default: + return line + } + if idx := strings.Index(rest, "→ run:"); idx >= 0 { + rest = rest[:idx] + env.CyanString(rest[idx:]) + } + return indent + prefix + rest +} diff --git a/core/sling/assist/history.go b/core/sling/assist/history.go new file mode 100644 index 000000000..a895cafba --- /dev/null +++ b/core/sling/assist/history.go @@ -0,0 +1,430 @@ +package assist + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + tea "github.com/charmbracelet/bubbletea" + "github.com/charmbracelet/lipgloss" + "github.com/flarco/g" + "gopkg.in/yaml.v3" +) + +// AnswersFile is the persisted state of an assist run — saved to +// //answers.yaml. +type AnswersFile struct { + SchemaVersion int `yaml:"schema_version" json:"schema_version"` + Name string `yaml:"name" json:"name"` + Task string `yaml:"task" json:"task"` + TemplateVersion string `yaml:"template_version,omitempty" json:"template_version"` + SlingVersion string `yaml:"sling_version" json:"sling_version"` + Created time.Time `yaml:"created" json:"created"` + Agent string `yaml:"agent" json:"agent"` + Parent string `yaml:"parent,omitempty" json:"parent"` + Cwd string `yaml:"cwd" json:"cwd"` + Answers map[string]any `yaml:"answers" json:"answers"` +} + +// Meta is the runtime side-record of an entry — saved to //meta.json. +type Meta struct { + ID string `json:"id"` + Task string `json:"task"` + Agent string `json:"agent"` + Model string `json:"model,omitempty"` + HarnessSessionID string `json:"harness_session_id,omitempty"` + LaunchedAt *time.Time `json:"launched_at"` + Doctor map[string]any `json:"doctor,omitempty"` + Parent string `json:"parent,omitempty"` +} + +// Entry is a loaded view of one history dir, used by the listing/picker. +type Entry struct { + ID string + Path string + Answers AnswersFile + Meta Meta +} + +// SaveEntry writes //{answers.yaml, prompt.md, meta.json}. +// id is generated from Created + slug if empty. +func SaveEntry(a AnswersFile, prompt string, m Meta) (string, error) { + if a.SchemaVersion == 0 { + a.SchemaVersion = SchemaVersion + } + if a.Created.IsZero() { + a.Created = time.Now().UTC() + } + id := m.ID + if id == "" { + slug := slugify(a.Name) + if slug == "" { + slug = slugify(a.Task) + } + id = a.Created.UTC().Format("2006-01-02_15-04-05") + "_" + slug + m.ID = id + } + dir := filepath.Join(HistoryDir(), id) + if err := os.MkdirAll(dir, 0o755); err != nil { + return "", g.Error(err, "mkdir %s", dir) + } + + ay, err := yaml.Marshal(a) + if err != nil { + return "", g.Error(err, "marshal answers") + } + if err := os.WriteFile(filepath.Join(dir, "answers.yaml"), ay, 0o644); err != nil { + return "", err + } + if err := os.WriteFile(filepath.Join(dir, "prompt.md"), []byte(prompt), 0o644); err != nil { + return "", err + } + + e := Entry{ID: id, Path: dir, Answers: a, Meta: m} + if err := e.saveMeta(); err != nil { + return "", err + } + return id, nil +} + +func (e Entry) saveMeta() error { + mj, err := json.MarshalIndent(e.Meta, "", " ") + if err != nil { + return err + } + mj = append(mj, '\n') + return os.WriteFile(filepath.Join(e.Path, "meta.json"), mj, 0o644) +} + +// LoadEntry reads //. +func LoadEntry(id string) (Entry, error) { + dir := filepath.Join(HistoryDir(), id) + e := Entry{ID: id, Path: dir} + ay, err := os.ReadFile(filepath.Join(dir, "answers.yaml")) + if err != nil { + return e, g.Error(err, "read answers") + } + if err := yaml.Unmarshal(ay, &e.Answers); err != nil { + return e, g.Error(err, "parse answers") + } + if mj, err := os.ReadFile(filepath.Join(dir, "meta.json")); err == nil { + _ = json.Unmarshal(mj, &e.Meta) + } + return e, nil +} + +// ListEntries returns all entries in ~/.sling/assist/history/, most-recent first. +func ListEntries() ([]Entry, error) { + root := HistoryDir() + dirs, err := os.ReadDir(root) + if err != nil { + return nil, g.Error(err, "read %s", root) + } + out := []Entry{} + for _, d := range dirs { + if !d.IsDir() || strings.HasPrefix(d.Name(), ".") { + continue + } + e, err := LoadEntry(d.Name()) + if err != nil { + continue + } + out = append(out, e) + } + sort.Slice(out, func(i, j int) bool { + return out[i].Answers.Created.After(out[j].Answers.Created) + }) + return out, nil +} + +// AutoTrim deletes the oldest entries until at most HistoryMaxEntries remain. +func AutoTrim() error { + entries, err := ListEntries() + if err != nil { + return err + } + if len(entries) <= HistoryMaxEntries { + return nil + } + var first error + for _, e := range entries[HistoryMaxEntries:] { + if rmErr := os.RemoveAll(e.Path); rmErr != nil && first == nil { + first = g.Error(rmErr, "remove history %s", e.Path) + } + } + return first +} + +// FormatRelative returns a short human relative time like "3h ago", "yesterday", +// "2026-05-01" (for entries older than a week). +func FormatRelative(t time.Time) string { + d := time.Since(t) + switch { + case d < time.Minute: + return "just now" + case d < time.Hour: + return fmt.Sprintf("%dm ago", int(d.Minutes())) + case d < 24*time.Hour: + return fmt.Sprintf("%dh ago", int(d.Hours())) + case d < 48*time.Hour: + return "yesterday" + case d < 7*24*time.Hour: + return fmt.Sprintf("%d days ago", int(d.Hours()/24)) + default: + return t.Format("2006-01-02") + } +} + +// maxSlugLen caps the slug so _ stays a short directory name. +const maxSlugLen = 40 + +func slugify(s string) string { + s = strings.ToLower(strings.TrimSpace(s)) + out := strings.Builder{} + last := rune(0) + for _, r := range s { + switch { + case r >= 'a' && r <= 'z', r >= '0' && r <= '9': + out.WriteRune(r) + last = r + case r == ' ' || r == '_' || r == '-': + if last == '-' { + continue // collapse runs of separators + } + out.WriteRune('-') + last = '-' + } + } + res := strings.Trim(out.String(), "-") + if res == "" { + return "entry" + } + return truncateSlug(res, maxSlugLen) +} + +// truncateSlug cuts at the last word boundary within n, so the slug stays readable. +func truncateSlug(s string, n int) string { + if len(s) <= n { + return s + } + cut := s[:n] + if i := strings.LastIndexByte(cut, '-'); i > 0 { + cut = cut[:i] + } + return strings.Trim(cut, "-") +} + +func collapseHome(p string) string { + home := userHome() + if home == "" { + return p + } + if strings.HasPrefix(p, home) { + return "~" + strings.TrimPrefix(p, home) + } + return p +} + +func mustGetwd() string { + wd, err := os.Getwd() + if err != nil { + g.Debug("assist: getwd failed: %s", err.Error()) + return "" + } + return wd +} + +// PickHistoryEntry opens a searchable table of recent sessions. +// Returns ErrUserAborted when the user cancels. +func PickHistoryEntry() (Entry, error) { + entries, err := ListEntries() + if err != nil { + return Entry{}, err + } + if len(entries) == 0 { + return Entry{}, g.Error("no sessions yet — run `sling assist` first") + } + if !isTTY(os.Stdin) || !isTTY(os.Stdout) { + return Entry{}, g.Error("pass a session id (`sling assist --resume `) when not on a TTY") + } + + m := newPickerModel(entries) + p := tea.NewProgram(m, tea.WithAltScreen()) + final, err := p.Run() + if err != nil { + return Entry{}, g.Error(err, "session picker") + } + got, ok := final.(pickerModel) + if !ok || got.chosen == nil { + return Entry{}, ErrUserAborted + } + return *got.chosen, nil +} + +func filterEntries(entries []Entry, query string) []Entry { + q := strings.ToLower(strings.TrimSpace(query)) + if q == "" { + return entries + } + out := []Entry{} + for _, e := range entries { + if entryMatches(e, q) { + out = append(out, e) + } + } + return out +} + +func entryMatches(e Entry, q string) bool { + ask := "" + if e.Answers.Answers != nil { + if v, ok := e.Answers.Answers["ask"].(string); ok { + ask = v + } + if v, ok := e.Answers.Answers["intention"].(string); ok && ask == "" { + ask = v + } + } + hay := strings.ToLower(strings.Join([]string{ + e.ID, e.Answers.Name, e.Answers.Task, e.Answers.Agent, e.Answers.Cwd, ask, e.Meta.Agent, + }, " ")) + return strings.Contains(hay, q) +} + +type pickerModel struct { + all []Entry + filtered []Entry + query string + cursor int + chosen *Entry + width int + height int + quit bool +} + +func newPickerModel(entries []Entry) pickerModel { + return pickerModel{ + all: entries, + filtered: entries, + width: 80, + height: 24, + } +} + +func (m pickerModel) Init() tea.Cmd { return nil } + +func (m pickerModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { + switch msg := msg.(type) { + case tea.WindowSizeMsg: + m.width = msg.Width + m.height = msg.Height + case tea.KeyMsg: + switch msg.String() { + case "ctrl+c", "esc": + m.quit = true + return m, tea.Quit + case "enter": + if m.cursor >= 0 && m.cursor < len(m.filtered) { + e := m.filtered[m.cursor] + m.chosen = &e + } + return m, tea.Quit + case "up", "k": + if m.cursor > 0 { + m.cursor-- + } + case "down", "j": + if m.cursor < len(m.filtered)-1 { + m.cursor++ + } + case "backspace": + if m.query != "" { + r := []rune(m.query) + m.query = string(r[:len(r)-1]) + m.filtered = filterEntries(m.all, m.query) + if m.cursor >= len(m.filtered) { + m.cursor = max(0, len(m.filtered)-1) + } + } + default: + if msg.Type == tea.KeyRunes { + m.query += string(msg.Runes) + m.filtered = filterEntries(m.all, m.query) + m.cursor = 0 + } + } + } + return m, nil +} + +func (m pickerModel) View() string { + var b strings.Builder + title := lipgloss.NewStyle().Bold(true).Render("Resume session") + fmt.Fprintf(&b, "%s\n", title) + fmt.Fprintf(&b, " search: %s█\n\n", m.query) + + header := fmt.Sprintf(" %-28s %-16s %-10s %-12s %s", "ID", "TASK", "AGENT", "CREATED", "NAME") + fmt.Fprintln(&b, lipgloss.NewStyle().Faint(true).Render(header)) + + if len(m.filtered) == 0 { + fmt.Fprintln(&b, " (no matches)") + return b.String() + } + + rows := m.height - 8 + if rows < 3 { + rows = 3 + } + start := 0 + if m.cursor >= rows { + start = m.cursor - rows + 1 + } + end := start + rows + if end > len(m.filtered) { + end = len(m.filtered) + } + + sel := lipgloss.NewStyle().Reverse(true) + for i := start; i < end; i++ { + e := m.filtered[i] + agent := e.Answers.Agent + if agent == "" { + agent = e.Meta.Agent + } + if agent == "" { + agent = "—" + } + line := fmt.Sprintf(" %-28s %-16s %-10s %-12s %s", + clipRunes(e.ID, 28), + clipRunes(e.Answers.Task, 16), + clipRunes(agent, 10), + FormatRelative(e.Answers.Created), + clipRunes(e.Answers.Name, 24), + ) + if i == m.cursor { + line = sel.Render(line) + } + fmt.Fprintln(&b, line) + } + fmt.Fprintln(&b) + fmt.Fprintln(&b, lipgloss.NewStyle().Faint(true).Render(" ↑/↓ move enter resume esc abort")) + return b.String() +} + +func clipRunes(s string, n int) string { + if n <= 0 { + return "" + } + r := []rune(s) + if len(r) <= n { + return s + } + if n == 1 { + return "…" + } + return string(r[:n-1]) + "…" +} diff --git a/core/sling/assist/install.go b/core/sling/assist/install.go new file mode 100644 index 000000000..ebd3f0ff0 --- /dev/null +++ b/core/sling/assist/install.go @@ -0,0 +1,746 @@ +package assist + +import ( + "context" + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/charmbracelet/huh" + "github.com/flarco/g" + "github.com/slingdata-io/sling-cli/core" +) + +// InstallOptions controls `sling assist setup` (install path). +type InstallOptions struct { + Reconfigure bool + SkillsOnly bool + MCPOnly bool + Scope Scope + NonInteractive bool + DefaultAgent string // used only on non-interactive first-run +} + +// InstallResult summarizes what install touched. +type InstallResult struct { + ProfileWritten bool + CanonicalSkillsDir string + WiredClients []ClientResult + SkippedClients []string // not detected +} + +// ClientResult is one row of the install summary. +type ClientResult struct { + Name string + Kind ClientKind + WroteSkills bool + WroteMCP bool + Authed AuthStatus + Notes string +} + +func requestedAgent(opts InstallOptions) string { + agent := strings.ToLower(strings.TrimSpace(opts.DefaultAgent)) + if agent != "" { + return agent + } + if prof, exists, err := LoadProfile(); err == nil && exists { + return strings.ToLower(strings.TrimSpace(prof.Agent)) + } + return "" +} + +func anyUsableCLIAgent() bool { + for _, c := range CLIAgents() { + if c.Detect() { + return true + } + } + return false +} + +// maybeEnsureOpenCode downloads opencode only when the user picked it, or when +// no other CLI agent is usable (bundled fallback). System binaries still win. +func maybeEnsureOpenCode(opts InstallOptions) error { + agent := requestedAgent(opts) + switch { + case agent == "opencode": + // user picked opencode + case agent != "" && agent != "auto": + return nil + case anyUsableCLIAgent(): + return nil + } + if _, err := EnsureBinOpenCode(); err != nil { + return err + } + return ApplyHarnessProviderConfig() +} + +// Install is idempotent install/refresh. Honors ctx between clients. +func Install(ctx context.Context, opts InstallOptions) (*InstallResult, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, err + } + res := &InstallResult{ + CanonicalSkillsDir: CanonicalSkillsDir(), + } + + if err := maybeEnsureOpenCode(opts); err != nil { + return nil, err + } + if !opts.SkillsOnly { + if err := maybeEnsureAgentBrowser(opts); err != nil { + return nil, err + } + } + + detected := DetectedClients() + if len(detected) == 0 { + return nil, g.Error("no AI agent on $PATH; install one of: claude, codex, gemini, cursor, opencode, pi, grok") + } + + prof, exists, err := LoadProfile() + if err != nil { + return nil, err + } + if !exists || opts.Reconfigure { + prof = DefaultProfile() + switch { + case opts.DefaultAgent != "": + prof.Agent = opts.DefaultAgent + default: + for _, c := range detected { + if c.Kind() == KindCLIAgent { + prof.Agent = c.Name() + break + } + } + if prof.Agent == "" { + prof.Agent = "auto" + } + } + if err := SaveProfile(prof); err != nil { + return nil, err + } + res.ProfileWritten = true + } + + skillNames := listSkillNames() + if !opts.MCPOnly { + if err := writeCanonicalBundle(skillNames); err != nil { + return nil, err + } + pruneRetiredSkills(ctx, opts.Scope) + } + + var wireErrs []string + for _, c := range detected { + if err := ctx.Err(); err != nil { + return res, err + } + row := ClientResult{Name: c.Name(), Kind: c.Kind(), Authed: c.AuthState()} + if !opts.MCPOnly { + if err := c.WriteSkills(ctx, skillNames, opts.Scope); err != nil { + row.Notes = fmt.Sprintf("skills: %v", err) + wireErrs = append(wireErrs, fmt.Sprintf("%s skills: %v", c.Name(), err)) + } else { + row.WroteSkills = true + } + } + if !opts.SkillsOnly { + if err := c.WriteMCP(ctx, opts.Scope); err != nil { + if row.Notes != "" { + row.Notes += "; " + } + row.Notes += fmt.Sprintf("mcp: %v", err) + wireErrs = append(wireErrs, fmt.Sprintf("%s mcp: %v", c.Name(), err)) + } else { + row.WroteMCP = true + } + } + res.WiredClients = append(res.WiredClients, row) + } + for _, c := range AllClients() { + if !c.Detect() { + res.SkippedClients = append(res.SkippedClients, c.Name()) + } + } + + if len(wireErrs) > 0 { + // Do not stamp on partial failure — retry on next install. + return res, g.Error("install incomplete: %s", strings.Join(wireErrs, "; ")) + } + + if err := os.WriteFile(VersionFilePath(), []byte(core.Version), 0o644); err != nil { + return res, g.Error(err, "could not stamp %s", VersionFilePath()) + } + + return res, nil +} + +// UninstallOptions controls `sling assist setup --uninstall`. +type UninstallOptions struct { + All bool + SkillsOnly bool + MCPOnly bool + Scope Scope + NonInteractive bool + IncludeClients []string // empty = all detected +} + +// versionUninstalled prevents AutoRefresh from resurrecting after uninstall. +const versionUninstalled = "uninstalled" + +// Uninstall removes sling skills/MCP only (never other tools' entries). +func Uninstall(ctx context.Context, opts UninstallOptions) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + skillNames := listSkillNames() + + pickClient := func(name string) bool { + if len(opts.IncludeClients) == 0 { + return true + } + for _, n := range opts.IncludeClients { + if n == name { + return true + } + } + return false + } + + var errs []string + for _, c := range AllClients() { + if err := ctx.Err(); err != nil { + return err + } + if !c.Detect() { + continue + } + if !pickClient(c.Name()) { + continue + } + if !opts.MCPOnly { + if err := c.RemoveSkills(ctx, skillNames, opts.Scope); err != nil { + errs = append(errs, fmt.Sprintf("%s skills: %v", c.Name(), err)) + } + } + if !opts.SkillsOnly { + if err := c.RemoveMCP(ctx, opts.Scope); err != nil { + errs = append(errs, fmt.Sprintf("%s mcp: %v", c.Name(), err)) + } + } + } + if !opts.MCPOnly { + if err := removeCanonicalBundle(skillNames); err != nil { + errs = append(errs, err.Error()) + } + // Mark uninstalled so AutoRefresh will not re-wire on upgrade. + if err := os.MkdirAll(AssistDir(), 0o755); err != nil { + errs = append(errs, fmt.Sprintf("mkdir assist: %v", err)) + } else if err := os.WriteFile(VersionFilePath(), []byte(versionUninstalled), 0o644); err != nil { + errs = append(errs, fmt.Sprintf("stamp uninstalled: %v", err)) + } + } + if len(errs) > 0 { + return g.Error("uninstall completed with errors: %s", strings.Join(errs, "; ")) + } + return nil +} + +// AutoRefresh updates installed skills to match the embedded bundle. +// It also removes retired skill directories. +// This function runs on each `sling assist` command. +// If no current skills exist, it does not install them. +// If the user ran uninstall, it does not install them again. +// Returns a notice line when something changed. +func AutoRefresh(ctx context.Context) (string, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return "", err + } + + stamp, err := os.ReadFile(VersionFilePath()) + if err != nil && !os.IsNotExist(err) { + return "", g.Error(err, "could not read %s", VersionFilePath()) + } + s := strings.TrimSpace(string(stamp)) + if s == versionUninstalled { + return "", nil + } + + if !anyCurrentSkillsInstalled() { + // Remove leftover retired dirs. Do not install the current bundle. + pruneRetiredSkills(ctx, ScopeUser) + return "", nil + } + + lockPath := filepath.Join(AssistDir(), ".refresh-lock") + unlock, ok := tryRefreshLock(lockPath) + if !ok { + return "", nil + } + defer unlock() + + skillNames := listSkillNames() + changed := []string{} + var refreshErrs []string + for _, name := range skillNames { + if err := ctx.Err(); err != nil { + return "", err + } + didChange, err := syncCanonicalSkill(name) + if err != nil { + refreshErrs = append(refreshErrs, fmt.Sprintf("%s: %v", name, err)) + continue + } + if didChange { + changed = append(changed, name) + } + } + pruned := pruneRetiredSkills(ctx, ScopeUser) + + // Files already match the embed and the stamp is current. Skip re-wire. + if len(changed) == 0 && len(pruned) == 0 && s == core.Version && len(refreshErrs) == 0 { + return "", nil + } + + clients := []string{} + for _, c := range DetectedClients() { + if err := ctx.Err(); err != nil { + return "", err + } + if err := c.WriteSkills(ctx, skillNames, ScopeUser); err != nil { + refreshErrs = append(refreshErrs, fmt.Sprintf("%s: %v", c.Name(), err)) + continue + } + clients = append(clients, c.Name()) + } + + // Only stamp on full success so partial failure retries next time. + if len(refreshErrs) > 0 { + return "", g.Error("auto-refresh incomplete: %s", strings.Join(refreshErrs, "; ")) + } + if err := os.WriteFile(VersionFilePath(), []byte(core.Version), 0o644); err != nil { + return "", g.Error(err, "could not stamp %s", VersionFilePath()) + } + + if len(changed) == 0 && len(pruned) == 0 { + return "", nil + } + return fmt.Sprintf("sling: refreshed AI skills for v%s (%s)", core.Version, strings.Join(clients, ", ")), nil +} + +const refreshLockStale = 5 * time.Minute + +// tryRefreshLock acquires an exclusive lock file; unlock removes only if we still own it. +func tryRefreshLock(lockPath string) (func(), bool) { + token := fmt.Sprintf("%d-%d", os.Getpid(), time.Now().UnixNano()) + + create := func() (*os.File, error) { + return os.OpenFile(lockPath, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644) + } + + f, err := create() + if err != nil { + if !os.IsExist(err) { + return nil, false + } + info, statErr := os.Stat(lockPath) + if statErr != nil || time.Since(info.ModTime()) < refreshLockStale { + return nil, false + } + // Stale lock: reclaim (racing peer loses on O_EXCL). + _ = os.Remove(lockPath) + f, err = create() + if err != nil { + return nil, false + } + } + _, _ = f.Write([]byte(token)) + _ = f.Close() + + unlock := func() { + got, err := os.ReadFile(lockPath) + if err == nil && string(got) == token { + _ = os.Remove(lockPath) + } + } + return unlock, true +} + +// retiredSkillNames are Sling-owned skill names from earlier bundle versions +// that no longer exist in the embed. Pruned on install/refresh so agents stop +// picking up their stale content. +var retiredSkillNames = []string{"sling-hooks", "sling-transforms", "sling-troubleshooting"} + +// pruneRetiredSkills removes retired skills from the canonical bundle and from +// per-skill client redirects. Best-effort: failures only debug-log. +// vscode is skipped — RemoveSkills unwires the whole bundle, +// and it references the canonical dir, so the canonical prune covers it. +// Returns the retired names that were present on disk and removed. +func pruneRetiredSkills(ctx context.Context, scope Scope) []string { + root := CanonicalSkillsDir() + pruned := []string{} + for _, name := range retiredSkillNames { + p := filepath.Join(root, name) + if !g.PathExists(p) { + continue + } + if err := os.RemoveAll(p); err != nil { + g.Debug("assist: prune retired skill %s: %s", name, err.Error()) + continue + } + pruned = append(pruned, name) + } + for _, c := range DetectedClients() { + if ctx.Err() != nil { + return pruned + } + if c.Name() == "vscode" { + continue + } + if err := c.RemoveSkills(ctx, retiredSkillNames, scope); err != nil { + g.Debug("assist: prune retired skills for %s: %s", c.Name(), err.Error()) + } + } + return pruned +} + +// anyCurrentSkillsInstalled reports whether at least one embedded skill +// is present in ~/.agents/skills/. Absence means the user has not run setup. +func anyCurrentSkillsInstalled() bool { + for _, name := range listSkillNames() { + if g.PathExists(canonicalSkillPath(name)) { + return true + } + } + return false +} + +// MD5OfFile returns the MD5 hex of a file. +func MD5OfFile(path string) (string, error) { + data, err := os.ReadFile(path) + if err != nil { + return "", err + } + sum := md5.Sum(data) + return hex.EncodeToString(sum[:]), nil +} + +// MD5OfEmbeddedSkill returns the MD5 of an embedded skill file. +func MD5OfEmbeddedSkill(rel string) (string, error) { + data, err := SkillsFS.ReadFile(filepath.ToSlash(filepath.Join("skills", rel))) + if err != nil { + return "", err + } + sum := md5.Sum(data) + return hex.EncodeToString(sum[:]), nil +} + +// SetupAction is what RunSetupActionForm returns. Drives the post-doctor +// branching in `sling assist setup` for users who already have a profile. +type SetupAction string + +const ( + SetupActionRefresh SetupAction = "refresh" // re-install everything (idempotent) + SetupActionInstallMissing SetupAction = "install_missing" // install only the failing components + SetupActionReconfigure SetupAction = "reconfigure" // re-prompt the profile form, then install + SetupActionUninstall SetupAction = "uninstall" // wipe everything + SetupActionExit SetupAction = "exit" // do nothing +) + +// ErrUserAborted is returned by interactive forms when the user declines. +var ErrUserAborted = errors.New("user aborted") + +// RunSetupActionForm runs after doctor has printed its report. +func RunSetupActionForm(report *DoctorReport) (SetupAction, error) { + missingLabel := "Install missing components" + hasFailures := report != nil && !report.OK + opts := []huh.Option[string]{} + if hasFailures { + opts = append(opts, huh.NewOption(missingLabel+" (recommended)", string(SetupActionInstallMissing))) + opts = append(opts, huh.NewOption("Re-install everything (refresh)", string(SetupActionRefresh))) + } else { + opts = append(opts, huh.NewOption("Re-install everything (refresh)", string(SetupActionRefresh))) + } + opts = append(opts, + huh.NewOption("Reconfigure (change preferred agent / scope)", string(SetupActionReconfigure)), + huh.NewOption("Uninstall everything", string(SetupActionUninstall)), + huh.NewOption("Exit (do nothing)", string(SetupActionExit)), + ) + + chosen := opts[0].Value + form := huh.NewForm( + huh.NewGroup( + huh.NewSelect[string](). + Title("What would you like to do?"). + Description("Doctor already ran — pick your next action."). + Options(opts...). + Value(&chosen), + ), + ).WithTheme(huh.ThemeCharm()) + if err := form.Run(); err != nil { + return SetupActionExit, g.Error(err, "setup form aborted") + } + return SetupAction(chosen), nil +} + +// HarnessConfirmResult is the first-run / setup confirm form. +type HarnessConfirmResult struct { + Agent string + HintInErrors bool + Scope string + Components []string +} + +func agentAuthLabel(a RankedAgent) string { + if a.Bundled { + choice := HarnessProviderChoice() + if choice.Disclosure != "" { + return fmt.Sprintf("%s (bundled fallback) — %s", a.Name, choice.Disclosure) + } + return fmt.Sprintf("%s (bundled fallback, keyed provider)", a.Name) + } + auth := "no auth" + switch a.Auth { + case AuthOK: + auth = "auth ok" + case AuthUnknown: + auth = "auth unknown" + } + return fmt.Sprintf("%s (detected, %s)", a.Name, auth) +} + +func harnessAgentDescription(ranked []RankedAgent) string { + base := "Detected agents with install and auth state. Authenticated first." + hasBundled := false + for _, a := range ranked { + if a.Bundled { + hasBundled = true + break + } + } + if !hasBundled { + return base + } + choice := HarnessProviderChoice() + if choice.Disclosure != "" { + return base + " OpenCode downloads only when no other agent is usable. Keyed alternative: set ANTHROPIC_API_KEY or OPENAI_API_KEY. " + choice.Disclosure + "." + } + return base + " OpenCode downloads only when no other agent is usable." +} + +// RunHarnessConfirmForm lists detected agents with install+auth state. +func RunHarnessConfirmForm(prefill Profile) (*HarnessConfirmResult, error) { + agent, err := resolveSetupAgent(prefill.Agent) + if err != nil { + return nil, err + } + + res := &HarnessConfirmResult{ + Agent: agent, + HintInErrors: prefill.HintInErrors, + Scope: "user", + Components: []string{"skills", "mcp"}, + } + if prefill.DefaultInstallScope != "" { + res.Scope = prefill.DefaultInstallScope + } + + fields := []huh.Field{ + huh.NewMultiSelect[string](). + Title("What would you like to install?"). + Description("Skills are markdown guides; MCP wires the Sling MCP server into each client."). + Options( + huh.NewOption("Skills (canonical bundle + per-client redirects)", "skills").Selected(true), + huh.NewOption("MCP server registration", "mcp").Selected(true), + ). + Value(&res.Components), + huh.NewConfirm(). + Title("Show AI hint in run errors?"). + Description("Append `sling assist error ` to failed `sling run` / `conns test` output."). + Value(&res.HintInErrors), + huh.NewSelect[string](). + Title("Install scope"). + Description("Where to wire skills + MCP. `user` writes to ~/./; `project` writes to ././"). + Options( + huh.NewOption("user (recommended)", "user"), + huh.NewOption("project", "project"), + ). + Value(&res.Scope), + } + if opts, ranked, ok := setupAgentSelectOptions(); ok { + fields = append([]huh.Field{ + huh.NewSelect[string](). + Title("Preferred agent"). + Description(harnessAgentDescription(ranked)). + Options(opts...). + Filtering(filteringFor(opts)). + Value(&res.Agent), + }, fields...) + } + + form := huh.NewForm(huh.NewGroup(fields...)).WithTheme(huh.ThemeCharm()) + if err := form.Run(); err != nil { + return nil, g.Error(err, "setup form aborted") + } + if len(res.Components) == 0 { + return nil, g.Error("no components selected") + } + return res, nil +} + +// InstallFormResult is what the first-run install form returns. +type InstallFormResult struct { + Agent string + HintInErrors bool + Scope string +} + +// RunInstallForm shows the first-run interactive form and returns the user's +// choices. +func RunInstallForm(prefill Profile) (*InstallFormResult, error) { + agent, err := resolveSetupAgent(prefill.Agent) + if err != nil { + return nil, err + } + + res := &InstallFormResult{ + Agent: agent, + HintInErrors: prefill.HintInErrors, + Scope: "user", + } + if prefill.DefaultInstallScope != "" { + res.Scope = prefill.DefaultInstallScope + } + + fields := []huh.Field{ + huh.NewConfirm(). + Title("Show AI hint in run errors?"). + Description("Append `sling assist error ` to failed `sling run` / `conns test` output."). + Value(&res.HintInErrors), + huh.NewSelect[string](). + Title("Install scope"). + Description("Where to wire skills + MCP. `user` writes to ~/./; `project` writes to ././"). + Options( + huh.NewOption("user (recommended)", "user"), + huh.NewOption("project", "project"), + ). + Value(&res.Scope), + } + if opts, ranked, ok := setupAgentSelectOptions(); ok { + fields = append([]huh.Field{ + huh.NewSelect[string](). + Title("Preferred agent"). + Description(harnessAgentDescription(ranked)). + Options(opts...). + Filtering(filteringFor(opts)). + Value(&res.Agent), + }, fields...) + } + + form := huh.NewForm(huh.NewGroup(fields...)).WithTheme(huh.ThemeCharm()) + if err := form.Run(); err != nil { + return nil, g.Error(err, "install form aborted") + } + return res, nil +} + +// resolveSetupAgent picks the agent for setup: the only one on PATH, else +// the authenticated one (via a later picker), else bundled OpenCode after confirm. +func resolveSetupAgent(current string) (string, error) { + agents, bundled := pathRanked() + switch { + case len(agents) == 0: + if bundled == nil { + return "", g.Error("no AI agent on $PATH; install one of: claude, codex, gemini, cursor, opencode, pi, grok") + } + if err := confirmInstallOpenCode(); err != nil { + return "", err + } + return bundled.Name, nil + case len(agents) == 1: + return agents[0].Name, nil + default: + if current != "" { + for _, a := range agents { + if a.Name == current { + return current, nil + } + } + } + return agents[0].Name, nil + } +} + +// setupAgentSelectOptions is the picker for two or more PATH agents. +func setupAgentSelectOptions() ([]huh.Option[string], []RankedAgent, bool) { + agents, _ := pathRanked() + if len(agents) < 2 { + return nil, agents, false + } + opts := make([]huh.Option[string], 0, len(agents)) + for i, a := range agents { + label := agentAuthLabel(a) + if i == 0 { + label += " — recommended" + } + opts = append(opts, huh.NewOption(label, a.Name)) + } + return opts, agents, true +} + +func confirmInstallOpenCode() error { + ok := false + form := huh.NewForm( + huh.NewGroup( + huh.NewConfirm(). + Title("No AI agent on $PATH"). + Description("Sling can install OpenCode and use it as the agent."). + Affirmative("Install OpenCode"). + Negative("Cancel"). + Value(&ok), + ), + ).WithTheme(huh.ThemeCharm()) + if err := form.Run(); err != nil { + return g.Error(err, "setup form aborted") + } + if !ok { + return ErrUserAborted + } + return nil +} + +func EnsureAssistReady() error { + prof, exists, err := LoadProfile() + if err != nil { + return err + } + if !exists { + return g.Error("Sling assist is not set up yet. Run:\n\n sling assist setup\n") + } + if _, err := ResolveAgent("", prof); err != nil { + return g.Error("no AI agent ready for assist. Run:\n\n sling assist setup\n\n(%s)", err.Error()) + } + return nil +} + +func filteringFor(opts []huh.Option[string]) bool { + return len(opts) >= 6 +} diff --git a/core/sling/assist/investigate.go b/core/sling/assist/investigate.go new file mode 100644 index 000000000..c63d5b612 --- /dev/null +++ b/core/sling/assist/investigate.go @@ -0,0 +1,1169 @@ +// Investigate surface: error signatures, snapshots, local exec picker, hints, sensitivity. + +package assist + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strings" + "time" + "unicode" + + "github.com/flarco/g" + "github.com/slingdata-io/sling-cli/core/dbio" + "github.com/slingdata-io/sling-cli/core/env" + "github.com/spf13/cast" +) + +// AlgorithmVersion labels the normalizer generation (not a material prefix). +const AlgorithmVersion = "v1" + +// Material prefixes distinguish pattern vs edge digests (composite signature). +const ( + PatternMaterialPrefix = "v1p" // skeleton-only + EdgeMaterialPrefix = "v1e" // source|target|skeleton +) + +// PartIDLen is hex chars kept for each of pattern and edge digests. +const PartIDLen = 8 + +// CompositeIDLen is PatternID + EdgeID (16 hex chars, no separator). +const CompositeIDLen = PartIDLen * 2 + +// SignatureIDLen is an alias for PartIDLen. +const SignatureIDLen = PartIDLen + +// SignMeta holds connector types only (never hostnames, DB names, or task type). +// Task type (db-db, file-db, …) is not stored: it is fully determined by the +// Kind of SourceType and TargetType (see InferredTaskType). +type SignMeta struct { + SourceType dbio.Type // e.g. dbio.TypeDbPrometheus + TargetType dbio.Type // e.g. dbio.TypeDbPostgres +} + +func MakeSignMeta() (sm SignMeta) { + taskMap, _ := g.UnmarshalMap(cast.ToString(env.TelMap["task"])) + src := cast.ToString(taskMap["source_type"]) + tgt := cast.ToString(taskMap["target_type"]) + if src == "" { + src = cast.ToString(env.TelMap["conn_type"]) + } + return SignMeta{SourceType: dbio.Type(src), TargetType: dbio.Type(tgt)} +} + +// Signature is the full result of SignError (composite: pattern + edge). +// +// PatternID = hex(sha256("v1p|" + skeleton))[0:8] +// EdgeID = hex(sha256("v1e|" + src + "|" + tgt + "|" + skeleton))[0:8] +// ID = PatternID + EdgeID // 16 hex, machine form +// IDDashed = PatternID + "-" + EdgeID +// +// Same skeleton ⇒ same PatternID across all source/target pairs. +// Same skeleton + same types ⇒ same EdgeID and ID. +type Signature struct { + ID string // 16 hex chars (pattern||edge) + PatternID string // 8 hex — skeleton only + EdgeID string // 8 hex — types + skeleton + Skeleton string // normalized message body + PatternMaterial string // v1p| + EdgeMaterial string // v1e||| + ShortLabel string // human-only slug (not hashed) + Algorithm string // e.g. v1 + Meta SignMeta +} + +// SignError normalizes errText + meta into a stable composite signature. +// Pure function: no I/O, deterministic across machines. +func SignError(errText string, meta SignMeta) Signature { + meta = normalizeMeta(meta) + skel := Skeleton(errText) + pMat := PatternMaterial(skel) + eMat := EdgeMaterial(skel, meta) + patternID := HashPart(pMat) + edgeID := HashPart(eMat) + return Signature{ + ID: patternID + edgeID, + PatternID: patternID, + EdgeID: edgeID, + Skeleton: skel, + PatternMaterial: pMat, + EdgeMaterial: eMat, + ShortLabel: ShortLabel(skel), + Algorithm: AlgorithmVersion, + Meta: meta, + } +} + +// IDDashed returns pattern-edge with a hyphen for humans / copy-paste. +func (s Signature) IDDashed() string { + if s.PatternID == "" || s.EdgeID == "" { + if len(s.ID) == CompositeIDLen { + return s.ID[:PartIDLen] + "-" + s.ID[PartIDLen:] + } + return s.ID + } + return s.PatternID + "-" + s.EdgeID +} + +// Display formats the signature for the failure footer. +// Example: a1b2c3d4-e4f9c2a1 (prometheus→postgres · no_stream_columns) +func (s Signature) Display() string { + src := typeToken(s.Meta.SourceType) + tgt := typeToken(s.Meta.TargetType) + label := s.ShortLabel + if label == "" { + label = "unknown" + } + return fmt.Sprintf("%s (%s→%s · %s)", s.IDDashed(), src, tgt, label) +} + +// ParseSignatureID normalizes user input into compact 8 or 16 hex lowercase. +// Accepts "aabbccdd", "aabbccdd-eeff0011", "aabbccddeeff0011", or a Display() line. +// Returns compact hex and whether it is pattern-only (len 8) vs full composite (len 16). +func ParseSignatureID(raw string) (compact string, patternOnly bool, err error) { + s := strings.ToLower(strings.TrimSpace(raw)) + if i := strings.IndexAny(s, " \t("); i > 0 { + s = s[:i] + } + s = strings.ReplaceAll(s, "-", "") + if len(s) != PartIDLen && len(s) != CompositeIDLen { + return "", false, fmt.Errorf("invalid error signature %q (expected 8 or 16 hex chars)", raw) + } + for _, r := range s { + if r < '0' || (r > '9' && r < 'a') || r > 'f' { + return "", false, fmt.Errorf("invalid error signature %q (expected hex)", raw) + } + } + return s, len(s) == PartIDLen, nil +} + +// InferredTaskType returns the job-type slug (db-db, file-db, api-file, …) +// derived from connector kinds. Empty when either side is unknown. +// Not part of the hash material — source+target types already encode this. +func (m SignMeta) InferredTaskType() string { + m = normalizeMeta(m) + sk, tk := kindAbbrev(m.SourceType.Kind()), kindAbbrev(m.TargetType.Kind()) + if sk == "" || tk == "" { + return "" + } + return sk + "-" + tk +} + +func kindAbbrev(k dbio.Kind) string { + switch k { + case dbio.KindDatabase: + return "db" + case dbio.KindFile: + return "file" + case dbio.KindAPI: + return "api" + default: + return "" + } +} + +func normalizeMeta(m SignMeta) SignMeta { + return SignMeta{ + SourceType: dbio.Type(strings.ToLower(strings.TrimSpace(string(m.SourceType)))), + TargetType: dbio.Type(strings.ToLower(strings.TrimSpace(string(m.TargetType)))), + } +} + +func typeToken(t dbio.Type) string { + s := strings.TrimSpace(string(t)) + if s == "" { + return "-" + } + return s +} + +// PatternMaterial builds the pattern-layer hash input (skeleton only). +// +// v1p| +func PatternMaterial(skeleton string) string { + if skeleton == "" { + skeleton = "unknown_error" + } + return PatternMaterialPrefix + "|" + skeleton +} + +// EdgeMaterial builds the edge-layer hash input (types + skeleton). +// +// v1e||| +// +// Task type is omitted: it is redundant given source and target connector types. +func EdgeMaterial(skeleton string, meta SignMeta) string { + meta = normalizeMeta(meta) + if skeleton == "" { + skeleton = "unknown_error" + } + return strings.Join([]string{ + EdgeMaterialPrefix, + typeToken(meta.SourceType), + typeToken(meta.TargetType), + skeleton, + }, "|") +} + +// Material is an alias for EdgeMaterial (edge-layer input). +func Material(skeleton string, meta SignMeta) string { + return EdgeMaterial(skeleton, meta) +} + +// HashPart returns the first PartIDLen hex chars of SHA-256(material). +func HashPart(material string) string { + sum := sha256.Sum256([]byte(material)) + return hex.EncodeToString(sum[:])[:PartIDLen] +} + +// HashMaterial is an alias for HashPart. +func HashMaterial(material string) string { + return HashPart(material) +} + +// ShortLabel is a human-only slug from the skeleton (not part of the hash). +func ShortLabel(skeleton string) string { + if skeleton == "" || skeleton == "unknown_error" { + return "unknown_error" + } + lines := strings.Split(skeleton, "\n") + // Prefer a vendor code when present (stable, short). + for i := len(lines) - 1; i >= 0; i-- { + if strings.HasPrefix(lines[i], "code:") { + return slugLabel(strings.TrimPrefix(lines[i], "code:")) + } + } + // Else last non-empty message line (usually the leaf driver message). + pick := lines[0] + for i := len(lines) - 1; i >= 0; i-- { + l := lines[i] + if l == "" { + continue + } + pick = l + break + } + return slugLabel(pick) +} + +func slugLabel(pick string) string { + // Slugify: non-alnum → _, collapse + var b strings.Builder + prevUnderscore := false + for _, r := range pick { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + b.WriteRune(r) + prevUnderscore = false + continue + } + if !prevUnderscore { + b.WriteByte('_') + prevUnderscore = true + } + } + s := strings.Trim(b.String(), "_") + if len(s) > 40 { + s = s[:40] + s = strings.TrimRight(s, "_") + } + if s == "" { + return "unknown_error" + } + return s +} + +// --- normalization ----------------------------------------------------------- + +var ( + // Stack frames: --- task_run.go:140 func2 --- + reStackFrame = regexp.MustCompile(`(?i)^---\s+\S+\.go:\d+\s+`) + + // Vendor / driver codes (uppercase-ish tokens in brackets or parentheses) + reBracketCode = regexp.MustCompile(`\[([A-Z][A-Z0-9_.]+)\]`) + reParenCode = regexp.MustCompile(`\(([A-Z][A-Z0-9_]{2,})\)`) + reSQLState = regexp.MustCompile(`(?i)\bSQLSTATE[:\s]+([0-9A-Z]{5})\b`) + reCHCode = regexp.MustCompile(`\bCode:\s*(\d+)\b`) + + // Placeholders — order matters (more specific first). + // Quoted URL / path before bare forms so later "" does not swallow them. + // URLs on slingdata.io (or subdomains) are kept as-is (see replaceURL). + reQuotedURL = regexp.MustCompile(`(?i)"(?:https?|s3|gs|file|azure|abfs|abfss)://[^"]*"`) + reQuotedPath = regexp.MustCompile(`"(?:/|~/|~\\)[^"]*"`) + reURL = regexp.MustCompile(`(?i)\b(?:https?|s3|gs|file|azure|abfs|abfss)://[^\s"'<>]+`) + // Absolute / home paths (unix + windows drive). Match path token only. + reUnixPath = regexp.MustCompile(`(?:^|[\s"'=(])(/[^\s"'<>]+)`) + reWinPath = regexp.MustCompile(`(?i)(?:^|[\s"'=(])([a-z]:\\[^\s"'<>]+)`) + reHomePath = regexp.MustCompile(`(?:^|[\s"'=(])(~/[^\s"'<>]*)`) + reUUID = regexp.MustCompile(`(?i)\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b`) + // ksuid / sling exec ids / long hex (exclude already-replaced tokens) + reLongID = regexp.MustCompile(`\b(?:exec_[A-Za-z0-9]+|[0-9A-Za-z]{24,}|[0-9a-fA-F]{16,})\b`) + reISOTs = regexp.MustCompile(`\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?\b`) + // IPv4 (+ optional port) + reIPv4 = regexp.MustCompile(`\b\d{1,3}(?:\.\d{1,3}){3}(?::\d{1,5})?\b`) + // Quoted / backtick identifiers (no spaces — keeps prose messages intact). + // Matches schema.table, columns, etc. after URL/path quoted forms. + reIdentDQ = regexp.MustCompile(`"[^"\s]{1,256}"`) + // Backtick pattern assembled so raw-string delimiters do not collide. + reIdentBT = regexp.MustCompile("`" + `[^` + "`" + `\s]{1,256}` + "`") + // Temp table names sling generates + reTempTable = regexp.MustCompile(`\btemp[A-Za-z0-9]{3,}\b`) + // Version banners from drivers + reVersionBanner = regexp.MustCompile(`\(version\s+[^)]+\)`) + // Long digit runs (≥3) — last, so short codes survive earlier patterns + reDigits = regexp.MustCompile(`\b\d{3,}\b`) + + reMultiSpace = regexp.MustCompile(`[ \t]+`) +) + +// Skeleton normalizes a full error chain into a stable multi-line skeleton. +func Skeleton(errText string) string { + rawLines := strings.Split(errText, "\n") + kept := make([]string, 0, len(rawLines)) + codeSet := map[string]struct{}{} + var codes []string + + for _, raw := range rawLines { + line := strings.TrimSpace(raw) + if line == "" { + continue + } + if reStackFrame.MatchString(line) { + continue + } + // Stream/section banners like "----- name -----" + if isSectionBanner(line) { + continue + } + if strings.HasPrefix(line, "~ ") { + line = strings.TrimSpace(strings.TrimPrefix(line, "~ ")) + } else if strings.HasPrefix(line, "~") { + line = strings.TrimSpace(strings.TrimPrefix(line, "~")) + } + if line == "" { + continue + } + + for _, c := range extractCodes(line) { + if _, ok := codeSet[c]; !ok { + codeSet[c] = struct{}{} + codes = append(codes, c) + } + } + + line = substitutePlaceholders(line) + line = strings.ToLower(line) + line = reMultiSpace.ReplaceAllString(line, " ") + line = strings.TrimSpace(line) + if line == "" { + continue + } + kept = append(kept, line) + } + + if len(kept) == 0 && len(codes) == 0 { + return "unknown_error" + } + + // Append stable vendor codes last so the leaf message stays first for labels. + for _, c := range codes { + kept = append(kept, "code:"+strings.ToLower(c)) + } + return strings.Join(kept, "\n") +} + +func isSectionBanner(line string) bool { + // e.g. --------------------------- lxp_app_oomkilled_count --------------------------- + if len(line) < 10 { + return false + } + trimmed := strings.Trim(line, "- \t") + if trimmed == "" { + return false + } + // mostly dashes on the outside + return strings.HasPrefix(strings.TrimSpace(line), "---") && + strings.HasSuffix(strings.TrimSpace(line), "---") && + !reStackFrame.MatchString(line) +} + +func extractCodes(line string) []string { + var out []string + for _, m := range reBracketCode.FindAllStringSubmatch(line, -1) { + out = append(out, m[1]) + } + for _, m := range reParenCode.FindAllStringSubmatch(line, -1) { + // Skip common non-code paren groups + c := m[1] + if c == "official" || c == "build" { + continue + } + out = append(out, c) + } + for _, m := range reSQLState.FindAllStringSubmatch(line, -1) { + out = append(out, m[1]) + } + for _, m := range reCHCode.FindAllStringSubmatch(line, -1) { + out = append(out, "CH_"+m[1]) + } + return out +} + +// replaceURL masks a URL match, unless it points at slingdata.io (or a subdomain). +func replaceURL(match string) string { + u := strings.Trim(match, `"`) + if i := strings.Index(u, "://"); i >= 0 { + host := u[i+3:] + if j := strings.IndexAny(host, "/?#"); j >= 0 { + host = host[:j] + } + if j := strings.LastIndex(host, ":"); j >= 0 { + host = host[:j] + } + host = strings.ToLower(host) + if host == "slingdata.io" || strings.HasSuffix(host, ".slingdata.io") { + return match + } + } + return "" +} + +func substitutePlaceholders(line string) string { + // Order: more specific first. + line = reQuotedURL.ReplaceAllStringFunc(line, replaceURL) + line = reQuotedPath.ReplaceAllString(line, "") + line = reURL.ReplaceAllStringFunc(line, replaceURL) + line = reISOTs.ReplaceAllString(line, "") + line = reUUID.ReplaceAllString(line, "") + line = reIPv4.ReplaceAllString(line, "") + line = reVersionBanner.ReplaceAllString(line, "") + line = reTempTable.ReplaceAllString(line, "") + line = reLongID.ReplaceAllString(line, "") + line = reIdentDQ.ReplaceAllString(line, "") + line = reIdentBT.ReplaceAllString(line, "") + // Bare paths: keep leading delimiter, replace path token. + line = replacePathKeepLead(reUnixPath, line) + line = replacePathKeepLead(reWinPath, line) + line = replacePathKeepLead(reHomePath, line) + line = reDigits.ReplaceAllString(line, "") + return line +} + +// replacePathKeepLead replaces path matches of the form (lead)(path) with lead+"". +func replacePathKeepLead(re *regexp.Regexp, line string) string { + return re.ReplaceAllStringFunc(line, func(m string) string { + sub := re.FindStringSubmatch(m) + if len(sub) < 2 { + return m + } + path := sub[1] + lead := strings.TrimSuffix(m, path) + return lead + "" + }) +} + +// --- assist error lookup ----------------------------------------------------- + +// ErrorLookupResult is the response shape for `sling assist error `. +// Worker-backed known-issue lookup lands later; v1 returns local status only. +type ErrorLookupResult struct { + Signature string `json:"signature"` // compact 8 or 16 hex + PatternID string `json:"pattern_id,omitempty"` // first 8 of composite + EdgeID string `json:"edge_id,omitempty"` // last 8 when full composite + PatternOnly bool `json:"pattern_only,omitempty"` // true when caller passed 8 hex + Status string `json:"status"` // known_config | known_bug | pending | unknown + Title string `json:"title,omitempty"` + Guidance string `json:"guidance,omitempty"` + DocsURL string `json:"docs_url,omitempty"` + FixedIn string `json:"fixed_in,omitempty"` + IssueURL string `json:"issue_url,omitempty"` +} + +// LookupError validates a signature id and returns guidance when known. +// Accepts composite (16 hex), pattern-only (8 hex), dashed form, or Display() line. +// Network lookup is not wired yet; well-formed ids return status "unknown". +func LookupError(signature string) (ErrorLookupResult, error) { + compact, patternOnly, err := ParseSignatureID(signature) + if err != nil { + return ErrorLookupResult{}, err + } + out := ErrorLookupResult{ + Signature: compact, + PatternOnly: patternOnly, + Status: "unknown", + Title: "No published guidance yet", + Guidance: "This signature is not in the known-issue registry yet. Run `sling assist` to debug the latest failed run locally, or check docs.slingdata.io.", + DocsURL: "https://docs.slingdata.io/", + } + if patternOnly { + out.PatternID = compact + } else { + out.PatternID = compact[:PartIDLen] + out.EdgeID = compact[PartIDLen:] + } + return out, nil +} + +// Sensitivity classifies what may leave the machine in a log/submit bundle. +type Sensitivity int + +const ( + // SensitivityPublic is safe to ship as-is (no credentials by design). + SensitivityPublic Sensitivity = iota + // SensitivityInternal may contain operational detail; redact values before ship. + SensitivityInternal + // SensitivitySecret must never leave the machine (credentials, tokens, backups of same). + SensitivitySecret +) + +func (s Sensitivity) String() string { + switch s { + case SensitivityPublic: + return "public" + case SensitivityInternal: + return "internal" + case SensitivitySecret: + return "secret" + default: + return "unknown" + } +} + +// SensitiveClass describes one path/category the assist package may touch. +type SensitiveClass struct { + // ID is a stable key for manifests (e.g. "env.yaml", "claude.json"). + ID string `json:"id"` + // Glob is matched against absolute or home-relative paths (slash-normalized). + // Supports * and ** suffix style via pathMatch. + Glob string `json:"glob"` + // Class is the sensitivity tier. + Class Sensitivity `json:"class"` + // Reason is a short human explanation (never contains secret values). + Reason string `json:"reason"` +} + +// SensitivityManifest returns the static inventory of sensitive surface area. +// Used by submit/bundle builders to decide include / redact / exclude. +func SensitivityManifest() []SensitiveClass { + return []SensitiveClass{ + { + ID: "env.yaml", Glob: "**/env.yaml", Class: SensitivitySecret, + Reason: "may contain connection credentials and env secrets", + }, + { + ID: "claude.json", Glob: "**/.claude.json", Class: SensitivitySecret, + Reason: "Claude Code OAuth session and user MCP config", + }, + { + ID: "claude-mcp-project", Glob: "**/.mcp.json", Class: SensitivityInternal, + Reason: "project MCP server definitions; may reference env vars", + }, + { + ID: "codex-config", Glob: "**/.codex/config.toml", Class: SensitivityInternal, + Reason: "may include MCP env blocks", + }, + { + ID: "gemini-settings", Glob: "**/.gemini/settings.json", Class: SensitivityInternal, + Reason: "MCP and model settings", + }, + { + ID: "cursor-mcp", Glob: "**/.cursor/mcp.json", Class: SensitivityInternal, + Reason: "MCP server definitions", + }, + { + ID: "vscode-mcp", Glob: "**/mcp.json", Class: SensitivityInternal, + Reason: "VS Code MCP servers (user or .vscode)", + }, + { + ID: "config-backup", Glob: "**/*.backup", Class: SensitivitySecret, + Reason: "backups of credential-bearing config files", + }, + { + ID: "run-error", Glob: "**/assist/errors/**/error.txt", Class: SensitivityInternal, + Reason: "error chains may embed query fragments or object names", + }, + { + ID: "run-stderr", Glob: "**/assist/errors/**/stderr.log", Class: SensitivityInternal, + Reason: "debug logs may include connection props if not redacted at write", + }, + { + ID: "run-meta", Glob: "**/assist/errors/**/meta.json", Class: SensitivityPublic, + Reason: "exec metadata; argv must be redacted at write time", + }, + { + ID: "run-config-snapshot", Glob: "**/assist/errors/**/config.snapshot.yaml", Class: SensitivityInternal, + Reason: "resolved config; secrets must be masked at write time", + }, + { + ID: "doctor.json", Glob: "**/doctor.json", Class: SensitivityPublic, + Reason: "install health only; no connection secrets", + }, + { + ID: "assist-history-prompt", Glob: "**/assist/history/*/prompt.md", Class: SensitivityInternal, + Reason: "user intention and log tails; may include business context", + }, + { + ID: "canonical-skills", Glob: "**/.agents/skills/**", Class: SensitivityPublic, + Reason: "embedded public skill docs", + }, + } +} + +// ClassifyPath returns the highest-sensitivity class matching path. +// Unknown paths default to SensitivityInternal (safe default: redact before ship). +// When multiple globs match, Secret > Internal > Public. +func ClassifyPath(path string) Sensitivity { + n := filepath.ToSlash(path) + matched := false + best := SensitivityPublic + for _, c := range SensitivityManifest() { + if !pathMatch(c.Glob, n) { + continue + } + matched = true + if c.Class == SensitivitySecret { + return SensitivitySecret + } + if c.Class == SensitivityInternal { + best = SensitivityInternal + } + } + if !matched { + return SensitivityInternal + } + return best +} + +// pathMatch supports a small glob dialect used by SensitivityManifest: +// - "**/" prefix = match anywhere +// - "*" = one path segment +// - "**" = one or more segments (including zero when trailing) +// - "*.ext" basename wildcards +func pathMatch(glob, path string) bool { + glob = filepath.ToSlash(glob) + path = filepath.ToSlash(path) + if glob == path { + return true + } + gParts := strings.Split(glob, "/") + pParts := strings.Split(path, "/") + return matchParts(gParts, pParts) +} + +func matchParts(gParts, pParts []string) bool { + // Recursive glob matcher for segment lists. + var rec func(gi, pi int) bool + rec = func(gi, pi int) bool { + for gi < len(gParts) { + g := gParts[gi] + if g == "**" { + // "**" matches zero or more segments. + if gi == len(gParts)-1 { + return true // trailing ** + } + // Try consuming 0..N path segments. + for k := pi; k <= len(pParts); k++ { + if rec(gi+1, k) { + return true + } + } + return false + } + if pi >= len(pParts) { + return false + } + if g == "*" || matchSeg(g, pParts[pi]) { + gi++ + pi++ + continue + } + return false + } + return pi == len(pParts) + } + return rec(0, 0) +} + +func matchSeg(pat, seg string) bool { + if pat == "*" || pat == seg { + return true + } + // basename wildcard: *.backup + if strings.HasPrefix(pat, "*.") { + return strings.HasSuffix(seg, pat[1:]) // ".backup" + } + if strings.Contains(pat, "*") { + // simple prefix*suffix + i := strings.Index(pat, "*") + return strings.HasPrefix(seg, pat[:i]) && strings.HasSuffix(seg, pat[i+1:]) + } + return false +} + +// FailureSnapshot is the minimal set of fields written when a command fails +// (run, conns test, conns discover) so `sling assist` can probe the error. +type FailureSnapshot struct { + ExecID string + ErrMsg string + ConfigPath string // replication / pipeline config path when known + ConnName string // connection name for `sling conns test|discover` + Rows string + Duration string + // RunLog is the captured log tail (env.RecentLogs). Written to stderr.log. + RunLog string + // ConfigBody is the replication/pipeline config file content. Written to + // config.snapshot.yaml when set. Not used for conns test/discover. + ConfigBody string + // SignMeta optional connector types for error_signature. + SignMeta SignMeta + // Extra is merged into meta.json as-is (connector types, etc.). + Extra map[string]any +} + +const ( + reservedExecutionsDir = "executions" + reservedSignaturesDir = "signatures" +) + +func isReservedErrorName(name string) bool { + return name == reservedExecutionsDir || name == reservedSignaturesDir +} + +// WriteFailureSnapshot writes ~/.sling/assist/errors/executions//{meta.json, +// error.txt, stderr.log}. Best-effort: no-op when execID is empty; never fails +// the caller run (errors are logged via g.Debug only). +func WriteFailureSnapshot(s FailureSnapshot) { + if strings.TrimSpace(s.ExecID) == "" { + return + } + + // load config body + if ext := strings.ToLower(filepath.Ext(s.ConfigPath)); g.In(ext, ".yaml", ".yml", ".json") && s.ConfigBody == "" { + if b, err := os.ReadFile(s.ConfigPath); err == nil && int64(len(b)) <= 64*1024 { + s.ConfigBody = string(b) + } + } + + dir := filepath.Join(ExecutionsDir(), s.ExecID) + if err := os.MkdirAll(dir, 0o755); err != nil { + g.Debug("assist: could not create error dir %s: %s", dir, err.Error()) + return + } + + errMsg := s.ErrMsg + if errMsg == "" { + errMsg = "(no error message captured)" + } + if err := os.WriteFile(filepath.Join(dir, "error.txt"), []byte(errMsg), 0o644); err != nil { + g.Debug("assist: could not write error.txt: %s", err.Error()) + } + // stderr.log holds the captured run log. Fall back to the error text when + // nothing was buffered, so the file is never empty. + runLog := s.RunLog + if strings.TrimSpace(runLog) == "" { + runLog = errMsg + } + if err := os.WriteFile(filepath.Join(dir, "stderr.log"), []byte(runLog), 0o644); err != nil { + g.Debug("assist: could not write stderr.log: %s", err.Error()) + } + if body := strings.TrimSpace(s.ConfigBody); body != "" { + if err := os.WriteFile(filepath.Join(dir, "config.snapshot.yaml"), []byte(body), 0o644); err != nil { + g.Debug("assist: could not write config.snapshot.yaml: %s", err.Error()) + } + } + + sig := SignError(errMsg, s.SignMeta) + + meta := map[string]any{ + "exec_id": s.ExecID, + "exit_code": 1, + "when": time.Now().UTC().Format(time.RFC3339), + "error_signature": sig.ID, // 16 hex: pattern||edge + "error_pattern_id": sig.PatternID, + "error_edge_id": sig.EdgeID, + "error_algorithm": sig.Algorithm, + "error_short_label": sig.ShortLabel, + } + if s.SignMeta.SourceType != "" { + meta["source_type"] = s.SignMeta.SourceType.String() + } + if s.SignMeta.TargetType != "" { + meta["target_type"] = s.SignMeta.TargetType.String() + } + if tt := s.SignMeta.InferredTaskType(); tt != "" { + meta["task_type"] = tt // derived; not part of signature hash + } + if s.ConfigPath != "" { + meta["config_path"] = s.ConfigPath + } + if s.ConnName != "" { + meta["conn_name"] = s.ConnName + } + if s.Rows != "" { + meta["rows"] = s.Rows + } + if s.Duration != "" { + meta["duration"] = s.Duration + } + for k, v := range s.Extra { + if _, exists := meta[k]; !exists { + meta[k] = v + } + } + body, err := json.MarshalIndent(meta, "", " ") + if err != nil { + g.Debug("assist: could not marshal meta.json: %s", err.Error()) + return + } + if err := os.WriteFile(filepath.Join(dir, "meta.json"), body, 0o644); err != nil { + g.Debug("assist: could not write meta.json: %s", err.Error()) + } + + if err := AutoTrimExecs(); err != nil { + g.Debug("assist: auto-trim execs: %s", err.Error()) + } +} + +// FailureFooterOpts controls the post-failure hint line. +// A command only — never an interactive prompt (TTY or not). +type FailureFooterOpts struct { + ExecID string + ErrMsg string + SignMeta SignMeta +} + +// PrintFailureFooter prints one indented hint line after a failure. +// The error signature is not printed: it is agent context, written to +// meta.json by WriteFailureSnapshot and read back by Probe. +// Suppressed when HintInErrors is false or SLING_ASSIST_HINT is falsey. +func PrintFailureFooter(opts FailureFooterOpts) { + if envDisabled("SLING_ASSIST_HINT") { + return + } + + prof, exists, err := LoadProfile() + if err != nil { + return + } + hintOn := true + if exists { + hintOn = prof.HintInErrors + } + if !hintOn { + return + } + + line := " sling assist setup" + if execID := strings.TrimSpace(opts.ExecID); execID != "" { + line = " sling assist --id " + ShortExecID(execID) + } else if exists || len(DetectedClients()) > 0 { + return + } + + if isTTY(os.Stderr) && !env.NoColor { + label := terminalLink(AssistDocsURL, "investigate with AI") + line = " " + label + " -> " + env.CyanString(strings.TrimSpace(line)) + } + fmt.Fprintln(os.Stderr, "") + fmt.Fprintln(os.Stderr, line) + fmt.Fprintln(os.Stderr, "") +} + +// AssistDocsURL is linked from the failure hint. +const AssistDocsURL = "https://docs.slingdata.io/sling-cli/assist" + +// terminalLink wraps text in an OSC 8 hyperlink. Terminals without OSC 8 +// support drop the escape codes and show only the text. +func terminalLink(url, text string) string { + return "\x1b]8;;" + url + "\x1b\\" + text + "\x1b]8;;\x1b\\" +} + +// ShortExecIDLen is the exec-id prefix length shown in the failure hint. +// ResolveLocalExec accepts any unique prefix. +const ShortExecIDLen = 8 + +// ShortExecID trims an exec id to the prefix shown to users. +func ShortExecID(id string) string { + id = strings.TrimSpace(id) + if len(id) > ShortExecIDLen { + return id[:ShortExecIDLen] + } + return id +} + +// MaybePrintErrorHint is kept for callers/tests that only have an exec id. +// Prefer PrintFailureFooter when error text is available. +func MaybePrintErrorHint(execID string) { + PrintFailureFooter(FailureFooterOpts{ExecID: execID}) +} + +// envDisabled is true when the named env var is set to a falsey value. +func envDisabled(key string) bool { + v := os.Getenv(key) + if v == "" { + return false + } + return !cast.ToBool(v) +} + +// LocalExec is one failed-run snapshot under ~/.sling/assist/errors/. +// New snapshots live in errors/executions//; legacy dirs stay readable. +type LocalExec struct { + ID string + When time.Time + Status string // "ok" | "err" | "?" + ConfigPath string // replication / pipeline config path + ConnName string // connection name for conns test/discover + Rows string + Duration string + LogDir string // absolute path to the exec's snapshot dir +} + +func (e LocalExec) displayObject() string { + if e.ConfigPath != "" { + return e.ConfigPath + } + return e.ConnName +} + +// LogsRoot returns ~/.sling/logs (SLING_LOG_DIR day files). Not used for failure snapshots. +func LogsRoot() string { + return filepath.Join(slingHome(), "logs") +} + +// ListLocalExecs scans errors/executions// then legacy errors// +// and returns the 20 most-recent execs by mtime. Unreadable dirs are skipped. +func ListLocalExecs() ([]LocalExec, error) { + ids, err := listLocalExecIDs() + if err != nil { + return nil, err + } + out := []LocalExec{} + for _, id := range ids { + dir := findLocalExecDir(id) + if dir == "" { + continue + } + le := LocalExec{ID: id, LogDir: dir, Status: "?"} + if info, err := os.Stat(dir); err == nil { + le.When = info.ModTime() + } + loadLocalExecMeta(&le) + out = append(out, le) + } + sort.Slice(out, func(i, j int) bool { + return out[i].When.After(out[j].When) + }) + if len(out) > 20 { + out = out[:20] + } + return out, nil +} + +// AutoTrimExecs deletes the oldest failure snapshots until at most +// ExecsMaxEntries remain. Covers both errors/executions// and the legacy +// errors// layout. Best-effort: a snapshot that cannot be removed is +// reported, and the rest still get trimmed. +func AutoTrimExecs() error { + ids, err := listLocalExecIDs() + if err != nil { + return err + } + if len(ids) <= ExecsMaxEntries { + return nil + } + + type entry struct { + dir string + when time.Time + } + entries := make([]entry, 0, len(ids)) + for _, id := range ids { + dir := findLocalExecDir(id) + if dir == "" { + continue + } + e := entry{dir: dir} + if info, statErr := os.Stat(dir); statErr == nil { + e.when = info.ModTime() + } + entries = append(entries, e) + } + if len(entries) <= ExecsMaxEntries { + return nil + } + + sort.Slice(entries, func(i, j int) bool { + return entries[i].when.After(entries[j].when) + }) + + var first error + for _, e := range entries[ExecsMaxEntries:] { + if rmErr := os.RemoveAll(e.dir); rmErr != nil && first == nil { + first = g.Error(rmErr, "remove exec snapshot %s", e.dir) + } + } + return first +} + +func listLocalExecIDs() ([]string, error) { + seen := map[string]struct{}{} + var ids []string + addFrom := func(root string, skipReserved bool) error { + if root == "" || !g.PathExists(root) { + return nil + } + entries, err := os.ReadDir(root) + if err != nil { + return g.Error(err, "read %s", root) + } + for _, e := range entries { + if !e.IsDir() { + continue + } + name := e.Name() + if skipReserved && isReservedErrorName(name) { + continue + } + if _, ok := seen[name]; ok { + continue + } + seen[name] = struct{}{} + ids = append(ids, name) + } + return nil + } + if err := addFrom(ExecutionsDir(), false); err != nil { + return nil, err + } + if err := addFrom(ErrorsDir(), true); err != nil { + return nil, err + } + return ids, nil +} + +func firstMetaString(doc map[string]any, keys ...string) string { + for _, k := range keys { + if v, _ := doc[k].(string); strings.TrimSpace(v) != "" { + return v + } + } + return "" +} + +func loadLocalExecMeta(e *LocalExec) { + metaPath := filepath.Join(e.LogDir, "meta.json") + doc, err := jsonReadOrEmpty(metaPath) + if err != nil || len(doc) == 0 { + return + } + if v, ok := doc["exit_code"]; ok { + if fmt.Sprintf("%v", v) == "0" { + e.Status = "ok" + } else { + e.Status = "err" + } + } + if v := firstMetaString(doc, "config_path", "object"); v != "" { + e.ConfigPath = v + } + if v, _ := doc["conn_name"].(string); v != "" { + e.ConnName = v + } + if v, _ := doc["rows"].(string); v != "" { + e.Rows = v + } + if v, _ := doc["duration"].(string); v != "" { + e.Duration = v + } +} + +// LookupLocalExec resolves an exec id to its snapshot. Accepts the full id or +// a unique prefix. Returns false when unknown or when a prefix is ambiguous. +func LookupLocalExec(id string) (LocalExec, bool) { + le, err := ResolveLocalExec(id) + return le, err == nil +} + +// ResolveLocalExec is LookupLocalExec with a reason: unknown vs ambiguous. +func ResolveLocalExec(id string) (LocalExec, error) { + id = strings.TrimSpace(id) + if id == "" { + return LocalExec{}, g.Error("empty exec id") + } + if dir := findLocalExecDir(id); dir != "" { + le := LocalExec{ID: id, LogDir: dir, Status: "?"} + if info, err := os.Stat(dir); err == nil { + le.When = info.ModTime() + } + loadLocalExecMeta(&le) + return le, nil + } + // Prefix match against every snapshot, not just the 20 ListLocalExecs keeps. + ids, err := listLocalExecIDs() + if err != nil { + return LocalExec{}, g.Error("unknown exec id %q", id) + } + hits := []string{} + for _, name := range ids { + if strings.HasPrefix(name, id) { + hits = append(hits, name) + } + } + switch len(hits) { + case 0: + return LocalExec{}, g.Error("unknown exec id %q", id) + case 1: + return ResolveLocalExec(hits[0]) + default: + sort.Strings(hits) + return LocalExec{}, g.Error("exec id %q is ambiguous (%d matches: %s)", + id, len(hits), strings.Join(hits[:2], ", ")+", …") + } +} + +// findLocalExecDir returns the snapshot dir for id. New layout first, then legacy. +func findLocalExecDir(id string) string { + if id == "" || isReservedErrorName(id) { + return "" + } + if strings.ContainsAny(id, `/\`) || strings.Contains(id, "..") { + return "" + } + if candidate := filepath.Join(ExecutionsDir(), id); g.PathExists(candidate) { + return candidate + } + if candidate := filepath.Join(ErrorsDir(), id); g.PathExists(candidate) { + return candidate + } + return "" +} + +const maxErrorTailBytes = 16 * 1024 + +// sanitizeLogForPrompt caps length, scrubs local connection secrets, and +// neutralizes triple-backtick fences so hostile log content cannot escape +// the markdown code blocks in prompts.yaml. +func sanitizeLogForPrompt(s string, maxBytes int) string { + s = scrubLocalConnSecrets(s) + if maxBytes > 0 && len(s) > maxBytes { + // Keep the tail (errors are usually at the end). + s = s[len(s)-maxBytes:] + if i := strings.IndexByte(s, '\n'); i >= 0 && i < 200 { + s = s[i+1:] + } + s = "[...truncated...]\n" + s + } + // Break ``` fences so log content cannot close the surrounding fence. + s = strings.ReplaceAll(s, "```", "'''") + return s +} + +func scrubLocalConnSecrets(s string) string { + return env.ScrubLine(s) +} diff --git a/core/sling/assist/investigate_test.go b/core/sling/assist/investigate_test.go new file mode 100644 index 000000000..005006863 --- /dev/null +++ b/core/sling/assist/investigate_test.go @@ -0,0 +1,1276 @@ +package assist + +import ( + "bytes" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/flarco/g" + "github.com/slingdata-io/sling-cli/core/dbio" + "github.com/slingdata-io/sling-cli/core/dbio/connection" + "github.com/slingdata-io/sling-cli/core/env" + "github.com/spf13/cast" +) + +func TestSignError_NoStreamColumns(t *testing.T) { + errText := `--- task_run.go:140 func2 --- +--- task_run.go:830 runDbToDb --- +~ Could not WriteToDb +--- task_run_write.go:168 WriteToDb --- +no stream columns detected` + meta := SignMeta{SourceType: dbio.TypeDbPrometheus, TargetType: dbio.TypeDbPostgres} + sig := SignError(errText, meta) + + if sig.Algorithm != "v1" { + t.Fatalf("algorithm: got %q", sig.Algorithm) + } + if len(sig.ID) != CompositeIDLen { + t.Fatalf("id len: got %q (want %d hex)", sig.ID, CompositeIDLen) + } + if len(sig.PatternID) != PartIDLen || len(sig.EdgeID) != PartIDLen { + t.Fatalf("parts: pattern=%q edge=%q", sig.PatternID, sig.EdgeID) + } + if sig.ID != sig.PatternID+sig.EdgeID { + t.Fatalf("composite != pattern||edge: %s vs %s%s", sig.ID, sig.PatternID, sig.EdgeID) + } + // Stack frames must not appear in skeleton. + if strings.Contains(sig.Skeleton, "task_run") || strings.Contains(sig.Skeleton, ".go:") { + t.Fatalf("skeleton still has frames:\n%s", sig.Skeleton) + } + if !strings.Contains(sig.Skeleton, "could not writetodb") { + t.Fatalf("skeleton missing context msg:\n%s", sig.Skeleton) + } + if !strings.Contains(sig.Skeleton, "no stream columns detected") { + t.Fatalf("skeleton missing leaf:\n%s", sig.Skeleton) + } + // Deterministic + sig2 := SignError(errText, meta) + if sig.ID != sig2.ID || sig.PatternID != sig2.PatternID || sig.EdgeID != sig2.EdgeID { + t.Fatalf("not deterministic: %v vs %v", sig, sig2) + } + // Line numbers must not change signature. + errAlt := strings.ReplaceAll(errText, "140", "999") + errAlt = strings.ReplaceAll(errAlt, "830", "1") + errAlt = strings.ReplaceAll(errAlt, "168", "42") + if SignError(errAlt, meta).ID != sig.ID { + t.Fatalf("line numbers changed signature") + } + // Different target type → different composite / edge, same pattern + other := SignError(errText, SignMeta{SourceType: dbio.TypeDbPrometheus, TargetType: dbio.TypeDbSnowflake}) + if other.ID == sig.ID { + t.Fatalf("different meta should change composite id") + } + if other.PatternID != sig.PatternID { + t.Fatalf("same skeleton should share pattern_id: %s vs %s", sig.PatternID, other.PatternID) + } + if other.EdgeID == sig.EdgeID { + t.Fatalf("different target should change edge_id") + } +} + +func TestSignError_ConnectionRefusedScrubsIP(t *testing.T) { + errText := `--- proc.go:283 main --- +--- sling_cli.go:517 main --- +~ could not connect to database(try adding ` + "`sslmode=require`" + ` or ` + "`sslmode=disable`" + `) +dial tcp 192.168.176.200:5432: connect: connection refused` + meta := SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbPostgres} + sig := SignError(errText, meta) + if strings.Contains(sig.Skeleton, "192.168") { + t.Fatalf("IP not scrubbed:\n%s", sig.Skeleton) + } + if !strings.Contains(sig.Skeleton, "") { + t.Fatalf("expected placeholder:\n%s", sig.Skeleton) + } + err2 := strings.ReplaceAll(errText, "192.168.176.200", "10.0.0.5") + if SignError(err2, meta).ID != sig.ID { + t.Fatalf("different IPs should cluster") + } +} + +func TestSignError_AuthFailedExtractsCode(t *testing.T) { + errText := `--- task_run.go:142 func2 --- +~ Could not initialize target connection +--- database_clickhouse.go:74 Connect --- +~ could not connect to database +clickhouse [execute]:: 403 code: Code: 516. DB::Exception: bcdata: Authentication failed: password is incorrect, or there is no user with such name. (AUTHENTICATION_FAILED) (version 25.11.2.24 (official build)) +` + meta := SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbClickhouse} + sig := SignError(errText, meta) + if strings.Contains(sig.Skeleton, "25.11") { + t.Fatalf("version banner not scrubbed:\n%s", sig.Skeleton) + } + if !strings.Contains(sig.Skeleton, "code:authentication_failed") { + t.Fatalf("expected AUTHENTICATION_FAILED code:\n%s", sig.Skeleton) + } + if !strings.Contains(sig.Skeleton, "code:ch_516") { + t.Fatalf("expected CH code:\n%s", sig.Skeleton) + } + t.Logf("skeleton:\n%s\nid=%s dashed=%s label=%s", sig.Skeleton, sig.ID, sig.IDDashed(), sig.ShortLabel) +} + +func TestSignError_URLAndPathScrub(t *testing.T) { + errText := `--- database.go:709 Connect --- +Post "https://clickhouse.bcstuff.dev:443?database=bc_clickhouse_db&default_format=Native": dial tcp 136.41.64.93:443: i/o timeout +unable to open database "/root/.duckdb/extensions/v1.4.2/linux_amd64/motherduck.duckdb_extension" +` + meta := SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbClickhouse} + sig := SignError(errText, meta) + if strings.Contains(sig.Skeleton, "bcstuff") || strings.Contains(sig.Skeleton, "clickhouse.bc") { + t.Fatalf("URL host not scrubbed:\n%s", sig.Skeleton) + } + if !strings.Contains(sig.Skeleton, "") { + t.Fatalf("expected :\n%s", sig.Skeleton) + } + if strings.Contains(sig.Skeleton, "/root/") { + t.Fatalf("path not scrubbed:\n%s", sig.Skeleton) + } + if !strings.Contains(sig.Skeleton, "") { + t.Fatalf("expected :\n%s", sig.Skeleton) + } +} + +func TestSignError_DatabricksArityCode(t *testing.T) { + errText := `--- task_run.go:142 func2 --- +~ Could not WriteToDb +--- database_databricks.go:184 BulkImportFlow --- +~ could not insert into ` + "`safenet`.`approval_plan_ship_tmp`" + ` +databricks: execution error: failed to execute query: unexpected operation state ERROR_STATE: [COPY_INTO_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS] Cannot write to ` + "`abl_analytics_ws_prd`.`safenet`.`approval_plan_ship_tmp`" + `, the reason is not enough data columns compared to specified columns: +Specified columns: ` + "`approval_plan_id`" + `, ` + "`ship_id`" + `, ` + "`_sling_loaded_at`" + `. +Data columns: .` + meta := SignMeta{SourceType: dbio.TypeDbMySQL, TargetType: dbio.TypeDbDatabricks} + sig := SignError(errText, meta) + if !strings.Contains(sig.Skeleton, "code:copy_into_column_arity_mismatch.not_enough_data_columns") { + t.Fatalf("expected arity code:\n%s", sig.Skeleton) + } + if strings.Contains(sig.Skeleton, "safenet") || strings.Contains(sig.Skeleton, "approval_plan") { + t.Fatalf("idents not scrubbed:\n%s", sig.Skeleton) + } +} + +func TestSignError_IdentAndTempTable(t *testing.T) { + errText := `~ could not prepare Tx: COPY "public"."lxp_cpu_throttled_percentage_tmp" ("app") FROM STDIN +pq: relation "public.lxp_cpu_throttled_percentage_tmp" does not exist +~ Error executing: create unique index if not exists tempSipc0_idx on tempSipc0 ("timestamp") +` + meta := SignMeta{SourceType: dbio.TypeDbPrometheus, TargetType: dbio.TypeDbPostgres} + sig := SignError(errText, meta) + if strings.Contains(sig.Skeleton, "tempSipc0") { + t.Fatalf("temp table not scrubbed:\n%s", sig.Skeleton) + } + if strings.Contains(sig.Skeleton, "lxp_cpu") { + t.Fatalf("quoted ident not scrubbed:\n%s", sig.Skeleton) + } + if !strings.Contains(sig.Skeleton, "") || !strings.Contains(sig.Skeleton, "") { + t.Fatalf("expected placeholders:\n%s", sig.Skeleton) + } +} + +func TestSignError_Empty(t *testing.T) { + sig := SignError("", SignMeta{}) + if sig.Skeleton != "unknown_error" { + t.Fatalf("got skeleton %q", sig.Skeleton) + } + if sig.PatternMaterial != "v1p|unknown_error" { + t.Fatalf("pattern material: %q", sig.PatternMaterial) + } + if sig.EdgeMaterial != "v1e|-|-|unknown_error" { + t.Fatalf("edge material: %q", sig.EdgeMaterial) + } + if len(sig.ID) != CompositeIDLen { + t.Fatalf("id: %q", sig.ID) + } + // Known vector: empty meta + unknown_error + if sig.ID != "fb2398c014bc4249" { + t.Fatalf("empty id vector: got %s want fb2398c014bc4249", sig.ID) + } +} + +func TestSignError_MetaCaseInsensitive(t *testing.T) { + errText := "no stream columns detected" + a := SignError(errText, SignMeta{SourceType: dbio.Type("Postgres"), TargetType: dbio.Type("SNOWFLAKE")}) + b := SignError(errText, SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbSnowflake}) + if a.ID != b.ID { + t.Fatalf("meta case should not matter: %s vs %s", a.ID, b.ID) + } +} + +func TestComposite_KnownVector(t *testing.T) { + skel := "could not writetodb\nno stream columns detected" + meta := SignMeta{SourceType: dbio.TypeDbPrometheus, TargetType: dbio.TypeDbPostgres} + // Materials + pMat := PatternMaterial(skel) + eMat := EdgeMaterial(skel, meta) + if pMat != "v1p|"+skel { + t.Fatalf("pattern material: %q", pMat) + } + if eMat != "v1e|prometheus|postgres|"+skel { + t.Fatalf("edge material: %q", eMat) + } + // Known digests (python/sha256 vectors) + if HashPart(pMat) != "97d84811" { + t.Fatalf("pattern part: got %s want 97d84811", HashPart(pMat)) + } + if HashPart(eMat) != "5aede62c" { + t.Fatalf("edge part: got %s want 5aede62c", HashPart(eMat)) + } + // Full SignError on framed error yields same skeleton materials + errText := `--- task_run.go:140 func2 --- +~ Could not WriteToDb +--- task_run_write.go:168 WriteToDb --- +no stream columns detected` + sig := SignError(errText, meta) + if sig.PatternID != "97d84811" || sig.EdgeID != "5aede62c" { + t.Fatalf("parts: pattern=%s edge=%s", sig.PatternID, sig.EdgeID) + } + if sig.ID != "97d848115aede62c" { + t.Fatalf("composite: got %s", sig.ID) + } + if sig.IDDashed() != "97d84811-5aede62c" { + t.Fatalf("dashed: %s", sig.IDDashed()) + } +} + +func TestShortLabel(t *testing.T) { + lab := ShortLabel("could not writetodb\nno stream columns detected") + if lab != "no_stream_columns_detected" { + t.Fatalf("label: %q", lab) + } + lab2 := ShortLabel("unknown_error") + if lab2 != "unknown_error" { + t.Fatalf("label: %q", lab2) + } +} + +func TestDisplay(t *testing.T) { + sig := SignError("no stream columns detected", SignMeta{ + SourceType: dbio.TypeDbPrometheus, TargetType: dbio.TypeDbPostgres, + }) + d := sig.Display() + if !strings.Contains(d, sig.IDDashed()) || !strings.Contains(d, "prometheus→postgres") { + t.Fatalf("display: %q", d) + } + if !strings.Contains(d, "-") { + t.Fatalf("display should use dashed composite: %q", d) + } +} + +func TestParseSignatureID(t *testing.T) { + compact, patternOnly, err := ParseSignatureID("97d848115aede62c") + if err != nil || compact != "97d848115aede62c" || patternOnly { + t.Fatalf("16hex: %q %v %v", compact, patternOnly, err) + } + compact, patternOnly, err = ParseSignatureID("97d84811-5aede62c") + if err != nil || compact != "97d848115aede62c" || patternOnly { + t.Fatalf("dashed: %q %v %v", compact, patternOnly, err) + } + compact, patternOnly, err = ParseSignatureID("97d84811 (prometheus→postgres · no_stream_columns)") + if err != nil || compact != "97d84811" || !patternOnly { + t.Fatalf("pattern from display: %q %v %v", compact, patternOnly, err) + } + // Display line with dashed id + compact, patternOnly, err = ParseSignatureID("97d84811-5aede62c (prometheus→postgres · x)") + if err != nil || compact != "97d848115aede62c" || patternOnly { + t.Fatalf("dashed display: %q %v %v", compact, patternOnly, err) + } + if _, _, err := ParseSignatureID("not-a-sig"); err == nil { + t.Fatal("expected error") + } + if _, _, err := ParseSignatureID("abcd"); err == nil { + t.Fatal("expected error for short hex") + } +} + +func TestSkeleton_DropsSectionBanners(t *testing.T) { + errText := `~ failure running replication +--------------------------- lxp_app_oomkilled_count --------------------------- +~ Could not WriteToDb +no stream columns detected +--------------------------- lxp_app_middleware_pod_oomkilled_count --------------------------- +~ Could not WriteToDb +no stream columns detected` + skel := Skeleton(errText) + if strings.Contains(skel, "lxp_app_oomkilled") { + t.Fatalf("section banner leaked:\n%s", skel) + } + if strings.Count(skel, "no stream columns detected") < 1 { + t.Fatalf("missing leaf:\n%s", skel) + } +} + +func TestVariousErrorShapes(t *testing.T) { + cases := []struct { + name string + err string + meta SignMeta + want []string + deny []string + }{ + { + name: "ssl_not_enabled", + err: "--- sling_run.go:442 runTask ---\n~ could not connect to database\npq: SSL is not enabled on the server", + meta: SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbPostgres}, + want: []string{"ssl is not enabled"}, + deny: []string{"sling_run.go"}, + }, + { + name: "update_key_missing", + err: "--- task_run_read.go:149 ReadFromDB ---\ndid not find update_key: modified_at", + meta: SignMeta{SourceType: dbio.TypeDbMySQL, TargetType: dbio.TypeDbSnowflake}, + want: []string{"did not find update_key"}, + deny: []string{"task_run_read"}, + }, + { + name: "table_not_found_sqlserver", + err: "--- database_sqlserver.go:543 GetTableColumns ---\ndid not find table or synonym: \"V12PROD\".\"XL\"", + meta: SignMeta{SourceType: dbio.TypeDbSQLServer, TargetType: dbio.TypeDbClickhouse}, + want: []string{"did not find table or synonym", ""}, + deny: []string{"V12PROD", "database_sqlserver"}, + }, + { + name: "not_enough_space", + err: "clickhouse [execute]:: 500 code: Code: 243. DB::Exception: Cannot reserve 1.00 MiB, not enough space: While executing WaitForAsyncInsert. (NOT_ENOUGH_SPACE) (version 26.6.2.81 (official build))", + meta: SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbClickhouse}, + want: []string{"code:not_enough_space", "code:ch_243", "not enough space"}, + deny: []string{"26.6.2"}, + }, + { + name: "motherduck_auth", + err: `~ Failed to execute SQL +Error: unable to open database "md:warehouse": Invalid Input Error: Initialization function "motherduck_duckdb_cpp_init" from file "/root/.duckdb/extensions/v1.4.2/linux_amd64/motherduck.duckdb_extension" threw an exception: "Invalid Error: Request failed: Your request is not authenticated. Please check your MotherDuck token."`, + meta: SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbMotherDuck}, + want: []string{"not authenticated", ""}, + deny: []string{"/root/.duckdb"}, + }, + { + name: "eof_connect", + err: "~ could not connect to database(try adding `sslmode=require` or `sslmode=disable`)\nEOF", + meta: SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbPostgres}, + want: []string{"could not connect", "eof"}, + deny: []string{}, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + sig := SignError(tc.err, tc.meta) + if len(sig.ID) != CompositeIDLen { + t.Fatalf("bad id %q", sig.ID) + } + for _, w := range tc.want { + if !strings.Contains(sig.Skeleton, w) { + t.Fatalf("skeleton missing %q:\n%s", w, sig.Skeleton) + } + } + for _, d := range tc.deny { + if strings.Contains(sig.Skeleton, d) { + t.Fatalf("skeleton has denied %q:\n%s", d, sig.Skeleton) + } + } + if !strings.HasPrefix(sig.PatternMaterial, "v1p|") { + t.Fatalf("pattern material: %q", sig.PatternMaterial) + } + if !strings.HasPrefix(sig.EdgeMaterial, "v1e|") { + t.Fatalf("edge material: %q", sig.EdgeMaterial) + } + }) + } +} + +func TestInferredTaskType(t *testing.T) { + m := SignMeta{SourceType: dbio.TypeDbPrometheus, TargetType: dbio.TypeDbPostgres} + if m.InferredTaskType() != "db-db" { + t.Fatalf("got %q", m.InferredTaskType()) + } + m = SignMeta{SourceType: dbio.TypeFileS3, TargetType: dbio.TypeDbSnowflake} + if m.InferredTaskType() != "file-db" { + t.Fatalf("got %q", m.InferredTaskType()) + } + m = SignMeta{SourceType: dbio.TypeApi, TargetType: dbio.TypeFileLocal} + if m.InferredTaskType() != "api-file" { + t.Fatalf("got %q", m.InferredTaskType()) + } + if (SignMeta{}).InferredTaskType() != "" { + t.Fatal("empty meta should not infer task type") + } +} + +// --- ClickHouse live parity ---------------------------------------------------- + +func chQuery(t *testing.T, sql string) []map[string]any { + t.Helper() + entry := connection.GetLocalConns().Get("clickhouse_top") + if entry.Name == "" { + t.Skip("clickhouse_top connection not configured") + } + db, err := entry.Connection.AsDatabase() + if err != nil { + t.Skipf("AsDatabase: %v", err) + } + if err := db.Connect(); err != nil { + t.Skipf("Connect: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + data, err := db.Query(sql) + if err != nil { + t.Fatalf("query failed: %v\nsql=%s", err, truncate(sql, 400)) + } + return data.Records(true) +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "…" +} + +// TestClickHouseHashMatchesGo proves SHA-256 first-8 hex is identical for the +// same material bytes in Go and ClickHouse (pattern and edge materials). +func TestClickHouseHashMatchesGo(t *testing.T) { + skel := Skeleton(`--- task_run.go:140 func2 --- +~ Could not WriteToDb +no stream columns detected`) + meta := SignMeta{SourceType: dbio.TypeDbPrometheus, TargetType: dbio.TypeDbPostgres} + materials := []string{ + PatternMaterial(skel), + EdgeMaterial(skel, meta), + "v1p|unknown_error", + "v1e|-|-|unknown_error", + EdgeMaterial(Skeleton("dial tcp 10.0.0.1:5432: connection refused"), SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbMySQL}), + PatternMaterial(Skeleton(`[COPY_INTO_COLUMN_ARITY_MISMATCH.NOT_ENOUGH_DATA_COLUMNS] bad columns`)), + } + + for i, mat := range materials { + t.Run(fmt.Sprintf("mat_%d", i), func(t *testing.T) { + goSig := HashPart(mat) + sql := "SELECT " + chSQLHashPart(mat) + " AS sig" + rows := chQuery(t, sql) + if len(rows) != 1 { + t.Fatalf("expected 1 row, got %d", len(rows)) + } + chSig := cast.ToString(rows[0]["sig"]) + if chSig != goSig { + t.Fatalf("hash mismatch\n go=%s\n ch=%s\n material=%q", goSig, chSig, mat) + } + }) + } +} + +// TestClickHouseSignErrorMatchesGo compares full composite SignError against CH SQL. +func TestClickHouseSignErrorMatchesGo(t *testing.T) { + cases := []struct { + name string + err string + meta SignMeta + }{ + { + name: "no_stream_columns", + err: `--- task_run.go:140 func2 --- +--- task_run.go:830 runDbToDb --- +~ Could not WriteToDb +--- task_run_write.go:168 WriteToDb --- +no stream columns detected`, + meta: SignMeta{SourceType: dbio.TypeDbPrometheus, TargetType: dbio.TypeDbPostgres}, + }, + { + name: "connection_refused", + err: `--- proc.go:283 main --- +~ could not connect to database(try adding sslmode=require or sslmode=disable) +dial tcp 192.168.176.200:5432: connect: connection refused`, + meta: SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbPostgres}, + }, + { + name: "ssl_not_enabled", + err: `~ could not connect to database +pq: SSL is not enabled on the server`, + meta: SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbClickhouse}, + }, + { + name: "url_timeout", + err: `~ could not connect to database +Post "https://clickhouse.example.com:443?database=db&default_format=Native": dial tcp 136.41.64.93:443: i/o timeout`, + meta: SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbClickhouse}, + }, + { + name: "empty_meta", + err: "no stream columns detected", + meta: SignMeta{}, + }, + { + name: "path_scrub", + err: `unable to open database "/root/.duckdb/extensions/v1.4.2/linux_amd64/motherduck.duckdb_extension"`, + meta: SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbMotherDuck}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + goSig := SignError(tc.err, tc.meta) + + // Pattern + edge material hash parity + for _, part := range []struct { + name, mat, want string + }{ + {"pattern", goSig.PatternMaterial, goSig.PatternID}, + {"edge", goSig.EdgeMaterial, goSig.EdgeID}, + } { + rows := chQuery(t, "SELECT "+chSQLHashPart(part.mat)+" AS sig") + if cast.ToString(rows[0]["sig"]) != part.want { + t.Fatalf("%s hash mismatch go=%s ch=%s mat=%q", part.name, part.want, rows[0]["sig"], part.mat) + } + } + + // Full composite from CH normalizer + sqlFull := fmt.Sprintf( + "SELECT %s AS sig, %s AS pattern_id, %s AS edge_id, %s AS skel", + chSQLCompositeExpr(chQuote(tc.err), chQuote(string(tc.meta.SourceType)), chQuote(string(tc.meta.TargetType))), + chSQLPatternExpr(chQuote(tc.err)), + chSQLEdgeExpr(chQuote(tc.err), chQuote(string(tc.meta.SourceType)), chQuote(string(tc.meta.TargetType))), + chSkeletonExpr(chQuote(tc.err)), + ) + rows2 := chQuery(t, sqlFull) + chSig := cast.ToString(rows2[0]["sig"]) + chPat := cast.ToString(rows2[0]["pattern_id"]) + chEdge := cast.ToString(rows2[0]["edge_id"]) + chSkel := cast.ToString(rows2[0]["skel"]) + + if chSkel != goSig.Skeleton { + t.Fatalf("skeleton mismatch\n--- go ---\n%s\n--- ch ---\n%s", goSig.Skeleton, chSkel) + } + if chPat != goSig.PatternID || chEdge != goSig.EdgeID { + t.Fatalf("parts mismatch go=%s/%s ch=%s/%s", goSig.PatternID, goSig.EdgeID, chPat, chEdge) + } + if chSig != goSig.ID { + t.Fatalf("composite mismatch go=%s ch=%s", goSig.ID, chSig) + } + }) + } +} + +// TestClickHouseLiveErrors_GoAndCHMatch pulls real plausible_events rows. +func TestClickHouseLiveErrors_GoAndCHMatch(t *testing.T) { + sql := ` +SELECT + toString(meta_json.error) AS err, + JSONExtractString(ifNull(task_string, ''), 'source_type') AS source_type, + JSONExtractString(ifNull(task_string, ''), 'target_type') AS target_type +FROM analytics.plausible_events +WHERE timestamp > now() - INTERVAL 14 DAY + AND toString(meta_json.error) != '' + AND length(toString(meta_json.error)) BETWEEN 40 AND 2500 + AND JSONExtractString(ifNull(task_string, ''), 'source_type') != '' +LIMIT 40 +` + rows := chQuery(t, sql) + if len(rows) == 0 { + t.Skip("no recent error rows in plausible_events") + } + + var ( + hashMatches int + fullMatches int + fullChecked int + ) + + seenPattern := map[string]struct{}{} + seenComposite := map[string]struct{}{} + for i, row := range rows { + errText := cast.ToString(row["err"]) + meta := SignMeta{ + SourceType: dbio.Type(cast.ToString(row["source_type"])), + TargetType: dbio.Type(cast.ToString(row["target_type"])), + } + goSig := SignError(errText, meta) + + // Always: CH SHA256 of pattern/edge materials == Go parts + prows := chQuery(t, "SELECT "+chSQLHashPart(goSig.PatternMaterial)+" AS sig") + erows := chQuery(t, "SELECT "+chSQLHashPart(goSig.EdgeMaterial)+" AS sig") + if cast.ToString(prows[0]["sig"]) != goSig.PatternID || cast.ToString(erows[0]["sig"]) != goSig.EdgeID { + t.Errorf("row %d material hash mismatch go=%s/%s ch=%s/%s", + i, goSig.PatternID, goSig.EdgeID, prows[0]["sig"], erows[0]["sig"]) + continue + } + hashMatches++ + seenPattern[goSig.PatternID] = struct{}{} + seenComposite[goSig.ID] = struct{}{} + + if strings.Contains(goSig.Skeleton, "code:") { + continue + } + fullChecked++ + fullSQL := fmt.Sprintf( + "SELECT %s AS sig", + chSQLCompositeExpr(chQuote(errText), chQuote(string(meta.SourceType)), chQuote(string(meta.TargetType))), + ) + frows := chQuery(t, fullSQL) + chSig := cast.ToString(frows[0]["sig"]) + if chSig != goSig.ID { + skelSQL := "SELECT " + chSkeletonExpr(chQuote(errText)) + " AS skel" + srows := chQuery(t, skelSQL) + t.Errorf("row %d full sig mismatch go=%s ch=%s\n--- go skel ---\n%s\n--- ch skel ---\n%s\nerr_prefix=%q", + i, goSig.ID, chSig, goSig.Skeleton, cast.ToString(srows[0]["skel"]), truncate(errText, 160)) + continue + } + fullMatches++ + } + + t.Logf("live sample: rows=%d material_hash_ok=%d full_checked=%d full_ok=%d unique_patterns=%d unique_composites=%d", + len(rows), hashMatches, fullChecked, fullMatches, len(seenPattern), len(seenComposite)) + + if hashMatches != len(rows) { + t.Fatalf("material hash parity failed for %d/%d rows", len(rows)-hashMatches, len(rows)) + } + if fullChecked > 0 && fullMatches < fullChecked { + t.Fatalf("full CH normalizer parity failed for %d/%d non-code rows", fullChecked-fullMatches, fullChecked) + } +} + +// TestClickHouseLiveTopPatternStable: same skeleton, renumbered frames, same composite; +// different target shares pattern only. +func TestClickHouseLiveTopPatternStable(t *testing.T) { + errA := `--- task_run.go:140 func2 --- +--- task_run.go:830 runDbToDb --- +~ Could not WriteToDb +--- task_run_write.go:168 WriteToDb --- +no stream columns detected` + errB := `--- task_run.go:125 func2 --- +--- task_run.go:881 runDbToDb --- +~ Could not WriteToDb +--- task_run_write.go:450 WriteToDb --- +no stream columns detected` + meta := SignMeta{SourceType: dbio.TypeDbPrometheus, TargetType: dbio.TypeDbPostgres} + sa := SignError(errA, meta) + sb := SignError(errB, meta) + if sa.ID != sb.ID || sa.PatternID != sb.PatternID { + t.Fatalf("line renumber should not split: %s vs %s", sa.ID, sb.ID) + } + + // Different target → same pattern, different edge/composite + sc := SignError(errA, SignMeta{SourceType: dbio.TypeDbPrometheus, TargetType: dbio.TypeDbSnowflake}) + if sc.PatternID != sa.PatternID { + t.Fatalf("pattern should match across targets: %s vs %s", sa.PatternID, sc.PatternID) + } + if sc.ID == sa.ID || sc.EdgeID == sa.EdgeID { + t.Fatalf("edge/composite should differ by target") + } + + // CH agrees on both composites + for _, errText := range []string{errA, errB} { + sql := fmt.Sprintf("SELECT %s AS sig, %s AS pattern_id", + chSQLCompositeExpr(chQuote(errText), chQuote(string(meta.SourceType)), chQuote(string(meta.TargetType))), + chSQLPatternExpr(chQuote(errText))) + rows := chQuery(t, sql) + if cast.ToString(rows[0]["sig"]) != sa.ID { + t.Fatalf("CH composite: got %s want %s", rows[0]["sig"], sa.ID) + } + if cast.ToString(rows[0]["pattern_id"]) != sa.PatternID { + t.Fatalf("CH pattern: got %s want %s", rows[0]["pattern_id"], sa.PatternID) + } + } + t.Logf("stable composite for no_stream_columns: %s (%s) pattern=%s", sa.ID, sa.Display(), sa.PatternID) +} + +// --- ClickHouse SQL helpers (test-only parity with Go SignError) --------------- + +func chSQLHashPart(material string) string { + return fmt.Sprintf("lower(substring(hex(SHA256(%s)), 1, %d))", chQuote(material), PartIDLen) +} + +func chSQLPatternExpr(errExpr string) string { + skel := chSkeletonExpr(errExpr) + mat := fmt.Sprintf("concat(%s, '|', %s)", chQuote(PatternMaterialPrefix), skel) + return fmt.Sprintf("lower(substring(hex(SHA256(%s)), 1, %d))", mat, PartIDLen) +} + +func chSQLEdgeExpr(errExpr, sourceExpr, targetExpr string) string { + skel := chSkeletonExpr(errExpr) + src := fmt.Sprintf("if(empty(trimBoth(%s)), '-', lower(trimBoth(%s)))", sourceExpr, sourceExpr) + tgt := fmt.Sprintf("if(empty(trimBoth(%s)), '-', lower(trimBoth(%s)))", targetExpr, targetExpr) + mat := fmt.Sprintf( + "concat(%s, '|', %s, '|', %s, '|', %s)", + chQuote(EdgeMaterialPrefix), src, tgt, skel, + ) + return fmt.Sprintf("lower(substring(hex(SHA256(%s)), 1, %d))", mat, PartIDLen) +} + +func chSQLCompositeExpr(errExpr, sourceExpr, targetExpr string) string { + return fmt.Sprintf("concat(%s, %s)", + chSQLPatternExpr(errExpr), + chSQLEdgeExpr(errExpr, sourceExpr, targetExpr), + ) +} + +func chSkeletonExpr(errExpr string) string { + steps := []struct { + pat, repl string + }{ + {`(?m)^---\s+\S+\.go:\d+\s+.*\n?`, ``}, + {`(?m)^-{3,}[^-].*-{3,}\s*\n?`, ``}, + {`(?m)^~\s*`, ``}, + {`(?i)"(?:https?|s3|gs|file|azure|abfs|abfss)://[^"]*"`, ``}, + {`"(?:/|~/)[^"]*"`, ``}, + {`(?i)\b(?:https?|s3|gs|file|azure|abfs|abfss)://[^\s"'<>]+`, ``}, + {`\b\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?\b`, ``}, + {`(?i)\b[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\b`, ``}, + {`\b\d{1,3}(?:\.\d{1,3}){3}(?::\d{1,5})?\b`, ``}, + {`\(version\s+[^)]+\)`, ``}, + {`\btemp[A-Za-z0-9]{3,}\b`, ``}, + {`\b(?:exec_[A-Za-z0-9]+|[0-9A-Za-z]{24,}|[0-9a-fA-F]{16,})\b`, ``}, + {`"[^"\s]{1,256}"`, ``}, + {"`[^`\\s]{1,256}`", ``}, + {`(^|[\s"'=(])(/[^\s"'<>]+)`, `\1`}, + {`\b\d{3,}\b`, ``}, + } + + expr := errExpr + for _, s := range steps { + expr = fmt.Sprintf("replaceRegexpAll(%s, %s, %s)", expr, chQuote(s.pat), chQuote(s.repl)) + } + + lineMap := fmt.Sprintf( + `arrayMap(x -> trimBoth(replaceRegexpAll(lower(x), %s, ' ')), splitByChar('\n', %s))`, + chQuote(`[ \t]+`), + expr, + ) + filtered := fmt.Sprintf(`arrayFilter(x -> x != '', %s)`, lineMap) + joined := fmt.Sprintf(`arrayStringConcat(%s, '\n')`, filtered) + return fmt.Sprintf(`if(%s = '', 'unknown_error', %s)`, joined, joined) +} + +func chQuote(s string) string { + s = strings.ReplaceAll(s, `\`, `\\`) + s = strings.ReplaceAll(s, `'`, `\'`) + return "'" + s + "'" +} + +func TestWriteFailureSnapshotReadableByInvestigate(t *testing.T) { + withTempHomeDir(t) + + WriteFailureSnapshot(FailureSnapshot{ + ExecID: "exec_test123", + ErrMsg: "column missing: email_verified", + ConfigPath: "./r.yaml", + }) + + dir := findLocalExecDir("exec_test123") + if dir == "" { + t.Fatal("findLocalExecDir returned empty after WriteFailureSnapshot") + } + wantDir := filepath.Join(ExecutionsDir(), "exec_test123") + if dir != wantDir { + t.Fatalf("snapshot dir = %q want %q", dir, wantDir) + } + errBytes, err := os.ReadFile(filepath.Join(dir, "error.txt")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(errBytes), "email_verified") { + t.Fatalf("error.txt = %q", errBytes) + } + if _, err := os.Stat(filepath.Join(dir, "meta.json")); err != nil { + t.Fatal(err) + } + + execs, err := ListLocalExecs() + if err != nil { + t.Fatal(err) + } + found := false + for _, e := range execs { + if e.ID == "exec_test123" { + found = true + if e.Status != "err" { + t.Fatalf("status = %q, want err", e.Status) + } + if e.ConfigPath != "./r.yaml" { + t.Fatalf("config_path = %q", e.ConfigPath) + } + } + } + if !found { + t.Fatal("ListLocalExecs did not include written exec") + } +} + +func TestPrintFailureFooterRespectsEnv(t *testing.T) { + t.Setenv("SLING_ASSIST_HINT", "false") + PrintFailureFooter(FailureFooterOpts{ExecID: "exec_x", ErrMsg: "boom"}) + MaybePrintErrorHint("exec_x") +} + +func TestPrintFailureFooterErrorFlag(t *testing.T) { + withTempHomeDir(t) + t.Setenv("SLING_ASSIST_HINT", "true") + + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + old := os.Stderr + os.Stderr = w + PrintFailureFooter(FailureFooterOpts{ExecID: "exec_x", ErrMsg: "boom"}) + _ = w.Close() + os.Stderr = old + + var buf bytes.Buffer + if _, err := buf.ReadFrom(r); err != nil { + t.Fatal(err) + } + out := buf.String() + if !strings.Contains(out, "sling assist --id exec_x") { + t.Fatalf("missing --id hint in footer: %q", out) + } + // The signature is agent context (meta.json), never user-facing output. + if strings.Contains(out, "error_signature") || strings.Contains(out, "sling assist error ") { + t.Fatalf("signature leaked into footer: %q", out) + } + if strings.Contains(out, "--exec ") { + t.Fatalf("stale --exec flag in footer: %q", out) + } + // One hint line, plus one leading blank. + lines := []string{} + for _, ln := range strings.Split(strings.Trim(out, "\n"), "\n") { + if strings.TrimSpace(ln) != "" { + lines = append(lines, ln) + } + } + if len(lines) != 1 { + t.Fatalf("footer must be one line, got %d: %q", len(lines), out) + } + if !strings.HasPrefix(lines[0], " ") { + t.Fatalf("footer line must be indented: %q", lines[0]) + } +} + +func TestShortExecIDTrims(t *testing.T) { + long := "3IGpCdEfUbXfaOlpYjrXXy2uKbL" + if got := ShortExecID(long); got != "3IGpCdEf" { + t.Fatalf("ShortExecID = %q", got) + } + if got := ShortExecID("abc"); got != "abc" { + t.Fatalf("short id must pass through, got %q", got) + } +} + +func TestWriteFailureSnapshotConnName(t *testing.T) { + withTempHomeDir(t) + WriteFailureSnapshot(FailureSnapshot{ + ExecID: "exec_conntest", + ErrMsg: "connection refused", + ConnName: "MY_PG", + }) + dir := findLocalExecDir("exec_conntest") + if dir == "" { + t.Fatal("missing snapshot dir") + } + if _, err := os.Stat(filepath.Join(dir, "config.snapshot.yaml")); !os.IsNotExist(err) { + t.Fatal("conns test snapshot must not include config.snapshot.yaml") + } + le, err := ResolveLocalExec("exec_conntest") + if err != nil { + t.Fatal(err) + } + if le.ConnName != "MY_PG" { + t.Fatalf("ConnName=%q", le.ConnName) + } + if le.ConfigPath != "" { + t.Fatalf("ConfigPath should be empty, got %q", le.ConfigPath) + } + if le.displayObject() != "MY_PG" { + t.Fatalf("displayObject=%q", le.displayObject()) + } +} + +func TestWriteFailureSnapshotIncludesSignature(t *testing.T) { + withTempHomeDir(t) + errMsg := `--- task_run.go:140 func2 --- +~ Could not WriteToDb +no stream columns detected` + WriteFailureSnapshot(FailureSnapshot{ + ExecID: "sigExec1", + ErrMsg: errMsg, + SignMeta: SignMeta{ + SourceType: dbio.TypeDbPrometheus, + TargetType: dbio.TypeDbPostgres, + }, + }) + dir := findLocalExecDir("sigExec1") + if dir == "" { + t.Fatal("exec dir missing") + } + body, err := os.ReadFile(filepath.Join(dir, "meta.json")) + if err != nil { + t.Fatal(err) + } + meta := map[string]any{} + if err := json.Unmarshal(body, &meta); err != nil { + t.Fatal(err) + } + wantSig := SignError(errMsg, SignMeta{ + SourceType: dbio.TypeDbPrometheus, TargetType: dbio.TypeDbPostgres, + }) + if cast.ToString(meta["error_signature"]) != wantSig.ID { + t.Fatalf("error_signature = %v want %s", meta["error_signature"], wantSig.ID) + } + if cast.ToString(meta["error_pattern_id"]) != wantSig.PatternID { + t.Fatalf("pattern_id = %v", meta["error_pattern_id"]) + } + if cast.ToString(meta["error_edge_id"]) != wantSig.EdgeID { + t.Fatalf("edge_id = %v", meta["error_edge_id"]) + } + if cast.ToString(meta["error_algorithm"]) != "v1" { + t.Fatalf("algorithm = %v", meta["error_algorithm"]) + } +} + +func TestLookupError(t *testing.T) { + // Pattern-only (8 hex) + r, err := LookupError("97d84811") + if err != nil { + t.Fatal(err) + } + if r.Signature != "97d84811" || !r.PatternOnly || r.PatternID != "97d84811" { + t.Fatalf("%+v", r) + } + // Full composite + r2, err := LookupError("97d84811-5aede62c") + if err != nil || r2.Signature != "97d848115aede62c" || r2.PatternOnly { + t.Fatalf("dashed composite: %+v err=%v", r2, err) + } + if r2.PatternID != "97d84811" || r2.EdgeID != "5aede62c" { + t.Fatalf("parts: %+v", r2) + } + // Display form + r3, err := LookupError("97d84811-5aede62c (prometheus→postgres · no_stream_columns)") + if err != nil || r3.Signature != "97d848115aede62c" { + t.Fatalf("display form: %+v err=%v", r3, err) + } + if _, err := LookupError("not-a-sig"); err == nil { + t.Fatal("expected invalid signature error") + } +} + +func TestSanitizeLogForPromptCapsAndEscapesFences(t *testing.T) { + prev := env.Env + t.Cleanup(func() { env.Env = prev }) + env.Env = &env.EnvFile{Connections: map[string]map[string]any{}} + + in := "before\n```\ninject\n```\nafter" + out := sanitizeLogForPrompt(in, 0) + if strings.Contains(out, "```") { + t.Fatalf("fence not escaped: %q", out) + } + if !strings.Contains(out, "'''") { + t.Fatalf("expected escaped fence: %q", out) + } + big := strings.Repeat("x", maxErrorTailBytes+1000) + capped := sanitizeLogForPrompt(big, maxErrorTailBytes) + if len(capped) > maxErrorTailBytes+len("[...truncated...]\n")+10 { + t.Fatalf("cap too large: %d", len(capped)) + } + if !strings.HasPrefix(capped, "[...truncated...]\n") { + t.Fatalf("missing truncation marker: %q", capped[:40]) + } +} + +func TestSanitizeLogForPromptScrubsSecrets(t *testing.T) { + prev := env.Env + t.Cleanup(func() { env.Env = prev }) + env.Env = &env.EnvFile{Connections: map[string]map[string]any{ + "MY_PG": { + "type": "postgres", + "password": "super-secret-pass", + "secrets": map[string]any{ + "api_key": "nested-api-key", + }, + }, + }} + out := sanitizeLogForPrompt("failed auth super-secret-pass and nested-api-key", 0) + if strings.Contains(out, "super-secret-pass") { + t.Fatalf("password leaked: %q", out) + } + if strings.Contains(out, "nested-api-key") { + t.Fatalf("nested secret leaked: %q", out) + } + if !strings.Contains(out, "***") { + t.Fatalf("expected redaction marker: %q", out) + } +} + +func TestSensitivityClassify(t *testing.T) { + cases := []struct { + path string + want Sensitivity + }{ + {"/home/u/.sling/env.yaml", SensitivitySecret}, + {"/home/u/.claude.json", SensitivitySecret}, + {"/tmp/settings.json.backup", SensitivitySecret}, + {"/home/u/.sling/assist/errors/exec_x/meta.json", SensitivityPublic}, + {"/home/u/.sling/assist/errors/exec_x/error.txt", SensitivityInternal}, + {"/home/u/.agents/skills/sling/SKILL.md", SensitivityPublic}, + {"/unknown/random/file.txt", SensitivityInternal}, // default + } + for _, tc := range cases { + if got := ClassifyPath(tc.path); got != tc.want { + t.Errorf("ClassifyPath(%q)=%s want %s", tc.path, got, tc.want) + } + } +} + +func TestSensitivityManifestNonEmpty(t *testing.T) { + m := SensitivityManifest() + if len(m) < 5 { + t.Fatalf("manifest too small: %d", len(m)) + } + ids := map[string]bool{} + for _, c := range m { + if c.ID == "" || c.Glob == "" || c.Reason == "" { + t.Fatalf("incomplete entry: %+v", c) + } + if ids[c.ID] { + t.Fatalf("duplicate id %s", c.ID) + } + ids[c.ID] = true + } +} + +func TestLegacyExecDirStillReadable(t *testing.T) { + withTempHomeDir(t) + id := "legacy_exec1" + dir := filepath.Join(ErrorsDir(), id) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + meta := `{"exec_id":"` + id + `","exit_code":1,"object":"r.yaml"}` + if err := os.WriteFile(filepath.Join(dir, "meta.json"), []byte(meta), 0o644); err != nil { + t.Fatal(err) + } + if got := findLocalExecDir(id); got != dir { + t.Fatalf("findLocalExecDir = %q want %q", got, dir) + } + execs, err := ListLocalExecs() + if err != nil { + t.Fatal(err) + } + found := false + for _, e := range execs { + if e.ID == id { + found = true + } + } + if !found { + t.Fatal("ListLocalExecs missed legacy exec dir") + } +} + +func TestReservedErrorDirNamesNotListed(t *testing.T) { + withTempHomeDir(t) + _ = ExecutionsDir() + if err := os.MkdirAll(filepath.Join(ErrorsDir(), "signatures"), 0o755); err != nil { + t.Fatal(err) + } + execs, err := ListLocalExecs() + if err != nil { + t.Fatal(err) + } + for _, e := range execs { + if e.ID == "executions" || e.ID == "signatures" { + t.Fatalf("reserved name listed as exec: %q", e.ID) + } + } +} + +func TestLookupLocalExecPrefixAndAmbiguity(t *testing.T) { + withTempHomeDir(t) + for _, id := range []string{"abc111", "abc222", "zzz999"} { + dir := filepath.Join(ErrorsDir(), id) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + meta := `{"exec_id":"` + id + `","exit_code":1,"object":"r.yaml"}` + if err := os.WriteFile(filepath.Join(dir, "meta.json"), []byte(meta), 0o644); err != nil { + t.Fatal(err) + } + } + + if le, ok := LookupLocalExec("zzz999"); !ok || le.ID != "zzz999" { + t.Fatalf("full id: %+v ok=%v", le, ok) + } + if le, ok := LookupLocalExec("zzz"); !ok || le.ID != "zzz999" { + t.Fatalf("unique prefix: %+v ok=%v", le, ok) + } + if _, ok := LookupLocalExec("abc"); ok { + t.Fatal("ambiguous prefix must not resolve") + } + if _, err := ResolveLocalExec("abc"); err == nil || !strings.Contains(err.Error(), "ambiguous") { + t.Fatalf("want ambiguous error, got %v", err) + } + if _, err := ResolveLocalExec("nope"); err == nil || !strings.Contains(err.Error(), "unknown") { + t.Fatalf("want unknown error, got %v", err) + } +} + +func TestWriteFailureSnapshotKeepsRunLog(t *testing.T) { + withTempHomeDir(t) + WriteFailureSnapshot(FailureSnapshot{ + ExecID: "exec_runlog", + ErrMsg: "boom", + RunLog: "DBG opened conn\nINF execution failed", + }) + dir := filepath.Join(ExecutionsDir(), "exec_runlog") + b, err := os.ReadFile(filepath.Join(dir, "stderr.log")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(b), "DBG opened conn") { + t.Fatalf("stderr.log missing run log: %q", string(b)) + } + // error.txt keeps the error only, so runLogExcerpt sees them as distinct. + e, _ := os.ReadFile(filepath.Join(dir, "error.txt")) + if string(e) == string(b) { + t.Fatal("stderr.log must not duplicate error.txt when a run log exists") + } + if got := runLogExcerpt(LocalExec{LogDir: dir}); !strings.Contains(got, "execution failed") { + t.Fatalf("runLogExcerpt: %q", got) + } +} + +func TestRunLogExcerptEmptyWhenMirrored(t *testing.T) { + withTempHomeDir(t) + // No RunLog: stderr.log mirrors error.txt, so nothing extra reaches the prompt. + WriteFailureSnapshot(FailureSnapshot{ExecID: "exec_mirror", ErrMsg: "boom"}) + dir := filepath.Join(ExecutionsDir(), "exec_mirror") + if got := runLogExcerpt(LocalExec{LogDir: dir}); got != "" { + t.Fatalf("want empty excerpt for mirrored log, got %q", got) + } +} + +func TestAutoTrimExecsKeepsNewest(t *testing.T) { + withTempHomeDir(t) + + total := ExecsMaxEntries + 10 + base := time.Now().Add(-time.Duration(total) * time.Hour) + for i := 0; i < total; i++ { + id := fmt.Sprintf("exec_%03d", i) + dir := filepath.Join(ExecutionsDir(), id) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + // later index = newer + when := base.Add(time.Duration(i) * time.Hour) + if err := os.Chtimes(dir, when, when); err != nil { + t.Fatal(err) + } + } + + if err := AutoTrimExecs(); err != nil { + t.Fatal(err) + } + + ids, err := listLocalExecIDs() + if err != nil { + t.Fatal(err) + } + if len(ids) != ExecsMaxEntries { + t.Fatalf("kept %d snapshots, want %d", len(ids), ExecsMaxEntries) + } + // the 10 oldest must be gone, the newest must stay + if dir := findLocalExecDir("exec_000"); dir != "" { + t.Fatalf("oldest snapshot survived: %s", dir) + } + if dir := findLocalExecDir(fmt.Sprintf("exec_%03d", total-1)); dir == "" { + t.Fatal("newest snapshot was trimmed") + } +} + +func TestAutoTrimExecsNoopUnderCap(t *testing.T) { + withTempHomeDir(t) + + for i := 0; i < 5; i++ { + WriteFailureSnapshot(FailureSnapshot{ + ExecID: fmt.Sprintf("exec_keep%d", i), + ErrMsg: "boom", + }) + } + if err := AutoTrimExecs(); err != nil { + t.Fatal(err) + } + ids, err := listLocalExecIDs() + if err != nil { + t.Fatal(err) + } + if len(ids) != 5 { + t.Fatalf("kept %d snapshots, want 5", len(ids)) + } +} + +func TestAutoTrimExecsSkipsReservedDirs(t *testing.T) { + withTempHomeDir(t) + + total := ExecsMaxEntries + 5 + base := time.Now().Add(-time.Duration(total) * time.Hour) + for i := 0; i < total; i++ { + // legacy layout: errors// + dir := filepath.Join(ErrorsDir(), fmt.Sprintf("legacy_%03d", i)) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + when := base.Add(time.Duration(i) * time.Hour) + if err := os.Chtimes(dir, when, when); err != nil { + t.Fatal(err) + } + } + execsDir := ExecutionsDir() // reserved, must survive + + if err := AutoTrimExecs(); err != nil { + t.Fatal(err) + } + + if !g.PathExists(execsDir) { + t.Fatal("executions dir was removed") + } + ids, err := listLocalExecIDs() + if err != nil { + t.Fatal(err) + } + if len(ids) != ExecsMaxEntries { + t.Fatalf("kept %d snapshots, want %d", len(ids), ExecsMaxEntries) + } +} + +func TestWriteFailureSnapshotTrims(t *testing.T) { + withTempHomeDir(t) + + base := time.Now().Add(-time.Duration(ExecsMaxEntries+1) * time.Hour) + for i := 0; i < ExecsMaxEntries; i++ { + dir := filepath.Join(ExecutionsDir(), fmt.Sprintf("exec_old%03d", i)) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + when := base.Add(time.Duration(i) * time.Hour) + if err := os.Chtimes(dir, when, when); err != nil { + t.Fatal(err) + } + } + + WriteFailureSnapshot(FailureSnapshot{ExecID: "exec_newest", ErrMsg: "boom"}) + + ids, err := listLocalExecIDs() + if err != nil { + t.Fatal(err) + } + if len(ids) != ExecsMaxEntries { + t.Fatalf("kept %d snapshots, want %d", len(ids), ExecsMaxEntries) + } + if dir := findLocalExecDir("exec_newest"); dir == "" { + t.Fatal("new snapshot was trimmed by its own write") + } + if dir := findLocalExecDir("exec_old000"); dir != "" { + t.Fatalf("oldest snapshot survived: %s", dir) + } +} diff --git a/core/sling/assist/jsonedit.go b/core/sling/assist/jsonedit.go new file mode 100644 index 000000000..919c2d400 --- /dev/null +++ b/core/sling/assist/jsonedit.go @@ -0,0 +1,255 @@ +package assist + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + + "github.com/flarco/g" + "github.com/tidwall/gjson" + "github.com/tidwall/jsonc" + "github.com/tidwall/sjson" +) + +// jsonReadOrEmpty parses JSON/JSONC into a map. Missing/empty → empty map. +func jsonReadOrEmpty(path string) (map[string]any, error) { + out := map[string]any{} + data, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return out, nil + } + return nil, g.Error(err, "read %s", path) + } + if len(data) == 0 { + return out, nil + } + stripped := jsonc.ToJSON(data) + if err := json.Unmarshal(stripped, &out); err != nil { + return nil, g.Error(err, "parse %s", path) + } + return out, nil +} + +// jsonWritePretty writes sling-owned JSON (no user comments to preserve). +// For user configs use setJSONPath/deleteJSONPath. +func jsonWritePretty(path string, m map[string]any) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return g.Error(err, "mkdir %s", filepath.Dir(path)) + } + data, err := json.MarshalIndent(m, "", " ") + if err != nil { + return g.Error(err, "marshal %s", path) + } + data = append(data, '\n') + return writeBytesPreserveMode(path, data, 0o644) +} + +// jsonReadRaw reads JSON/JSONC for surgical sjson edits. +// Missing/empty → "{}". Leading non-JSON (e.g. VS Code banner) is split off +// so sjson does not compact the whole file; caller re-prepends via jsonWriteRaw. +func jsonReadRaw(path string) (prefix, body []byte, err error) { + data, rerr := os.ReadFile(path) + if rerr != nil { + if os.IsNotExist(rerr) { + return nil, []byte("{}"), nil + } + return nil, nil, g.Error(rerr, "read %s", path) + } + if len(bytes.TrimSpace(data)) == 0 { + return nil, []byte("{}"), nil + } + prefix, body = splitLeadingNonJSON(data) + return prefix, body, nil +} + +// splitLeadingNonJSON splits at the first `{` or `[`. +func splitLeadingNonJSON(data []byte) (prefix, body []byte) { + for i, c := range data { + if c == '{' || c == '[' { + if i == 0 { + return nil, data + } + return data[:i], data[i:] + } + } + return data, nil +} + +// jsonWriteRaw writes prefix+body, preserving mode (default 0600 for new files). +func jsonWriteRaw(path string, prefix, body []byte) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return g.Error(err, "mkdir %s", filepath.Dir(path)) + } + out := body + if len(prefix) > 0 { + out = append(append([]byte{}, prefix...), body...) + } + if len(out) == 0 || out[len(out)-1] != '\n' { + out = append(out, '\n') + } + return writeBytesPreserveMode(path, out, 0o600) +} + +var sjsonOpts = &sjson.Options{Optimistic: true} + +const backupSuffix = ".backup" + +func fileMode(path string, def os.FileMode) os.FileMode { + info, err := os.Stat(path) + if err != nil { + return def + } + return info.Mode().Perm() +} + +// writeBytesPreserveMode writes data, keeping existing mode when overwriting. +func writeBytesPreserveMode(path string, data []byte, defMode os.FileMode) error { + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return g.Error(err, "mkdir %s", filepath.Dir(path)) + } + mode := fileMode(path, defMode) + return os.WriteFile(path, data, mode) +} + +// backupBeforeEdit copies path → path.backup (no-op if missing). +func backupBeforeEdit(path string) error { + src, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return g.Error(err, "read %s for backup", path) + } + if len(src) == 0 { + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + return g.Error(err, "mkdir %s", filepath.Dir(path)) + } + mode := fileMode(path, 0o600) + if err := os.WriteFile(path+backupSuffix, src, mode); err != nil { + return g.Error(err, "write %s", path+backupSuffix) + } + return nil +} + +func restoreBackup(path string) error { + src, err := os.ReadFile(path + backupSuffix) + if err != nil { + return g.Error(err, "read %s", path+backupSuffix) + } + mode := fileMode(path+backupSuffix, fileMode(path, 0o600)) + return os.WriteFile(path, src, mode) +} + +func countTopLevelKeys(data []byte) int { + clean := jsonc.ToJSON(data) + res := gjson.ParseBytes(clean) + if !res.IsObject() { + return 0 + } + n := 0 + res.ForEach(func(_, _ gjson.Result) bool { + n++ + return true + }) + return n +} + +func countLines(data []byte) int { + if len(data) == 0 { + return 0 + } + n := 1 + for _, c := range data { + if c == '\n' { + n++ + } + } + return n +} + +// validateEditNotDestructive refuses edits that drop top-level keys or +// collapse multi-line docs (sjson banner bug). allowKeyDelta=1 for deletes. +func validateEditNotDestructive(before, after []byte, allowKeyDelta int) error { + oldKeys := countTopLevelKeys(before) + newKeys := countTopLevelKeys(after) + if newKeys < oldKeys-allowKeyDelta { + return g.Error("destructive edit refused: top-level keys went from %d to %d", oldKeys, newKeys) + } + oldLines := countLines(before) + newLines := countLines(after) + if oldLines >= 4 && newLines*2 < oldLines { + return g.Error("destructive edit refused: line count went from %d to %d", oldLines, newLines) + } + return nil +} + +// setJSONPath rewrites one path; backs up and refuses destructive rewrites. +func setJSONPath(path, jsonPath string, value any) error { + if err := backupBeforeEdit(path); err != nil { + return err + } + before, err := os.ReadFile(path) + if err != nil && !os.IsNotExist(err) { + return g.Error(err, "read %s", path) + } + prefix, body, err := jsonReadRaw(path) + if err != nil { + return err + } + out, serr := sjson.SetBytesOptions(body, jsonPath, value, sjsonOpts) + if serr != nil { + return g.Error(serr, "set %s in %s", jsonPath, path) + } + if len(before) > 0 { + full := append(append([]byte{}, prefix...), out...) + if verr := validateEditNotDestructive(before, full, 0); verr != nil { + return g.Error(verr, "would have corrupted %s — left original in place; previous content also at %s%s", path, path, backupSuffix) + } + } + return jsonWriteRaw(path, prefix, out) +} + +// deleteJSONPath removes one path; same backup + sanity checks as setJSONPath. +func deleteJSONPath(path, jsonPath string) error { + if err := backupBeforeEdit(path); err != nil { + return err + } + before, err := os.ReadFile(path) + if err != nil && !os.IsNotExist(err) { + return g.Error(err, "read %s", path) + } + prefix, body, err := jsonReadRaw(path) + if err != nil { + return err + } + out, derr := sjson.DeleteBytes(body, jsonPath) + if derr != nil { + return g.Error(derr, "delete %s in %s", jsonPath, path) + } + if len(before) > 0 { + full := append(append([]byte{}, prefix...), out...) + if verr := validateEditNotDestructive(before, full, 1); verr != nil { + return g.Error(verr, "would have corrupted %s — left original in place; previous content also at %s%s", path, path, backupSuffix) + } + } + return jsonWriteRaw(path, prefix, out) +} + +// gjsonGetArrayStrings reads a JSON string array (JSONC-safe). +func gjsonGetArrayStrings(data []byte, path string) []string { + clean := jsonc.ToJSON(data) + res := gjson.GetBytes(clean, path) + if !res.Exists() || !res.IsArray() { + return nil + } + out := []string{} + res.ForEach(func(_, v gjson.Result) bool { + out = append(out, v.String()) + return true + }) + return out +} diff --git a/core/sling/assist/opencode.go b/core/sling/assist/opencode.go new file mode 100644 index 000000000..00b6db28f --- /dev/null +++ b/core/sling/assist/opencode.go @@ -0,0 +1,426 @@ +package assist + +import ( + "context" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + + "github.com/flarco/g" + "github.com/flarco/g/net" + "github.com/slingdata-io/sling-cli/core/dbio/iop" + "github.com/slingdata-io/sling-cli/core/env" + "golang.org/x/sys/cpu" +) + +// OpenCodeVersion is the pinned CLI release. Override with OPENCODE_VERSION. +// Pin checked against https://github.com/anomalyco/opencode/releases (v1.18.18, 2026-08-13). +const OpenCodeVersion = "1.18.18" + +// ZenFreeDisclosure is shown when the harness uses a free Zen model. +const ZenFreeDisclosure = "free-model prompts may be used for training" + +// ZenFreeModel is the default free OpenCode Zen model id (`provider/model`). +const ZenFreeModel = "opencode/big-pickle" + +const openCodeGitHubBase = "https://github.com/anomalyco/opencode/releases/download/v{version}/{asset}" + +// openCodeTestDownloadURL replaces the GitHub asset URL in tests (httptest zip). +var openCodeTestDownloadURL string + +// zenModelsURL is the Zen catalog. Tests point this at httptest. +var zenModelsURL = "https://opencode.ai/zen/v1/models" + +var zenHTTPClient = &http.Client{Timeout: 5 * time.Second} + +var ( + openCodeHasAVX2 = func() bool { return cpu.X86.HasAVX2 } + openCodeIsMusl = linuxMuslPresent +) + +type openCodeInstall struct { + version string +} + +func newOpenCodeInstall() *openCodeInstall { + return &openCodeInstall{version: openCodeVersion()} +} + +func openCodeVersion() string { + if val := strings.TrimSpace(os.Getenv("OPENCODE_VERSION")); val != "" { + return strings.TrimPrefix(val, "v") + } + return OpenCodeVersion +} + +func openCodeBinName() string { + if runtime.GOOS == "windows" { + return "opencode.exe" + } + return "opencode" +} + +func (o *openCodeInstall) dest() string { + return filepath.Join(env.HomeBinDir(), "opencode", o.version) +} + +func (o *openCodeInstall) bundledPath() string { + return filepath.Join(o.dest(), openCodeBinName()) +} + +// BundledOpenCodePath is ~/.sling/bin/opencode//opencode[.exe]. +func BundledOpenCodePath() string { + return newOpenCodeInstall().bundledPath() +} + +func linuxMuslPresent() bool { + matches, _ := filepath.Glob("/lib/ld-musl-*") + return len(matches) > 0 +} + +func (o *openCodeInstall) assetName(goos, goarch string) (string, error) { + var osName, arch string + switch goos { + case "darwin", "linux", "windows": + osName = goos + default: + return "", g.Error("opencode is not available for %s/%s", goos, goarch) + } + switch goarch { + case "amd64": + arch = "x64" + case "arm64": + arch = "arm64" + default: + return "", g.Error("opencode is not available for %s/%s", goos, goarch) + } + + ext := "zip" + if goos == "linux" { + ext = "tar.gz" + } + + suffix := "" + if arch == "x64" && !openCodeHasAVX2() { + suffix += "-baseline" + } + if goos == "linux" && openCodeIsMusl() { + suffix += "-musl" + } + return fmt.Sprintf("opencode-%s-%s%s.%s", osName, arch, suffix, ext), nil +} + +// OpenCodeAssetName is the GitHub asset for goos/goarch (pinned layout, not /latest). +func OpenCodeAssetName(goos, goarch string) (string, error) { + return newOpenCodeInstall().assetName(goos, goarch) +} + +func (o *openCodeInstall) downloadURL() (string, error) { + if openCodeTestDownloadURL != "" { + return openCodeTestDownloadURL, nil + } + asset, err := o.assetName(runtime.GOOS, runtime.GOARCH) + if err != nil { + return "", err + } + return g.R(openCodeGitHubBase, "version", o.version, "asset", asset), nil +} + +func openCodeDownloadURL(version string) (string, error) { + return (&openCodeInstall{version: version}).downloadURL() +} + +func versionMatches(out, version string) bool { + s := strings.TrimSpace(out) + return strings.HasPrefix(s, version) || strings.HasPrefix(s, "v"+version) +} + +func (o *openCodeInstall) versionOK(binPath string) (bool, error) { + out, err := exec.Command(binPath, "--version").CombinedOutput() + if err != nil { + return false, g.Error(err, "could not get version for opencode: %s", strings.TrimSpace(string(out))) + } + return versionMatches(string(out), o.version), nil +} + +func (o *openCodeInstall) findBin(folder string) string { + want := openCodeBinName() + direct := filepath.Join(folder, want) + if g.PathExists(direct) { + return direct + } + var found string + _ = filepath.Walk(folder, func(p string, info os.FileInfo, err error) error { + if err != nil || info.IsDir() { + return nil + } + if info.Name() == want || info.Name() == "opencode" { + found = p + return filepath.SkipAll + } + return nil + }) + return found +} + +func (o *openCodeInstall) extract(archive, dest string) error { + if strings.HasSuffix(archive, ".tar.gz") || strings.HasSuffix(archive, ".tgz") { + if err := g.ExtractTarGz(archive, dest); err != nil { + return g.Error(err, "error extracting opencode archive") + } + return nil + } + if _, err := iop.Unzip(archive, dest); err != nil { + return g.Error(err, "error unzipping opencode archive") + } + return nil +} + +// EnsureBinOpenCode returns a usable opencode binary. +// Order: OPENCODE_PATH, $PATH, then a versioned download under ~/.sling/bin/opencode//. +func EnsureBinOpenCode() (binPath string, err error) { + return newOpenCodeInstall().ensure() +} + +func (o *openCodeInstall) ensure() (binPath string, err error) { + if envPath := os.Getenv("OPENCODE_PATH"); envPath != "" { + if !g.PathExists(envPath) { + return "", g.Error("opencode binary not found: %s", envPath) + } + if stat, _ := os.Stat(envPath); stat != nil && stat.IsDir() { + return "", g.Error("OPENCODE_PATH provided is a directory, should be a file: %s", envPath) + } + return envPath, nil + } + + if p, err := exec.LookPath("opencode"); err == nil { + return p, nil + } + + folderPath := o.dest() + binPath = o.bundledPath() + found := g.PathExists(binPath) + if found { + ok, verr := o.versionOK(binPath) + if verr != nil { + found = false + } else { + found = ok + } + } + + if !found { + downloadURL, uerr := o.downloadURL() + if uerr != nil { + return "", uerr + } + + ext := ".zip" + if strings.Contains(downloadURL, ".tar.gz") { + ext = ".tar.gz" + } + archivePath := filepath.Join(os.TempDir(), g.F("opencode-%s%s", o.version, ext)) + defer os.Remove(archivePath) + + g.Info("downloading opencode %s for %s/%s", o.version, runtime.GOOS, runtime.GOARCH) + if err = net.DownloadFile(downloadURL, archivePath); err != nil { + return "", g.Error(err, "unable to download opencode binary") + } + + if err = os.MkdirAll(folderPath, 0755); err != nil { + return "", g.Error(err, "could not create opencode folder") + } + + if err = o.extract(archivePath, folderPath); err != nil { + return "", err + } + + foundBin := o.findBin(folderPath) + if foundBin == "" { + return "", g.Error("cannot find opencode binary at %s after extraction", binPath) + } + if foundBin != binPath { + if err = os.Rename(foundBin, binPath); err != nil { + return "", g.Error(err, "could not move opencode binary to %s", binPath) + } + } + if !g.PathExists(binPath) { + return "", g.Error("cannot find opencode binary at %s after extraction", binPath) + } + if err = os.Chmod(binPath, 0755); err != nil { + return "", g.Error(err, "could not make opencode executable") + } + } + + ok, err := o.versionOK(binPath) + if err != nil { + return "", err + } + if !ok { + return "", g.Error("opencode at %s does not report version %s", binPath, o.version) + } + return binPath, nil +} + +// ProviderChoice is one harness LLM setup (Zen free vs keyed provider). +type ProviderChoice struct { + Kind string // zen-free | anthropic | openai | google | xai + Model string // provider/model + Provider map[string]any // opencode.json `provider` map; nil if unused + Disclosure string // set for zen-free +} + +func envFirst(keys ...string) string { + for _, k := range keys { + if v := strings.TrimSpace(os.Getenv(k)); v != "" { + return k + } + } + return "" +} + +func keyedProvider(kind, model, envKey string) ProviderChoice { + return ProviderChoice{ + Kind: kind, + Model: model, + Provider: map[string]any{ + kind: map[string]any{ + "options": map[string]any{ + "apiKey": "{env:" + envKey + "}", + }, + }, + }, + } +} + +// HarnessProviderChoice picks a keyed provider env if set, else free Zen. +func HarnessProviderChoice() ProviderChoice { + if k := envFirst("ANTHROPIC_API_KEY"); k != "" { + return keyedProvider("anthropic", "anthropic/claude-sonnet-4-5", k) + } + if k := envFirst("OPENAI_API_KEY"); k != "" { + return keyedProvider("openai", "openai/gpt-4o", k) + } + if k := envFirst("GEMINI_API_KEY", "GOOGLE_API_KEY"); k != "" { + return keyedProvider("google", "google/gemini-2.0-flash", k) + } + if k := envFirst("XAI_API_KEY"); k != "" { + return keyedProvider("xai", "xai/grok-3", k) + } + return ProviderChoice{ + Kind: "zen-free", + Model: ZenFreeModel, + Disclosure: ZenFreeDisclosure, + } +} + +// HarnessProviderConfig is the opencode.json fragment for the harness choice. +func HarnessProviderConfig() map[string]any { + c := HarnessProviderChoice() + out := map[string]any{"model": c.Model} + if len(c.Provider) > 0 { + out["provider"] = c.Provider + } + return out +} + +// ApplyHarnessProviderConfig writes model (and provider) when missing. +func ApplyHarnessProviderConfig() error { + path := filepath.Join(opencodeConfigDir(), "opencode.json") + doc, err := jsonReadOrEmpty(path) + if err != nil { + return err + } + cfg := HarnessProviderConfig() + if _, ok := doc["model"]; !ok { + if err := setJSONPath(path, "model", cfg["model"]); err != nil { + return err + } + } + if _, has := doc["provider"]; !has { + if p, ok := cfg["provider"]; ok { + if err := setJSONPath(path, "provider", p); err != nil { + return err + } + } + } + return nil +} + +func opencodeRelevant() bool { + if (&opencodeClient{}).Detect() { + return true + } + prof, exists, err := LoadProfile() + if err != nil || !exists { + return false + } + return prof.Agent == "opencode" +} + +var zenFreeModelTokens = []string{ + "big-pickle", + "mimo-v2.5-free", + "hy3-free", + "nemotron-3-ultra-free", + "nemotron-3.5-lightning-free", + "muse-spark-1.2-contributor-free", +} + +// ProbeZenFreeModel reports whether the free Zen catalog is reachable. +func ProbeZenFreeModel() (ok bool, detail string) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, zenModelsURL, nil) + if err != nil { + return false, err.Error() + } + resp, err := zenHTTPClient.Do(req) + if err != nil { + return false, err.Error() + } + defer resp.Body.Close() + body, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return false, fmt.Sprintf("HTTP %d", resp.StatusCode) + } + s := strings.ToLower(string(body)) + for _, tok := range zenFreeModelTokens { + if strings.Contains(s, strings.ToLower(tok)) { + return true, tok + } + } + return false, "no free model in catalog" +} + +func (r *DoctorReport) addZenFinding() { + if r == nil || !opencodeRelevant() { + return + } + if HarnessProviderChoice().Kind != "zen-free" { + return + } + ok, detail := ProbeZenFreeModel() + if ok { + r.AddFinding(DoctorFinding{ + ID: "opencode.zen", + OK: true, + Summary: "free model available", + Detail: detail, + }) + return + } + r.AddFinding(DoctorFinding{ + ID: "opencode.zen", + OK: false, + Summary: "free model unavailable", + Detail: detail, + Hint: "set ANTHROPIC_API_KEY or OPENAI_API_KEY for a keyed provider", + }) +} diff --git a/core/sling/assist/opencode_network_test.go b/core/sling/assist/opencode_network_test.go new file mode 100644 index 000000000..95c4e2b00 --- /dev/null +++ b/core/sling/assist/opencode_network_test.go @@ -0,0 +1,35 @@ +//go:build network + +package assist + +import ( + "net/http" + "testing" + "time" +) + +// Optional live check that the pinned GitHub asset exists. +// Default `go test ./core/sling/assist/` stays offline (this file is tagged). +func TestPinnedOpenCodeReleaseAssetExists(t *testing.T) { + openCodeTestDownloadURL = "" + u, err := openCodeDownloadURL(OpenCodeVersion) + if err != nil { + t.Fatal(err) + } + client := &http.Client{Timeout: 15 * time.Second, CheckRedirect: func(req *http.Request, via []*http.Request) error { + return nil + }} + req, err := http.NewRequest(http.MethodGet, u, nil) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Range", "bytes=0-0") + resp, err := client.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusPartialContent && resp.StatusCode != http.StatusFound && resp.StatusCode != http.StatusTemporaryRedirect { + t.Fatalf("pinned asset %s: HTTP %d", u, resp.StatusCode) + } +} diff --git a/core/sling/assist/opencode_test.go b/core/sling/assist/opencode_test.go new file mode 100644 index 000000000..d52d2cd7d --- /dev/null +++ b/core/sling/assist/opencode_test.go @@ -0,0 +1,432 @@ +package assist + +import ( + "archive/zip" + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "testing" +) + +func isolateOpenCodeEnv(t *testing.T) { + t.Helper() + home := withTempHomeDir(t) + bin := filepath.Join(home, "empty-bin") + if err := os.MkdirAll(bin, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", bin) + t.Setenv("OPENCODE_PATH", "") + t.Setenv("OPENCODE_VERSION", OpenCodeVersion) + t.Setenv("XDG_CONFIG_HOME", "") + t.Setenv("ANTHROPIC_API_KEY", "") + t.Setenv("OPENAI_API_KEY", "") + t.Setenv("GEMINI_API_KEY", "") + t.Setenv("GOOGLE_API_KEY", "") + t.Setenv("XAI_API_KEY", "") + t.Cleanup(func() { openCodeTestDownloadURL = "" }) +} + +func zipOpenCodeStub(t *testing.T, version string) []byte { + t.Helper() + buf := new(bytes.Buffer) + zw := zip.NewWriter(buf) + h := &zip.FileHeader{Name: "opencode", Method: zip.Deflate} + h.SetMode(0o755) + w, err := zw.CreateHeader(h) + if err != nil { + t.Fatal(err) + } + script := "#!/bin/sh\n" + + "if [ \"$1\" = \"--version\" ]; then echo \"" + version + "\"; exit 0; fi\n" + + "echo stub\n" + if _, err := io.WriteString(w, script); err != nil { + t.Fatal(err) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func serveOpenCodeZip(t *testing.T, version string) (*httptest.Server, *atomic.Int32) { + t.Helper() + payload := zipOpenCodeStub(t, version) + hits := &atomic.Int32{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.Header().Set("Content-Type", "application/zip") + _, _ = w.Write(payload) + })) + t.Cleanup(srv.Close) + return srv, hits +} + +func TestOpenCodeAssetNameLinuxX64(t *testing.T) { + prevAVX, prevMusl := openCodeHasAVX2, openCodeIsMusl + t.Cleanup(func() { + openCodeHasAVX2, openCodeIsMusl = prevAVX, prevMusl + }) + + cases := []struct { + avx2, musl bool + want string + }{ + {true, false, "opencode-linux-x64.tar.gz"}, + {false, false, "opencode-linux-x64-baseline.tar.gz"}, + {true, true, "opencode-linux-x64-musl.tar.gz"}, + {false, true, "opencode-linux-x64-baseline-musl.tar.gz"}, + } + for _, tc := range cases { + openCodeHasAVX2 = func() bool { return tc.avx2 } + openCodeIsMusl = func() bool { return tc.musl } + got, err := OpenCodeAssetName("linux", "amd64") + if err != nil { + t.Fatalf("avx2=%v musl=%v: %v", tc.avx2, tc.musl, err) + } + if got != tc.want { + t.Errorf("avx2=%v musl=%v: got %s want %s", tc.avx2, tc.musl, got, tc.want) + } + } +} + +func TestOpenCodeAssetNamePinnedPlatforms(t *testing.T) { + prevAVX, prevMusl := openCodeHasAVX2, openCodeIsMusl + t.Cleanup(func() { + openCodeHasAVX2, openCodeIsMusl = prevAVX, prevMusl + }) + openCodeHasAVX2 = func() bool { return true } + openCodeIsMusl = func() bool { return false } + + cases := map[string]string{ + "darwin/arm64": "opencode-darwin-arm64.zip", + "darwin/amd64": "opencode-darwin-x64.zip", + "windows/amd64": "opencode-windows-x64.zip", + "windows/arm64": "opencode-windows-arm64.zip", + "linux/arm64": "opencode-linux-arm64.tar.gz", + } + for plat, want := range cases { + parts := strings.Split(plat, "/") + got, err := OpenCodeAssetName(parts[0], parts[1]) + if err != nil { + t.Fatalf("%s: %v", plat, err) + } + if got != want { + t.Errorf("%s: got %s want %s", plat, got, want) + } + } + if _, err := OpenCodeAssetName("js", "wasm"); err == nil { + t.Fatal("expected error for js/wasm") + } +} + +func TestEnsureBinOpenCodeInstallFromZip(t *testing.T) { + isolateOpenCodeEnv(t) + srv, hits := serveOpenCodeZip(t, OpenCodeVersion) + openCodeTestDownloadURL = srv.URL + "/opencode.zip" + + bin, err := EnsureBinOpenCode() + if err != nil { + t.Fatalf("EnsureBinOpenCode: %v", err) + } + want := BundledOpenCodePath() + if bin != want { + t.Fatalf("bin = %s, want %s", bin, want) + } + info, err := os.Stat(bin) + if err != nil { + t.Fatal(err) + } + if info.Mode().Perm() != 0o755 { + t.Fatalf("perm = %o, want 0755", info.Mode().Perm()) + } + out, err := exec.Command(bin, "--version").CombinedOutput() + if err != nil { + t.Fatalf("--version: %v (%s)", err, out) + } + if !versionMatches(string(out), OpenCodeVersion) { + t.Fatalf("version %q does not match %s", out, OpenCodeVersion) + } + if hits.Load() != 1 { + t.Fatalf("download hits = %d, want 1", hits.Load()) + } + + bin2, err := EnsureBinOpenCode() + if err != nil { + t.Fatal(err) + } + if bin2 != bin { + t.Fatalf("second call bin = %s", bin2) + } + if hits.Load() != 1 { + t.Fatalf("stale dir should not re-download; hits = %d", hits.Load()) + } +} + +func TestEnsureBinOpenCodeStaleRedownload(t *testing.T) { + isolateOpenCodeEnv(t) + want := BundledOpenCodePath() + if err := os.MkdirAll(filepath.Dir(want), 0o755); err != nil { + t.Fatal(err) + } + stale := "#!/bin/sh\necho 0.0.1\n" + if err := os.WriteFile(want, []byte(stale), 0o755); err != nil { + t.Fatal(err) + } + + srv, hits := serveOpenCodeZip(t, OpenCodeVersion) + openCodeTestDownloadURL = srv.URL + "/opencode.zip" + + bin, err := EnsureBinOpenCode() + if err != nil { + t.Fatalf("EnsureBinOpenCode: %v", err) + } + if bin != want { + t.Fatalf("bin = %s, want %s", bin, want) + } + if hits.Load() != 1 { + t.Fatalf("stale binary should re-download; hits = %d", hits.Load()) + } + out, err := exec.Command(bin, "--version").CombinedOutput() + if err != nil { + t.Fatal(err) + } + if !versionMatches(string(out), OpenCodeVersion) { + t.Fatalf("after re-download version = %q", out) + } +} + +func TestEnsureBinOpenCodePathOverride(t *testing.T) { + isolateOpenCodeEnv(t) + dir := t.TempDir() + asDir := filepath.Join(dir, "as-dir") + if err := os.MkdirAll(asDir, 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("OPENCODE_PATH", asDir) + if _, err := EnsureBinOpenCode(); err == nil { + t.Fatal("expected error for directory OPENCODE_PATH") + } + + file := filepath.Join(dir, "opencode") + if err := os.WriteFile(file, []byte("x"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("OPENCODE_PATH", file) + got, err := EnsureBinOpenCode() + if err != nil { + t.Fatal(err) + } + if got != file { + t.Fatalf("got %s want %s", got, file) + } +} + +func TestEnsureBinOpenCodeSystemPathWins(t *testing.T) { + isolateOpenCodeEnv(t) + home := withTempHomeDir(t) + binDir := filepath.Join(home, "sys-bin") + if err := os.MkdirAll(binDir, 0o755); err != nil { + t.Fatal(err) + } + sys := filepath.Join(binDir, "opencode") + if err := os.WriteFile(sys, []byte("#!/bin/sh\necho sys\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir) + + hits := &atomic.Int32{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + w.WriteHeader(500) + })) + t.Cleanup(srv.Close) + openCodeTestDownloadURL = srv.URL + "/opencode.zip" + + got, err := EnsureBinOpenCode() + if err != nil { + t.Fatal(err) + } + if got != sys { + t.Fatalf("got %s want system %s", got, sys) + } + if hits.Load() != 0 { + t.Fatalf("system binary should skip download; hits = %d", hits.Load()) + } +} + +func TestHarnessProviderChoiceZenDisclosure(t *testing.T) { + isolateOpenCodeEnv(t) + c := HarnessProviderChoice() + if c.Kind != "zen-free" { + t.Fatalf("kind = %s", c.Kind) + } + if c.Model != ZenFreeModel { + t.Fatalf("model = %s", c.Model) + } + if c.Disclosure != ZenFreeDisclosure { + t.Fatalf("disclosure = %q", c.Disclosure) + } + cfg := HarnessProviderConfig() + if cfg["model"] != ZenFreeModel { + t.Fatalf("config model = %v", cfg["model"]) + } +} + +func TestHarnessProviderChoiceKeyedWins(t *testing.T) { + isolateOpenCodeEnv(t) + t.Setenv("ANTHROPIC_API_KEY", "sk-test") + c := HarnessProviderChoice() + if c.Kind != "anthropic" { + t.Fatalf("kind = %s", c.Kind) + } + if c.Disclosure != "" { + t.Fatalf("keyed provider should have no zen disclosure, got %q", c.Disclosure) + } + if !strings.Contains(c.Model, "anthropic/") { + t.Fatalf("model = %s", c.Model) + } +} + +func TestApplyHarnessProviderConfigWritesModel(t *testing.T) { + isolateOpenCodeEnv(t) + if err := ApplyHarnessProviderConfig(); err != nil { + t.Fatal(err) + } + path := filepath.Join(opencodeConfigDir(), "opencode.json") + body, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(body), ZenFreeModel) { + t.Fatalf("missing model in %s", body) + } + if err := setJSONPath(path, "model", "keep/me"); err != nil { + t.Fatal(err) + } + if err := ApplyHarnessProviderConfig(); err != nil { + t.Fatal(err) + } + body, _ = os.ReadFile(path) + if !strings.Contains(string(body), "keep/me") { + t.Fatalf("should not clobber existing model: %s", body) + } +} + +func TestDoctorFreeModelUnavailable(t *testing.T) { + isolateOpenCodeEnv(t) + if err := SaveProfile(Profile{Agent: "opencode", HintInErrors: true}); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + t.Cleanup(srv.Close) + prev := zenModelsURL + zenModelsURL = srv.URL + t.Cleanup(func() { zenModelsURL = prev }) + + r, err := Doctor(context.Background()) + if err != nil { + t.Fatal(err) + } + var found bool + for _, f := range r.Findings { + if f.ID == "opencode.zen" { + found = true + if f.OK { + t.Fatalf("want fail, got %+v", f) + } + if !strings.Contains(f.Summary, "free model unavailable") { + t.Fatalf("summary = %q", f.Summary) + } + } + } + if !found { + t.Fatalf("missing opencode.zen finding: %+v", r.Findings) + } + if r.OK { + t.Fatal("doctor should fail when free model is unavailable") + } +} + +func TestDoctorFreeModelAvailable(t *testing.T) { + isolateOpenCodeEnv(t) + if err := SaveProfile(Profile{Agent: "opencode", HintInErrors: true}); err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`{"data":[{"id":"big-pickle"}]}`)) + })) + t.Cleanup(srv.Close) + prev := zenModelsURL + zenModelsURL = srv.URL + t.Cleanup(func() { zenModelsURL = prev }) + + r, err := Doctor(context.Background()) + if err != nil { + t.Fatal(err) + } + var found bool + for _, f := range r.Findings { + if f.ID == "opencode.zen" { + found = true + if !f.OK { + t.Fatalf("want ok, got %+v", f) + } + } + } + if !found { + t.Fatalf("missing opencode.zen finding: %+v", r.Findings) + } +} + +func TestRankedCLIAgentsIncludesBundledOpenCode(t *testing.T) { + isolateOpenCodeEnv(t) + ranked := RankedCLIAgents() + if len(ranked) == 0 { + t.Fatal("expected bundled opencode fallback") + } + last := ranked[len(ranked)-1] + if !last.Bundled || last.Name != "opencode" { + t.Fatalf("want bundled opencode last, got %+v", last) + } + label := agentAuthLabel(last) + if !strings.Contains(label, ZenFreeDisclosure) { + t.Fatalf("bundled label missing disclosure: %s", label) + } + if !strings.Contains(harnessAgentDescription(ranked), "ANTHROPIC_API_KEY") { + t.Fatal("form description should mention keyed-provider alternative") + } +} + +func TestOpenCodeDownloadURLNeverLatest(t *testing.T) { + prevAVX, prevMusl := openCodeHasAVX2, openCodeIsMusl + t.Cleanup(func() { + openCodeHasAVX2, openCodeIsMusl = prevAVX, prevMusl + }) + openCodeHasAVX2 = func() bool { return true } + openCodeIsMusl = func() bool { return false } + openCodeTestDownloadURL = "" + u, err := openCodeDownloadURL(OpenCodeVersion) + if err != nil { + t.Fatal(err) + } + if strings.Contains(u, "/latest") { + t.Fatalf("must not use releases/latest: %s", u) + } + if !strings.Contains(u, "/v"+OpenCodeVersion+"/") { + t.Fatalf("missing pinned tag: %s", u) + } + if runtime.GOOS == "darwin" && runtime.GOARCH == "arm64" && !strings.HasSuffix(u, "opencode-darwin-arm64.zip") { + t.Fatalf("unexpected darwin arm64 url: %s", u) + } +} diff --git a/core/sling/assist/prompt.go b/core/sling/assist/prompt.go new file mode 100644 index 000000000..2ee93baeb --- /dev/null +++ b/core/sling/assist/prompt.go @@ -0,0 +1,945 @@ +package assist + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "text/template" + "time" + "unicode/utf8" + + "github.com/slingdata-io/sling-cli/core" + "github.com/slingdata-io/sling-cli/core/dbio/connection" + "github.com/slingdata-io/sling-cli/core/env" + "github.com/slingdata-io/sling-cli/core/sling/project" + "github.com/slingdata-io/sling-cli/core/sling/validate" + "github.com/spf13/cast" + "gopkg.in/yaml.v3" +) + +// maxPromptTokens is a soft cap for the assembled --out prompt (~4 chars/token). +const maxPromptTokens = 1500 + +// maxErrorExcerptLines caps the sanitized failure excerpt in Context. +const maxErrorExcerptLines = 15 + +const maxProbeConnections = 10 + +// canonicalFolders are the wave-5 project dirs scanned for file counts. +var canonicalFolders = []string{"replications", "pipelines", "models", "specs"} + +// ProbeConn is name/type/source from ConnEntries.List() (no payload fields). +type ProbeConn struct { + Name string + Type string + Source string +} + +// PromptContext is the assembled state for the five-section agent prompt. +type PromptContext struct { + Version string + Cwd string + ProjectName string + ProjectRoot string + HasProject bool + FileCounts map[string]int + Connections []ProbeConn + ConnectionExtra int + RecentRuns []LocalExec + TargetExec *LocalExec // set by --id: investigate this failure + PlatformExec *PlatformExec // set by --id when the exec is on the platform only + MCPWired bool + Signature string + Lookup *ErrorLookupResult + ErrorExcerpt string + RunLogExcerpt string + Ask string + Suggestions []string + Route string + Agent string +} + +type promptView struct { + Rules string + State string + Context string + Ask string + Objective string +} + +type promptBundle struct { + Skeleton string `yaml:"_skeleton"` + Rules string `yaml:"_rules"` + Objective string `yaml:"_objective"` +} + +func loadPromptBundle() (promptBundle, error) { + var b promptBundle + if err := yaml.Unmarshal(PromptsYAML, &b); err != nil { + return b, err + } + if strings.TrimSpace(b.Skeleton) == "" { + return b, fmt.Errorf("prompts.yaml missing _skeleton") + } + if strings.TrimSpace(b.Rules) == "" { + return b, fmt.Errorf("prompts.yaml missing _rules") + } + if strings.TrimSpace(b.Objective) == "" { + return b, fmt.Errorf("prompts.yaml missing _objective") + } + return b, nil +} + +// ProbeOptions is the user-facing input to Probe. +// IncludeFailure pulls signature/lookup/excerpt into Context. Set it only +// when the ask targets the failure (investigate pick, empty-ask fallback, +// landing card); other asks keep the one-line recent-runs summary. +type ProbeOptions struct { + Ask string + IncludeFailure bool + ExecID string // target this exec instead of the latest failure +} + +// Probe gathers local state for the agent prompt. +// It lists connections via ConnEntries.List() only (name/type/source). +func Probe(opts ProbeOptions) PromptContext { + cwd := workDir() + p := PromptContext{ + Version: core.Version, + Cwd: cwd, + Ask: strings.TrimSpace(opts.Ask), + FileCounts: map[string]int{}, + } + p.ProjectName, p.ProjectRoot, p.HasProject = detectProject(cwd) + scanRoot := cwd + if p.ProjectRoot != "" { + scanRoot = p.ProjectRoot + } + p.FileCounts = countCanonicalFiles(scanRoot) + p.Connections, p.ConnectionExtra = capProbeConnections(listProbeConnections()) + p.RecentRuns = recentRunsForPrompt() + p.MCPWired = probeMCPWired() + if prof, ok, _ := LoadProfile(); ok { + p.Agent = prof.Agent + } + if opts.ExecID != "" { + if le, ok := LookupLocalExec(opts.ExecID); ok { + p.TargetExec = &le + p.Signature, p.Lookup, p.ErrorExcerpt = failureDetails(le) + p.RunLogExcerpt = runLogExcerpt(le) + } + } else if opts.IncludeFailure { + if lead, ok := leadingFailure(p.RecentRuns); ok { + p.Signature, p.Lookup, p.ErrorExcerpt = failureDetails(lead) + p.RunLogExcerpt = runLogExcerpt(lead) + } + } + for _, s := range p.suggestions() { + p.Suggestions = append(p.Suggestions, s.Label) + } + p.Route = Route(p) + return p +} + +// Route picks the prompt ladder rung. Pure Go, no LLM. +// Order: ask > recent failed run > zero connections > no project in cwd > default. +// Free function: PromptContext already has a Route field. +func Route(p PromptContext) string { + if p.PlatformExec != nil { + return "platform_failed_run" + } + if p.TargetExec != nil { + return "failed_run" + } + if strings.TrimSpace(p.Ask) != "" { + return "ask" + } + if _, ok := leadingFailure(p.RecentRuns); ok { + return "failed_run" + } + if userConnectionCount(p.Connections) == 0 { + return "zero_connections" + } + if !p.HasProject { + return "no_project" + } + return "default" +} + +const ( + suggestionElseLabel = "Something else — just describe it" + maxSuggestionRows = 4 + maxSignalRows = 3 +) + +// suggestion is one open-screen row. Label is shown; Ask is the launch text. +// Investigate marks the failed-run row: only that pick puts failure details +// (signature, lookup, excerpt) into the prompt Context. +type suggestion struct { + Label string + Ask string // empty → free text ("something else") + Investigate bool +} + +func (p PromptContext) suggestions() []suggestion { + var signals []suggestion + if fail, ok := leadingFailure(p.RecentRuns); ok { + label := failedRunLabel(fail) + signals = append(signals, suggestion{Label: label, Ask: label, Investigate: true}) + } + if userConnectionCount(p.Connections) == 0 { + signals = append(signals, suggestion{ + Label: "Add your first connection", + Ask: "Add a connection", + }) + } + if !p.HasProject { + signals = append(signals, suggestion{ + Label: "Scaffold a project (`sling init`)", + Ask: "Scaffold a Sling project in this folder", + }) + } + out := signals + if len(out) < maxSignalRows { + out = append(out, suggestion{ + Label: "Create or update a replication / pipeline / model / API spec", + Ask: "Help me create or update a Sling config (replication, pipeline, model, or API spec). First ask me which one and which connections it involves.", + }) + } + if len(out) > maxSignalRows { + out = out[:maxSignalRows] + } + out = append(out, suggestion{Label: suggestionElseLabel}) + if len(out) > maxSuggestionRows { + out = append(out[:maxSuggestionRows-1], suggestion{Label: suggestionElseLabel}) + } + return out +} + +func failedRunLabel(r LocalExec) string { + idObj := strings.TrimSpace(r.ID + " " + r.displayObject()) + idObj = strings.Join(strings.Fields(idObj), " ") + s := "Investigate the failed run " + idObj + if r.When.IsZero() { + return s + } + rt := relTime(r.When) + if rt == "just now" { + return s + " (just now)" + } + return s + " (" + rt + " ago)" +} + +func userConnectionCount(conns []ProbeConn) int { + n := 0 + for _, c := range conns { + if c.Source == "built-in" { + continue + } + n++ + } + return n +} + +func leadingFailure(runs []LocalExec) (LocalExec, bool) { + for _, r := range runs { + if r.Status == "err" { + return r, true + } + } + return LocalExec{}, false +} + +func listProbeConnections() []ProbeConn { + entries := connection.GetLocalConns(true) + _, rows := entries.List() + out := make([]ProbeConn, 0, len(rows)) + for _, row := range rows { + if len(row) < 3 { + continue + } + out = append(out, ProbeConn{ + Name: cast.ToString(row[0]), + Type: cast.ToString(row[1]), + Source: cast.ToString(row[2]), + }) + } + sort.Slice(out, func(i, j int) bool { + si, sj := connSourceRank(out[i].Source), connSourceRank(out[j].Source) + if si != sj { + return si < sj + } + return out[i].Name < out[j].Name + }) + return out +} + +func connSourceRank(source string) int { + switch source { + case "sling env yaml": + return 0 + case "built-in": + return 1 + default: + return 2 + } +} + +func capProbeConnections(all []ProbeConn) ([]ProbeConn, int) { + if len(all) <= maxProbeConnections { + return all, 0 + } + return all[:maxProbeConnections], len(all) - maxProbeConnections +} + +func detectProject(cwd string) (name, root string, has bool) { + if cwd == "" { + return "", "", false + } + if r, err := project.FindRoot(cwd); err == nil && r != "" { + m, err := project.Load(r) + n := filepath.Base(r) + if err == nil && strings.TrimSpace(m.Name) != "" { + n = m.Name + } + return n, r, true + } + if hasCanonicalFolders(cwd) { + return filepath.Base(cwd), cwd, true + } + return "", cwd, false +} + +func hasCanonicalFolders(dir string) bool { + if project.HasManifest(dir) { + return true + } + for _, n := range canonicalFolders { + if _, err := os.Stat(filepath.Join(dir, n)); err == nil { + return true + } + } + return false +} + +func countCanonicalFiles(root string) map[string]int { + out := map[string]int{} + for _, folder := range canonicalFolders { + out[folder] = scanCanonicalFolder(root, folder) + } + return out +} + +func scanCanonicalFolder(root, folder string) int { + dir := filepath.Join(root, folder) + n := 0 + _ = filepath.WalkDir(dir, func(p string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil + } + ext := strings.ToLower(filepath.Ext(p)) + if ext != ".yaml" && ext != ".yml" && ext != ".sql" { + return nil + } + body, rerr := os.ReadFile(p) + if rerr != nil { + return nil + } + if validate.DetectFileKind(body, p) != validate.KindUnknown { + n++ + } + return nil + }) + return n +} + +func recentRunsForPrompt() []LocalExec { + execs, err := ListLocalExecs() + if err != nil || len(execs) == 0 { + return nil + } + sort.SliceStable(execs, func(i, j int) bool { + iFail := execs[i].Status == "err" + jFail := execs[j].Status == "err" + if iFail != jFail { + return iFail + } + return execs[i].When.After(execs[j].When) + }) + if len(execs) > 3 { + execs = execs[:3] + } + return execs +} + +func probeMCPWired() bool { + report, err := Doctor(context.Background()) + if err != nil || report == nil || report.Matrix == nil { + return false + } + for _, row := range report.Matrix.Rows { + if row.Label != "MCP" { + continue + } + for _, st := range row.Cells { + if st == CellOK { + return true + } + } + } + return false +} + +// maxRunLogLines caps the run-log tail that enters the prompt. +const maxRunLogLines = 40 + +// runLogExcerpt returns the tail of the captured run log for this exec. +// Empty when nothing was captured or it only mirrors error.txt. +func runLogExcerpt(le LocalExec) string { + b, err := os.ReadFile(filepath.Join(le.LogDir, "stderr.log")) + if err != nil { + return "" + } + errB, _ := os.ReadFile(filepath.Join(le.LogDir, "error.txt")) + if string(b) == string(errB) { + return "" // pre-capture snapshot: stderr.log duplicates error.txt + } + return capExcerptLines(sanitizeLogForPrompt(string(b), 0), maxRunLogLines) +} + +func failureDetails(le LocalExec) (sig string, lookup *ErrorLookupResult, excerpt string) { + metaPath := filepath.Join(le.LogDir, "meta.json") + if b, err := os.ReadFile(metaPath); err == nil { + doc := map[string]any{} + if json.Unmarshal(b, &doc) == nil { + sig = cast.ToString(doc["error_signature"]) + if sig == "" { + sig = cast.ToString(doc["error_pattern_id"]) + } + } + } + if sig != "" { + if r, err := LookupError(sig); err == nil { + lookup = &r + } + } + errPath := filepath.Join(le.LogDir, "error.txt") + if b, err := os.ReadFile(errPath); err == nil { + excerpt = capExcerptLines(sanitizeLogForPrompt(string(b), 0), maxErrorExcerptLines) + } + return sig, lookup, excerpt +} + +func capExcerptLines(s string, n int) string { + s = strings.TrimRight(s, "\n") + lines := strings.Split(s, "\n") + if len(lines) <= n { + return s + } + return strings.Join(lines[len(lines)-n:], "\n") +} + +// Render fills the five-section skeleton. Caps at ~maxPromptTokens. +func (p PromptContext) Render() (string, error) { + bundle, err := loadPromptBundle() + if err != nil { + return "", err + } + p = p.shrinkContextForBudget() + view := promptView{ + Rules: strings.TrimSpace(bundle.Rules), + State: p.renderState(), + Context: p.renderContext(), + Ask: p.renderAsk(), + Objective: p.renderObjective(bundle), + } + tmpl, err := template.New("skeleton").Parse(bundle.Skeleton) + if err != nil { + return "", err + } + var buf bytes.Buffer + if err := tmpl.Execute(&buf, view); err != nil { + return "", err + } + out := strings.TrimRight(buf.String(), "\n") + "\n" + return capPromptTokens(out), nil +} + +// RenderPrompt is the exported wrapper for (PromptContext).Render. +func RenderPrompt(p PromptContext) (string, error) { + return p.Render() +} + +func (p PromptContext) renderState() string { + var b strings.Builder + fmt.Fprintf(&b, "- Sling version: %s\n", p.Version) + fmt.Fprintf(&b, "- cwd: %s\n", p.Cwd) + switch { + case p.HasProject && p.ProjectName != "": + fmt.Fprintf(&b, "- project: %s (%s)\n", p.ProjectName, p.ProjectRoot) + case p.HasProject: + fmt.Fprintf(&b, "- project: %s\n", p.ProjectRoot) + default: + b.WriteString("- project: (none in cwd)\n") + } + if p.MCPWired { + b.WriteString("- MCP wired: yes\n") + } else { + b.WriteString("- MCP wired: no\n") + } + if p.Agent != "" { + fmt.Fprintf(&b, "- preferred agent: %s\n", p.Agent) + } + return strings.TrimRight(b.String(), "\n") +} + +func (p PromptContext) renderContext() string { + var b strings.Builder + b.WriteString("- connections: ") + b.WriteString(formatProbeConnections(p.Connections, p.ConnectionExtra)) + b.WriteByte('\n') + b.WriteString("- files: ") + b.WriteString(formatFileCounts(p.FileCounts)) + b.WriteByte('\n') + b.WriteString("- recent runs: ") + b.WriteString(formatRecentRuns(p.RecentRuns)) + b.WriteByte('\n') + if p.PlatformExec != nil { + pe := p.PlatformExec + fmt.Fprintf(&b, "- platform exec: %s [%s] type=%s job=%s file=%s host=%s\n", + pe.ExecID, pe.Status, pe.Type, pe.JobName, pe.FileName, pe.HostLabel) + if strings.TrimSpace(pe.ErrSummary) != "" { + fmt.Fprintf(&b, "- platform error: %s\n", pe.ErrSummary) + } + } + if p.Signature != "" { + fmt.Fprintf(&b, "- error signature: %s\n", p.Signature) + if p.Lookup != nil && p.Lookup.Title != "" { + fmt.Fprintf(&b, "- error lookup: %s (%s)\n", p.Lookup.Title, p.Lookup.Status) + } + } + if strings.TrimSpace(p.ErrorExcerpt) != "" { + b.WriteString("- error excerpt:\n") + for _, line := range strings.Split(p.ErrorExcerpt, "\n") { + fmt.Fprintf(&b, " %s\n", line) + } + } + if strings.TrimSpace(p.RunLogExcerpt) != "" { + b.WriteString("- run log (tail):\n") + for _, line := range strings.Split(p.RunLogExcerpt, "\n") { + fmt.Fprintf(&b, " %s\n", line) + } + } + if strings.TrimSpace(p.Ask) == "" && len(p.Suggestions) > 0 { + b.WriteString("- suggestions: ") + b.WriteString(strings.Join(p.Suggestions, "; ")) + b.WriteByte('\n') + } + return strings.TrimRight(b.String(), "\n") +} + +func formatProbeConnections(conns []ProbeConn, extra int) string { + if len(conns) == 0 { + return "(none — run `sling conns set --type ` with ${VAR} refs; never ask for credentials in chat)" + } + parts := make([]string, 0, len(conns)) + for _, c := range conns { + if c.Type != "" { + parts = append(parts, fmt.Sprintf("%s (%s)", c.Name, c.Type)) + } else { + parts = append(parts, c.Name) + } + } + s := strings.Join(parts, ", ") + if extra > 0 { + s += fmt.Sprintf(" and %d more — run `sling conns list`", extra) + } + return s +} + +func formatFileCounts(counts map[string]int) string { + if len(counts) == 0 { + return "(none)" + } + parts := make([]string, 0, len(canonicalFolders)) + total := 0 + for _, k := range canonicalFolders { + n := counts[k] + total += n + parts = append(parts, fmt.Sprintf("%s %d", k, n)) + } + if total == 0 { + return "(none)" + } + return strings.Join(parts, ", ") +} + +func formatRecentRuns(runs []LocalExec) string { + if len(runs) == 0 { + return "(none)" + } + parts := make([]string, 0, len(runs)) + for _, r := range runs { + label := r.ID + if obj := r.displayObject(); obj != "" { + label = r.ID + " " + obj + } + parts = append(parts, fmt.Sprintf("%s [%s]", label, r.Status)) + } + return strings.Join(parts, "; ") +} + +func (p PromptContext) renderAsk() string { + if strings.TrimSpace(p.Ask) != "" { + return p.Ask + } + if p.PlatformExec != nil { + return fmt.Sprintf("Investigate platform execution %s (%s)", p.PlatformExec.ExecID, p.PlatformExec.Status) + } + if p.TargetExec != nil { + return failedRunLabel(*p.TargetExec) + } + return "(none — ask the user what they want; the suggestions are in Context)" +} + +func (p PromptContext) renderObjective(bundle promptBundle) string { + return strings.TrimSpace(bundle.Objective) +} + +func estimateTokens(s string) int { + n := len([]rune(s)) + return (n + 3) / 4 +} + +const truncNote = "[...truncated to ~1500 tokens...]" + +func (p PromptContext) shrinkContextForBudget() PromptContext { + p.ErrorExcerpt = capExcerptLines(p.ErrorExcerpt, maxErrorExcerptLines) + p.RunLogExcerpt = capExcerptLines(p.RunLogExcerpt, maxRunLogLines) + for n := maxErrorExcerptLines; n > 3 && estimateTokens(p.renderContext()) > maxPromptTokens/2; n -= 4 { + p.ErrorExcerpt = capExcerptLines(p.ErrorExcerpt, n) + p.RunLogExcerpt = capExcerptLines(p.RunLogExcerpt, n) + } + if estimateTokens(p.renderContext()) > maxPromptTokens/2 && len(p.Connections) > 3 { + extra := len(p.Connections) - 3 + p.Connections = p.Connections[:3] + p.ConnectionExtra += extra + } + if estimateTokens(p.renderContext()) > maxPromptTokens/2 && len(p.Suggestions) > 1 { + p.Suggestions = p.Suggestions[len(p.Suggestions)-1:] + } + return p +} + +func capPromptTokens(s string) string { + if estimateTokens(s) <= maxPromptTokens { + return s + } + const ctxH = "# Context\n" + const askH = "\n# Ask\n" + ctxAt := strings.Index(s, ctxH) + askAt := strings.Index(s, askH) + if ctxAt < 0 || askAt <= ctxAt { + return s + } + prefix := s[:ctxAt+len(ctxH)] + ctxBody := s[ctxAt+len(ctxH) : askAt] + suffix := s[askAt:] + budget := maxPromptTokens * 4 + keep := budget - len([]rune(prefix)) - len([]rune(suffix)) - len([]rune(truncNote)) - 2 + if keep < 80 { + return prefix + truncNote + "\n" + suffix + } + runes := []rune(ctxBody) + if len(runes) <= keep { + return s + } + cut := keep + for cut > 0 && runes[cut-1] != '\n' { + cut-- + } + if cut < keep/2 { + cut = keep + } + return prefix + string(runes[:cut]) + "\n" + truncNote + "\n" + suffix +} + +// LandingKind is the bare-`sling` TTY card. +type LandingKind string + +const ( + LandingFresh LandingKind = "fresh" + LandingNoProject LandingKind = "no_project" + LandingProject LandingKind = "project" +) + +// IsFreshInstall is true when env.yaml is the seeded default and there is +// no assist history (sessions or failure snapshots). Does not create dirs. +func IsFreshInstall() bool { + home := slingHome() + // Any session or error snapshot under ~/.sling/assist means not fresh. + if home != "" { + for _, rel := range []string{ + filepath.Join("assist", "history"), + filepath.Join("assist", "errors"), + } { + entries, err := os.ReadDir(filepath.Join(home, rel)) + if err != nil { + continue + } + for _, e := range entries { + if e.IsDir() && !strings.HasPrefix(e.Name(), ".") { + return false + } + } + } + } + path := envFilePath() + // Missing env.yaml is fresh. A user connection or assist profile is not. + if path == "" { + return true + } + if _, err := os.Stat(path); err != nil { + return true + } + ef := env.LoadEnvFile(path) + if len(ef.Connections) > 0 { + return false + } + if _, ok := ef.Env[assistEnvKey]; ok { + return false + } + for _, v := range ef.Env { + if v == nil { + continue + } + if strings.TrimSpace(fmt.Sprint(v)) != "" { + return false + } + } + return true +} + +// ClassifyLanding picks the card. Fresh wins so a default home never +// falls through to the probe states. +func ClassifyLanding(p PromptContext, fresh bool) LandingKind { + if fresh { + return LandingFresh + } + if p.HasProject { + return LandingProject + } + return LandingNoProject +} + +// SuggestedCommand is the wave-7 ladder mapped to one runnable command. +func (p PromptContext) SuggestedCommand() string { + switch Route(p) { + case "failed_run": + if sig := strings.TrimSpace(p.Signature); sig != "" { + return "sling assist error " + sig + } + return "sling assist" + case "zero_connections": + return "sling assist" + case "no_project": + return "sling init" + default: + return "sling assist" + } +} + +// RenderLanding prints the TTY card for one probe state. width 0 → 80. +func RenderLanding(kind LandingKind, p PromptContext, width int) string { + if width <= 0 { + width = 80 + } + var body string + switch kind { + case LandingFresh: + // All three steps are `sling assist` so the card never points at `conns set`. + body = `Welcome to sling. Three steps to your first data flow: + + 1. Set up your agent sling assist + 2. Add a connection sling assist (or edit ~/.sling/env.yaml) + 3. Move some data sling assist + +Docs: https://docs.slingdata.io +` + case LandingNoProject: + // Conn count plus a pointer at `sling init`. + n := userConnectionCount(p.Connections) + noun := "connections" + if n == 1 { + noun = "connection" + } + body = fmt.Sprintf("%d %s configured (`sling conns list`). No project here — `sling init` scaffolds one.\n", n, noun) + default: + // Project card: name, linked?, file counts, last run, next command. + name := strings.TrimSpace(p.ProjectName) + if name == "" { + if p.ProjectRoot != "" { + name = filepath.Base(p.ProjectRoot) + } else { + name = "(unnamed)" + } + } + linked := "not linked" + if strings.TrimSpace(p.ProjectRoot) != "" { + // Linked when sling_project.yml (or .sling.json) has a project id. + if m, err := project.Load(p.ProjectRoot); err == nil && m.Linked() { + linked = "linked" + } + } + var b strings.Builder + fmt.Fprintf(&b, "On project %s (%s)\n", name, linked) + fmt.Fprintf(&b, " files: %s\n", formatFileCounts(p.FileCounts)) + if run, ok := latestRun(p.RecentRuns); ok { + label := run.ID + if obj := run.displayObject(); obj != "" { + label += " " + obj + } + s := fmt.Sprintf("%s [%s]", label, run.Status) + if !run.When.IsZero() { + rt := relTime(run.When) + if rt == "just now" { + s += " just now" + } else { + s += " " + rt + " ago" + } + } + fmt.Fprintf(&b, " last run: %s\n", s) + } else { + b.WriteString(" last run: none\n") + } + fmt.Fprintf(&b, "\n Next: %s\n", p.SuggestedCommand()) + body = b.String() + } + return wrapToWidth(strings.TrimRight(body, "\n")+"\n", width) +} + +// relTime is a short age phrase: "just now", "3m", "2h", "1d". +func relTime(t time.Time) string { + d := time.Since(t) + if d < 0 { + d = 0 + } + switch { + case d < time.Minute: + return "just now" + case d < time.Hour: + return fmt.Sprintf("%dm", int(d.Minutes())) + case d < 24*time.Hour: + return fmt.Sprintf("%dh", int(d.Hours())) + default: + return fmt.Sprintf("%dd", int(d.Hours()/24)) + } +} + +func latestRun(runs []LocalExec) (LocalExec, bool) { + var best LocalExec + ok := false + for _, r := range runs { + if !ok || r.When.After(best.When) { + best, ok = r, true + } + } + return best, ok +} + +func wrapToWidth(s string, width int) string { + if width <= 0 { + width = 80 + } + // Keep `backtick commands` as one token so a wrapped card stays copyable. + leadingSpaces := func(line string) string { + i := 0 + for i < len(line) && line[i] == ' ' { + i++ + } + return line[:i] + } + wrapTokens := func(in string) []string { + var words []string + var cur strings.Builder + inTick := false + for _, r := range in { + switch { + case r == '`': + inTick = !inTick + cur.WriteRune(r) + case r == ' ' && !inTick: + if cur.Len() > 0 { + words = append(words, cur.String()) + cur.Reset() + } + default: + cur.WriteRune(r) + } + } + if cur.Len() > 0 { + words = append(words, cur.String()) + } + return words + } + chunkRunes := func(in string, w int) []string { + runes := []rune(in) + var out []string + for len(runes) > w { + out = append(out, string(runes[:w])) + runes = runes[w:] + } + if len(runes) > 0 { + out = append(out, string(runes)) + } + return out + } + wrapLine := func(line string) []string { + if utf8.RuneCountInString(line) <= width { + return []string{line} + } + indent := leadingSpaces(line) + body := strings.TrimLeft(line, " ") + avail := width - len(indent) + if avail < 20 { + avail = 20 + indent = "" + } + words := wrapTokens(body) + if len(words) == 0 { + return []string{line} + } + var lines []string + cur := indent + words[0] + for _, w := range words[1:] { + trial := cur + " " + w + if utf8.RuneCountInString(trial) <= width { + cur = trial + continue + } + lines = append(lines, cur) + cur = indent + w + if utf8.RuneCountInString(cur) > width { + lines = append(lines, chunkRunes(cur, width)...) + cur = indent + } + } + if strings.TrimSpace(cur) != "" { + lines = append(lines, cur) + } + return lines + } + + lines := strings.Split(strings.TrimRight(s, "\n"), "\n") + out := make([]string, 0, len(lines)) + for _, line := range lines { + out = append(out, wrapLine(line)...) + } + return strings.Join(out, "\n") + "\n" +} diff --git a/core/sling/assist/prompt_test.go b/core/sling/assist/prompt_test.go new file mode 100644 index 000000000..cab47ba4c --- /dev/null +++ b/core/sling/assist/prompt_test.go @@ -0,0 +1,1036 @@ +package assist + +import ( + "bytes" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/slingdata-io/sling-cli/core/dbio/iop" +) + +func sampleLookup() *ErrorLookupResult { + r, err := LookupError("97d84811") + if err != nil { + return &ErrorLookupResult{Signature: "97d84811", Status: "unknown", Title: "No published guidance yet"} + } + return &r +} + +func snapshotCtx(route string) PromptContext { + base := PromptContext{ + Version: "dev", + Cwd: "/work", + ProjectName: "demo", + ProjectRoot: "/work", + HasProject: true, + FileCounts: map[string]int{"replications": 1, "pipelines": 0, "models": 0, "specs": 0}, + Connections: []ProbeConn{ + {Name: "MY_PG", Type: "PostgreSQL", Source: "sling env yaml"}, + {Name: "MY_SF", Type: "Snowflake", Source: "sling env yaml"}, + }, + MCPWired: true, + Agent: "claude", + Route: route, + } + switch route { + case "ask": + base.Ask = "backfill orders" + case "failed_run": + base.Signature = "97d84811" + base.Lookup = sampleLookup() + base.ErrorExcerpt = "column missing: email_verified" + base.RecentRuns = []LocalExec{{ID: "exec_fail1", Status: "err", ConfigPath: "./r.yaml"}} + case "zero_connections": + base.Connections = nil + base.HasProject = true + case "no_project": + base.HasProject = false + base.ProjectName = "" + base.ProjectRoot = "/tmp" + base.FileCounts = map[string]int{"replications": 0, "pipelines": 0, "models": 0, "specs": 0} + case "default": + base.Ask = "" + } + for _, s := range base.suggestions() { + base.Suggestions = append(base.Suggestions, s.Label) + } + return base +} + +func TestPrintSnapshots(t *testing.T) { + rungs := []string{"ask", "failed_run", "zero_connections", "no_project", "default"} + for _, rung := range rungs { + t.Run(rung, func(t *testing.T) { + got, err := RenderPrompt(snapshotCtx(rung)) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(got, "# Rules") || !strings.Contains(got, "# State") || !strings.Contains(got, "# Context") { + t.Fatalf("missing section headings:\n%s", got) + } + if strings.Contains(got, "super-secret") || strings.Contains(got, "password") { + t.Fatalf("prompt leaked a secret-shaped value:\n%s", got) + } + path := filepath.Join("testdata", "print_"+rung+".golden") + if os.Getenv("UPDATE_GOLDEN") == "1" { + if err := os.MkdirAll("testdata", 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(got), 0o644); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read golden %s: %v (set UPDATE_GOLDEN=1 to write)", path, err) + } + if got != string(want) { + t.Errorf("golden mismatch for %s\n--- got ---\n%s\n--- want ---\n%s", rung, got, want) + } + }) + } +} + +func TestRouteLadder(t *testing.T) { + plat := PromptContext{PlatformExec: &PlatformExec{ExecID: "x"}, Ask: "do it"} + if Route(plat) != "platform_failed_run" { + t.Fatalf("platform rung lost: %s", Route(plat)) + } + ask := PromptContext{Ask: "do it", RecentRuns: []LocalExec{{Status: "err"}}} + if Route(ask) != "ask" { + t.Fatalf("ask rung lost to failure: %s", Route(ask)) + } + fail := PromptContext{RecentRuns: []LocalExec{{Status: "err"}}} + if Route(fail) != "failed_run" { + t.Fatalf("got %s", Route(fail)) + } + zero := PromptContext{Connections: []ProbeConn{{Name: "LOCAL", Source: "built-in"}}, HasProject: true} + if Route(zero) != "zero_connections" { + t.Fatalf("got %s", Route(zero)) + } + noProj := PromptContext{Connections: []ProbeConn{{Name: "MY_PG", Source: "sling env yaml"}}} + if Route(noProj) != "no_project" { + t.Fatalf("got %s", Route(noProj)) + } + def := PromptContext{ + HasProject: true, + Connections: []ProbeConn{{Name: "MY_PG", Source: "sling env yaml"}}, + } + if Route(def) != "default" { + t.Fatalf("got %s", Route(def)) + } +} + +func TestProbeListsNamesNotSecrets(t *testing.T) { + dir := withTempHomeDir(t) + envPath := filepath.Join(dir, "env.yaml") + body := "connections:\n MY_PG:\n type: postgres\n url: postgresql://user:super-secret-pass@localhost/db\n" + if err := os.WriteFile(envPath, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + p := Probe(ProbeOptions{Ask: "sync users"}) + out, err := RenderPrompt(p) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "# Rules") || !strings.Contains(out, "# State") || !strings.Contains(out, "# Context") { + t.Fatalf("missing headings:\n%s", out) + } + if !strings.Contains(out, "MY_PG") { + t.Fatalf("expected connection name MY_PG:\n%s", out) + } + if strings.Contains(out, "super-secret-pass") { + t.Fatalf("secret leaked into prompt:\n%s", out) + } + if strings.Contains(out, "postgresql://") { + t.Fatalf("connection URL leaked into prompt:\n%s", out) + } +} + +func TestFailureDetailsGatedByIntent(t *testing.T) { + withTempHomeDir(t) + WriteFailureSnapshot(FailureSnapshot{ + ExecID: "exec_gate1", + ErrMsg: "column missing: email_verified", + ConfigPath: "./r.yaml", + }) + + // A normal ask keeps the one-line run summary only. + p := Probe(ProbeOptions{Ask: "create a replication"}) + out, err := RenderPrompt(p) + if err != nil { + t.Fatal(err) + } + if p.Signature != "" || p.ErrorExcerpt != "" { + t.Fatalf("failure details leaked into ask mode: sig=%q excerpt=%q", p.Signature, p.ErrorExcerpt) + } + if strings.Contains(out, "error excerpt") || strings.Contains(out, "error signature") { + t.Fatalf("ask-mode prompt contains failure details:\n%s", out) + } + if !strings.Contains(out, "exec_gate1") || !strings.Contains(out, "[err]") { + t.Fatalf("ask-mode prompt lost the recent-runs summary:\n%s", out) + } + + // The investigate intent pulls the full details in. + p = Probe(ProbeOptions{Ask: "Investigate the failed run exec_gate1 ./r.yaml", IncludeFailure: true}) + out, err = RenderPrompt(p) + if err != nil { + t.Fatal(err) + } + if p.Signature == "" { + t.Fatal("investigate mode missing signature") + } + if !strings.Contains(out, "error excerpt") || !strings.Contains(out, "column missing: email_verified") { + t.Fatalf("investigate prompt missing failure details:\n%s", out) + } +} + +func TestPromptAssemblyDoesNotCallExpandEnvVars(t *testing.T) { + files := []string{"prompt.go", "session.go", "assist_platform..go"} + for _, name := range files { + b, err := os.ReadFile(name) + if err != nil { + // a rename must fail the guard, not skip it + t.Fatalf("cannot read %s: %s", name, err) + } + if bytes.Contains(b, []byte("ExpandEnvVars")) { + t.Errorf("%s must not call ExpandEnvVars", name) + } + if bytes.Contains(b, []byte(".Data")) && bytes.Contains(b, []byte("Connection")) { + t.Errorf("%s must not read connection payload fields", name) + } + } +} + +func TestConnectionCap(t *testing.T) { + all := make([]ProbeConn, 12) + for i := range all { + all[i] = ProbeConn{Name: "C" + strings.Repeat("X", 1), Type: "PostgreSQL"} + all[i].Name = "CONN_" + string(rune('A'+i)) + } + got, extra := capProbeConnections(all) + if len(got) != maxProbeConnections || extra != 2 { + t.Fatalf("len=%d extra=%d", len(got), extra) + } + s := formatProbeConnections(got, extra) + if !strings.Contains(s, "and 2 more — run `sling conns list`") { + t.Fatalf("cap suffix missing: %s", s) + } +} + +func TestErrorExcerptCaps15Lines(t *testing.T) { + var lines []string + for i := 0; i < 20; i++ { + lines = append(lines, "line") + } + got := capExcerptLines(strings.Join(lines, "\n"), maxErrorExcerptLines) + if n := strings.Count(got, "\n") + 1; n != 15 { + t.Fatalf("lines=%d want 15", n) + } +} + +func TestNestedLaunchPrintsInsteadOfSpawn(t *testing.T) { + withTempHomeDir(t) + t.Setenv("CLAUDECODE", "1") + buf := &bytes.Buffer{} + prev := assistOut + assistOut = buf + defer func() { assistOut = prev }() + + out, err := Session(SessionOptions{Ask: "backfill orders"}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(buf.String(), "# Rules") { + t.Fatalf("stdout missing Rules:\n%s", buf.String()) + } + if !strings.Contains(out, "backfill orders") { + t.Fatalf("ask missing from prompt:\n%s", out) + } +} + +func TestAPISpecsSkillMentionsAgentBrowser(t *testing.T) { + b, err := SkillsFS.ReadFile("skills/sling-api-specs/SKILL.md") + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(b, []byte("agent-browser")) { + t.Fatal("sling-api-specs/SKILL.md must mention agent-browser") + } + if bytes.Contains(b, []byte("sling assist --browse")) { + t.Fatal("sling-api-specs/SKILL.md still mentions sling assist --browse") + } +} + +func TestPipelineSkillsTeachStateResult(t *testing.T) { + // Skills must teach the runtime shape, not the rejected one. + skill, err := SkillsFS.ReadFile("skills/sling-pipelines/SKILL.md") + if err != nil { + t.Fatal(err) + } + steps, err := SkillsFS.ReadFile("skills/sling-pipelines/STEPS.md") + if err != nil { + t.Fatal(err) + } + if !bytes.Contains(skill, []byte("state.count_query.result[0]")) { + t.Fatal("SKILL.md must teach state..result[0]") + } + if bytes.Contains(skill, []byte("state.count_query[0]")) { + t.Fatal("SKILL.md still teaches the rejected state.[0] shape") + } + if !bytes.Contains(steps, []byte("state.my_query.result[0]")) { + t.Fatal("STEPS.md must teach state..result[0]") + } + if bytes.Contains(steps, []byte("state.my_query[0]")) { + t.Fatal("STEPS.md still teaches the rejected state.[0] shape") + } +} + +func TestSkillsExpressionsParse(t *testing.T) { + // Every {…} expression in the skill bundle must pass the real parser. + eval := iop.NewEvaluator([]string{ + "env", "state", "secrets", "auth", "response", "request", "sync", + "context", "record", "queue", "source", "target", "stream", "object", + "timestamp", "store", "execution", "loop", "run", + }) + var failed []string + err := fs.WalkDir(SkillsFS, "skills", func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() { + return err + } + if !strings.HasSuffix(path, ".md") && !strings.HasSuffix(path, ".yaml") && !strings.HasSuffix(path, ".yml") { + return nil + } + body, err := SkillsFS.ReadFile(path) + if err != nil { + return err + } + text := string(body) + for _, span := range incorrectExampleSpans(text) { + text = text[:span[0]] + strings.Repeat(" ", span[1]-span[0]) + text[span[1]:] + } + for _, expr := range extractBraceExprs(text) { + if err := eval.Check(expr); err != nil { + failed = append(failed, path+": "+err.Error()) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if len(failed) > 0 { + t.Fatalf("skills teach expressions the runtime rejects:\n%s", strings.Join(failed, "\n")) + } +} + +func extractBraceExprs(s string) []string { + var out []string + for i := 0; i < len(s); i++ { + if s[i] != '{' { + continue + } + if i+1 < len(s) && (s[i+1] == '{' || s[i+1] == '#' || s[i+1] == '%') { + continue + } + depth := 1 + j := i + 1 + for j < len(s) && depth > 0 { + switch s[j] { + case '{': + depth++ + case '}': + depth-- + } + j++ + } + if depth != 0 { + continue + } + expr := strings.TrimSpace(s[i+1 : j-1]) + if !looksLikeRuntimeExpr(expr) { + i = j - 1 + continue + } + out = append(out, expr) + i = j - 1 + } + return out +} + +func incorrectExampleSpans(s string) [][2]int { + var out [][2]int + for _, marker := range []string{"# ❌", "❌ Incorrect", "Incorrect - single quotes"} { + i := 0 + for { + j := strings.Index(s[i:], marker) + if j < 0 { + break + } + start := i + j + end := strings.Index(s[start:], "\n```") + if end < 0 { + end = len(s) - start + } + out = append(out, [2]int{start, start + end}) + i = start + len(marker) + } + } + return out +} + +func looksLikeRuntimeExpr(expr string) bool { + expr = strings.TrimSpace(expr) + if expr == "" || strings.HasPrefix(expr, "#") || strings.HasPrefix(expr, "%") { + return false + } + // Python / JSON dicts in skill examples: {'a': 1}, {"id": 1} + if strings.HasPrefix(expr, "'") || (strings.HasPrefix(expr, `"`) && strings.Contains(expr, ":")) { + return false + } + if strings.ContainsAny(expr, "()") { + return true + } + if strings.Contains(expr, " + ") || strings.Contains(expr, " - ") || + strings.Contains(expr, " == ") || strings.Contains(expr, " != ") { + return true + } + return false +} + +func TestGatherFirstIntroDoNotAskWhenResolved(t *testing.T) { + names := []string{ + "sling-replications/SKILL.md", + "sling-pipelines/SKILL.md", + "sling-build/SKILL.md", + "sling-api-specs/SKILL.md", + "sling-connections/SKILL.md", + } + needle := []byte("If every row resolves, do not ask") + for _, name := range names { + b, err := SkillsFS.ReadFile("skills/" + name) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + if !bytes.Contains(b, needle) { + t.Errorf("%s missing Gather first intro", name) + } + } +} + +func TestSuggestionPriorityFullStack(t *testing.T) { + p := PromptContext{ + HasProject: false, + Connections: []ProbeConn{{Name: "LOCAL", Source: "built-in"}}, + RecentRuns: []LocalExec{{ID: "exec1", Status: "err", ConfigPath: "./r.yaml"}}, + } + got := p.suggestions() + if len(got) != 4 { + t.Fatalf("len=%d want 4: %+v", len(got), got) + } + if !strings.Contains(got[0].Label, "Investigate the failed run") { + t.Fatalf("first: %s", got[0].Label) + } + if got[1].Label != "Add your first connection" { + t.Fatalf("second: %s", got[1].Label) + } + if !strings.Contains(got[2].Label, "Scaffold a project") { + t.Fatalf("third: %s", got[2].Label) + } + if got[3].Label != suggestionElseLabel { + t.Fatalf("last: %s", got[3].Label) + } + for _, s := range got { + if strings.Contains(s.Label, "Create or update") { + t.Fatal("filler must drop when three signals fill the cap") + } + } +} + +func TestSuggestionFillerWhenSlotsRemain(t *testing.T) { + p := PromptContext{ + HasProject: true, + Connections: []ProbeConn{{Name: "MY_PG", Source: "sling env yaml"}}, + } + got := p.suggestions() + if len(got) != 2 { + t.Fatalf("len=%d want 2: %+v", len(got), got) + } + if !strings.Contains(got[0].Label, "Create or update") { + t.Fatalf("filler first: %s", got[0].Label) + } + if got[1].Label != suggestionElseLabel { + t.Fatalf("else last: %s", got[1].Label) + } +} + +func TestAskModeObjectiveHasGatherFirst(t *testing.T) { + out, err := RenderPrompt(snapshotCtx("ask")) + if err != nil { + t.Fatal(err) + } + if strings.Contains(out, " 1. ") { + t.Fatalf("ask mode must not print a menu:\n%s", out) + } + if !strings.Contains(out, "Gather first") { + t.Fatalf("Objective missing Gather first:\n%s", out) + } + if !strings.Contains(out, "Load the matching Sling skill") { + t.Fatalf("Objective missing skill rule:\n%s", out) + } + if strings.Contains(out, "- suggestions:") { + t.Fatalf("ask mode must not list suggestions:\n%s", out) + } +} + +func TestEmptyAskFallbackSuggestionsLine(t *testing.T) { + out, err := RenderPrompt(snapshotCtx("default")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "- suggestions:") { + t.Fatalf("empty-ask Context missing suggestions:\n%s", out) + } + if !strings.Contains(out, suggestionElseLabel) { + t.Fatalf("missing else row:\n%s", out) + } + if !strings.Contains(out, "ask the user what they want") { + t.Fatalf("missing empty-ask fallback:\n%s", out) + } +} + +func TestTruncationNeverCutsObjective(t *testing.T) { + excerpt := strings.Repeat("error line that is fairly long for the cap test\n", 80) + conns := make([]ProbeConn, 10) + for i := range conns { + conns[i] = ProbeConn{Name: "CONN_" + strings.Repeat("X", 40) + string(rune('A'+i)), Type: "PostgreSQL", Source: "sling env yaml"} + } + p := PromptContext{ + Version: "dev", + Cwd: "/work", + HasProject: true, + ProjectName: "demo", + ProjectRoot: "/work", + FileCounts: map[string]int{"replications": 1, "pipelines": 0, "models": 0, "specs": 0}, + Connections: conns, + ErrorExcerpt: excerpt, + Ask: "backfill orders", + Suggestions: []string{ + strings.Repeat("Investigate the failed run a-very-long-id ./r.yaml (3m ago)", 8), + suggestionElseLabel, + }, + } + out, err := RenderPrompt(p) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "# Objective") { + t.Fatalf("Objective heading cut:\n%s", out) + } + if !strings.Contains(out, "Gather first") { + t.Fatalf("Objective body cut:\n%s", out) + } + if !strings.Contains(out, "Load the matching Sling skill") { + t.Fatalf("Objective skill rule cut:\n%s", out) + } +} + +func TestSessionPrintTwiceStable(t *testing.T) { + withTempHomeDir(t) + opts := SessionOptions{Ask: "backfill orders", Print: true} + buf1, buf2 := &bytes.Buffer{}, &bytes.Buffer{} + prev := assistOut + defer func() { assistOut = prev }() + assistOut = buf1 + a, err := Session(opts) + if err != nil { + t.Fatal(err) + } + assistOut = buf2 + b, err := Session(opts) + if err != nil { + t.Fatal(err) + } + if a != b || buf1.String() != buf2.String() { + t.Fatalf("print not stable\n---1---\n%s\n---2---\n%s", buf1.String(), buf2.String()) + } + for _, heading := range []string{"# Rules", "# State", "# Context"} { + if !strings.Contains(a, heading) { + t.Fatalf("missing %s", heading) + } + } +} + +func TestRelTime(t *testing.T) { + now := time.Now() + cases := []struct { + t time.Time + want string + }{ + {now.Add(-10 * time.Second), "just now"}, + {now.Add(-3 * time.Minute), "3m"}, + {now.Add(-2 * time.Hour), "2h"}, + {now.Add(-25 * time.Hour), "1d"}, + } + for _, tc := range cases { + if got := relTime(tc.t); got != tc.want { + t.Errorf("relTime(%v)=%q want %q", tc.t, got, tc.want) + } + } +} + +func TestRenderOpenCardFailedRunRow(t *testing.T) { + p := PromptContext{ + HasProject: true, + ProjectName: "demo", + Connections: []ProbeConn{{Name: "MY_PG", Source: "sling env yaml"}}, + RecentRuns: []LocalExec{{ + ID: "exec_fail1", + Status: "err", + ConfigPath: "./r.yaml", + When: time.Now().Add(-3 * time.Minute), + }}, + } + got := renderOpenCard(p, 80) + if !strings.Contains(got, "On project demo") { + t.Fatalf("missing project line:\n%s", got) + } + if !strings.Contains(got, "Investigate the failed run exec_fail1 ./r.yaml (3m ago)") { + t.Fatalf("missing failed-run row:\n%s", got) + } + if !strings.Contains(got, "3m ago") { + t.Fatalf("missing rel-time:\n%s", got) + } + if !strings.Contains(got, suggestionElseLabel) { + t.Fatalf("missing else row:\n%s", got) + } +} + +func TestRenderOpenCardZeroConnRow(t *testing.T) { + p := PromptContext{ + HasProject: true, + ProjectName: "demo", + Connections: []ProbeConn{{Name: "LOCAL", Source: "built-in"}}, + } + got := renderOpenCard(p, 80) + if !strings.Contains(got, "Add your first connection") { + t.Fatalf("missing zero-conn row:\n%s", got) + } + if !strings.Contains(got, "0 connections") { + t.Fatalf("missing conn count:\n%s", got) + } +} + +func TestResolveOpenPick(t *testing.T) { + p := PromptContext{ + HasProject: false, + Connections: []ProbeConn{{Name: "LOCAL", Source: "built-in"}}, + RecentRuns: []LocalExec{{ID: "exec1", Status: "err", ConfigPath: "./r.yaml"}}, + } + opts := p.suggestions() + var buf bytes.Buffer + ask, investigate, ok := readOpenAsk(strings.NewReader("1\n"), &buf, opts) + if !ok { + t.Fatal("pick 1 aborted") + } + if !strings.Contains(ask, "Investigate the failed run") { + t.Fatalf("ask=%q", ask) + } + if !investigate { + t.Fatal("investigate pick must set the investigate flag") + } + + ask, investigate, ok = readOpenAsk(strings.NewReader("2\n"), &buf, opts) + if !ok || ask != "Add a connection" || investigate { + t.Fatalf("pick 2: ok=%v ask=%q investigate=%v", ok, ask, investigate) + } + + ask, investigate, ok = readOpenAsk(strings.NewReader("3\n"), &buf, opts) + if !ok || ask != "Scaffold a Sling project in this folder" || investigate { + t.Fatalf("pick 3: ok=%v ask=%q investigate=%v", ok, ask, investigate) + } + + ask, investigate, ok = readOpenAsk(strings.NewReader("backfill orders\n"), &buf, opts) + if !ok || ask != "backfill orders" || investigate { + t.Fatalf("free text: ok=%v ask=%q investigate=%v", ok, ask, investigate) + } +} + +func TestReadOpenAskEmptyLineExit(t *testing.T) { + opts := []suggestion{{Label: suggestionElseLabel}} + var buf bytes.Buffer + ask, _, ok := readOpenAsk(strings.NewReader("\n\n"), &buf, opts) + if ok || ask != "" { + t.Fatalf("ok=%v ask=%q", ok, ask) + } + if !strings.Contains(buf.String(), `sling assist ""`) { + t.Fatalf("missing hint:\n%s", buf.String()) + } +} + +func TestSessionOpenScreenDoesNotLaunchBeforePick(t *testing.T) { + withTempHomeDir(t) + prevTTY, prevIn, prevOut := ttyCheck, assistIn, assistOut + t.Cleanup(func() { + ttyCheck = prevTTY + assistIn = prevIn + assistOut = prevOut + }) + ttyCheck = func(*os.File) bool { return true } + assistIn = strings.NewReader("\n\n") + buf := &bytes.Buffer{} + assistOut = buf + + id, err := Session(SessionOptions{}) + if err != nil { + t.Fatalf("empty abort should exit 0: %v", err) + } + if id != "" { + t.Fatalf("launched session %q", id) + } + if !strings.Contains(buf.String(), suggestionElseLabel) { + t.Fatalf("card missing:\n%s", buf.String()) + } + if !strings.Contains(buf.String(), `sling assist ""`) { + t.Fatalf("missing hint:\n%s", buf.String()) + } +} + +func TestSessionHeadlessNoAskErrors(t *testing.T) { + withTempHomeDir(t) + _, err := Session(SessionOptions{Headless: true}) + if err == nil { + t.Fatal("expected error") + } + if !strings.Contains(err.Error(), "no ask given") { + t.Fatalf("got %v", err) + } +} + +func TestSessionModeAskWrittenToMeta(t *testing.T) { + dir := withTempHomeDir(t) + if err := os.MkdirAll(dir+"/bin", 0o755); err != nil { + t.Fatal(err) + } + stub := dir + "/bin/claude" + if err := os.WriteFile(stub, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+"/bin"+string(os.PathListSeparator)+os.Getenv("PATH")) + if err := SaveProfile(Profile{Agent: "claude"}); err != nil { + t.Fatal(err) + } + + id, err := Session(SessionOptions{Ask: "backfill orders", Headless: true, Agent: "claude"}) + if err != nil { + t.Fatal(err) + } + e, err := LoadEntry(id) + if err != nil { + t.Fatal(err) + } + if e.Meta.Task != modeAsk { + t.Fatalf("Meta.Task=%q want %q", e.Meta.Task, modeAsk) + } + if e.Answers.Task != modeAsk { + t.Fatalf("Answers.Task=%q want %q", e.Answers.Task, modeAsk) + } +} + +func TestSessionOpenModeWrittenToMeta(t *testing.T) { + dir := withTempHomeDir(t) + if err := os.MkdirAll(dir+"/bin", 0o755); err != nil { + t.Fatal(err) + } + stub := dir + "/bin/claude" + if err := os.WriteFile(stub, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir+"/bin"+string(os.PathListSeparator)+os.Getenv("PATH")) + if err := SaveProfile(Profile{Agent: "claude"}); err != nil { + t.Fatal(err) + } + + prevTTY, prevIn, prevOut := ttyCheck, assistIn, assistOut + t.Cleanup(func() { + ttyCheck = prevTTY + assistIn = prevIn + assistOut = prevOut + }) + ttyCheck = func(*os.File) bool { return true } + assistIn = strings.NewReader("Add a connection\n") + assistOut = &bytes.Buffer{} + + id, err := Session(SessionOptions{Agent: "claude"}) + if err != nil { + t.Fatal(err) + } + e, err := LoadEntry(id) + if err != nil { + t.Fatal(err) + } + if e.Meta.Task != modeOpen { + t.Fatalf("Meta.Task=%q want %q", e.Meta.Task, modeOpen) + } + if e.Answers.Task != modeOpen { + t.Fatalf("Answers.Task=%q want %q", e.Answers.Task, modeOpen) + } +} + +func TestRunOpenScreenUsesReaderWhenStdinSwapped(t *testing.T) { + prev := assistIn + t.Cleanup(func() { assistIn = prev }) + assistIn = strings.NewReader("Add a connection\n") + + p := PromptContext{Connections: []ProbeConn{{Name: "LOCAL", Source: "built-in"}}} + prevOut := assistOut + t.Cleanup(func() { assistOut = prevOut }) + buf := &bytes.Buffer{} + assistOut = buf + + ask, investigate, ok := runOpenScreen(p) + if !ok || ask != "Add a connection" || investigate { + t.Fatalf("ok=%v ask=%q investigate=%v", ok, ask, investigate) + } + if !strings.Contains(buf.String(), suggestionElseLabel) { + t.Fatalf("numbered card missing:\n%s", buf.String()) + } +} + +func TestPickOpenAskNoOptions(t *testing.T) { + if ask, _, ok := pickOpenAsk(nil); ok || ask != "" { + t.Fatalf("ok=%v ask=%q", ok, ask) + } +} + +func landingCtx(kind LandingKind) PromptContext { + base := PromptContext{ + Version: "dev", + Cwd: "/work", + ProjectName: "demo", + ProjectRoot: "/work", + HasProject: true, + FileCounts: map[string]int{"replications": 2, "pipelines": 1, "models": 3, "specs": 0}, + Connections: []ProbeConn{ + {Name: "MY_PG", Type: "PostgreSQL", Source: "sling env yaml"}, + {Name: "MY_SF", Type: "Snowflake", Source: "sling env yaml"}, + {Name: "LOCAL", Type: "Local File System", Source: "built-in"}, + }, + Route: "default", + } + switch kind { + case LandingFresh: + base.HasProject = false + base.ProjectName = "" + base.ProjectRoot = "" + base.Connections = nil + base.FileCounts = map[string]int{"replications": 0, "pipelines": 0, "models": 0, "specs": 0} + case LandingNoProject: + base.HasProject = false + base.ProjectName = "" + base.ProjectRoot = "/tmp" + base.Route = "no_project" + base.FileCounts = map[string]int{"replications": 0, "pipelines": 0, "models": 0, "specs": 0} + case LandingProject: + base.RecentRuns = []LocalExec{ + {ID: "exec_old", Status: "err", ConfigPath: "./old.yaml", When: time.Now().Add(-48 * time.Hour)}, + {ID: "exec_new", Status: "ok", ConfigPath: "./r.yaml", When: time.Now().Add(-2 * time.Hour)}, + } + } + return base +} + +func TestClassifyLanding(t *testing.T) { + if got := ClassifyLanding(PromptContext{HasProject: true}, true); got != LandingFresh { + t.Fatalf("fresh wins over project: %s", got) + } + if got := ClassifyLanding(PromptContext{HasProject: true}, false); got != LandingProject { + t.Fatalf("got %s", got) + } + if got := ClassifyLanding(PromptContext{}, false); got != LandingNoProject { + t.Fatalf("got %s", got) + } +} + +func TestSuggestedCommandLadder(t *testing.T) { + fail := PromptContext{RecentRuns: []LocalExec{{Status: "err"}}, Signature: "97d84811"} + if got := fail.SuggestedCommand(); got != "sling assist error 97d84811" { + t.Fatalf("got %q", got) + } + failNoSig := PromptContext{RecentRuns: []LocalExec{{Status: "err"}}} + if got := failNoSig.SuggestedCommand(); got != "sling assist" { + t.Fatalf("got %q", got) + } + zero := PromptContext{Connections: []ProbeConn{{Name: "LOCAL", Source: "built-in"}}, HasProject: true} + if got := zero.SuggestedCommand(); got != "sling assist" { + t.Fatalf("got %q", got) + } + noProj := PromptContext{Connections: []ProbeConn{{Name: "MY_PG", Source: "sling env yaml"}}} + if got := noProj.SuggestedCommand(); got != "sling init" { + t.Fatalf("got %q", got) + } + def := PromptContext{HasProject: true, Connections: []ProbeConn{{Name: "MY_PG", Source: "sling env yaml"}}} + if got := def.SuggestedCommand(); got != "sling assist" { + t.Fatalf("got %q", got) + } +} + +func TestRenderLandingThreeStates(t *testing.T) { + cases := []struct { + kind LandingKind + want []string + skip []string + }{ + { + kind: LandingFresh, + want: []string{ + "Welcome to sling", + "sling assist", + "https://docs.slingdata.io", + }, + skip: []string{"conns set", "sling init"}, + }, + { + kind: LandingNoProject, + want: []string{ + "2 connections configured", + "sling conns list", + "sling init", + }, + skip: []string{"conns set", "Welcome to sling"}, + }, + { + kind: LandingProject, + want: []string{ + "On project demo", + "not linked", + "replications 2", + "pipelines 1", + "models 3", + "last run: exec_new ./r.yaml [ok]", + "Next: sling assist", + }, + skip: []string{"conns set", "Welcome to sling"}, + }, + } + for _, tc := range cases { + t.Run(string(tc.kind), func(t *testing.T) { + got := RenderLanding(tc.kind, landingCtx(tc.kind), 80) + for _, w := range tc.want { + if !strings.Contains(got, w) { + t.Errorf("missing %q\n%s", w, got) + } + } + for _, s := range tc.skip { + if strings.Contains(got, s) { + t.Errorf("must not contain %q\n%s", s, got) + } + } + if n := strings.Count(got, "conns set"); n != 0 { + t.Errorf("conns set leaked") + } + }) + } +} + +func TestRenderLandingFailedRunSuggestsError(t *testing.T) { + p := landingCtx(LandingProject) + p.RecentRuns = []LocalExec{{ID: "exec_fail", Status: "err", ConfigPath: "./r.yaml", When: time.Now()}} + p.Signature = "97d84811" + p.Route = "failed_run" + got := RenderLanding(LandingProject, p, 80) + if !strings.Contains(got, "Next: sling assist error 97d84811") { + t.Fatalf("missing error suggestion:\n%s", got) + } + if !strings.Contains(got, "last run: exec_fail ./r.yaml [err]") { + t.Fatalf("missing last run:\n%s", got) + } +} + +func TestRenderLandingZeroConnectionsNoProject(t *testing.T) { + p := PromptContext{ + Connections: []ProbeConn{{Name: "LOCAL", Source: "built-in"}}, + } + got := RenderLanding(LandingNoProject, p, 80) + if !strings.Contains(got, "0 connections configured") { + t.Fatalf("got:\n%s", got) + } + if !strings.Contains(got, "sling init") { + t.Fatalf("missing project init:\n%s", got) + } +} + +func TestRenderLandingRespectsWidth(t *testing.T) { + got := RenderLanding(LandingNoProject, landingCtx(LandingNoProject), 40) + for i, line := range strings.Split(strings.TrimRight(got, "\n"), "\n") { + if n := utf8.RuneCountInString(line); n > 40 { + t.Errorf("line %d len %d > 40: %q", i, n, line) + } + } +} + +func TestFreshLandingPointsOnlyAtAssist(t *testing.T) { + got := RenderLanding(LandingFresh, PromptContext{}, 80) + if strings.Count(got, "sling assist") < 3 { + t.Fatalf("expected 3 assist pointers:\n%s", got) + } + if strings.Contains(got, "conns set") { + t.Fatalf("conns set leaked:\n%s", got) + } +} + +func TestIsFreshInstall(t *testing.T) { + dir := withTempHomeDir(t) + + if !IsFreshInstall() { + t.Fatal("empty home should be fresh") + } + + envPath := filepath.Join(dir, "env.yaml") + body := "# Environment Credentials for Sling CLI\n# See https://docs.slingdata.io/sling-cli/environment\n\nconnections:\n\n\nvariables:\n" + if err := os.WriteFile(envPath, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if !IsFreshInstall() { + t.Fatal("seeded default env.yaml should be fresh") + } + + if err := os.WriteFile(envPath, []byte("connections:\n MY_PG:\n type: postgres\n"), 0o644); err != nil { + t.Fatal(err) + } + if IsFreshInstall() { + t.Fatal("user connection should not be fresh") + } + + if err := os.WriteFile(envPath, []byte("connections:\n\nenv:\n SLING_ASSIST:\n agent: claude\n"), 0o644); err != nil { + t.Fatal(err) + } + if IsFreshInstall() { + t.Fatal("assist profile should not be fresh") + } + + if err := os.WriteFile(envPath, []byte("connections:\n"), 0o644); err != nil { + t.Fatal(err) + } + hist := filepath.Join(dir, "assist", "history", "sess1") + if err := os.MkdirAll(hist, 0o755); err != nil { + t.Fatal(err) + } + if IsFreshInstall() { + t.Fatal("assist history should not be fresh") + } +} + +func TestLatestRunPicksNewestNotFailure(t *testing.T) { + oldFail := LocalExec{ID: "old", Status: "err", When: time.Now().Add(-time.Hour)} + newOK := LocalExec{ID: "new", Status: "ok", When: time.Now()} + got, ok := latestRun([]LocalExec{oldFail, newOK}) + if !ok || got.ID != "new" { + t.Fatalf("got %+v ok=%v", got, ok) + } +} diff --git a/core/sling/assist/prompts.yaml b/core/sling/assist/prompts.yaml new file mode 100644 index 000000000..94f1a0ad0 --- /dev/null +++ b/core/sling/assist/prompts.yaml @@ -0,0 +1,44 @@ +# Embedded prompt templates for `sling assist`. +# +# `_skeleton` is the five-section agent prompt (`sling assist --out -`). +# `_objective` is the single ask-mode Objective. +# +# Top-level keys: +# _skeleton — Rules / State / Context / Ask / Objective +# _rules — shared Rules body +# _objective — ask-mode Objective + +_skeleton: |- + # Rules + {{.Rules}} + + # State + {{.State}} + + # Context + {{.Context}} + + # Ask + {{.Ask}} + + # Objective + {{.Objective}} + +_rules: |- + - You help with Sling CLI (https://docs.slingdata.io/llms.txt). + - Use Sling MCP tools when they are wired (connection, database, replication, pipeline, api_spec, file_system). + - Validate YAML with MCP `validate`. Execute replications and pipelines with `sling run`. Execute SQL models with `sling build`. There is no MCP `run` or `build` action. + - Do not invent connection names, streams, or modes. Ask one focused question instead. + - Never ask for credentials in chat. Use ${VAR} refs and `sling conns set`. + - Do not read env.yaml or print secrets. Connection names and types only. + - Validate YAML with the validate tools before you declare done. + - Before you create or change a Sling file, load the matching skill and resolve its 'Gather first' checklist. Use the request, files, and MCP discovery first; ask the user only for what stays unresolved. Do not ask when the answers are already in context. Propose defaults. + - Do not start work the user did not request. + - The Ask is your only goal. Context is background — do not act on it unless the Ask points there. + - When you debug a failed run, read `sling/TROUBLESHOOTING.md`. Re-run only after the user accepts the fix. + - After you classify a failure as a sling defect (not user config), offer: "Report this? GitHub issue (public, needs account) or email to support." Never offer this for user-config errors. Never auto-send. There is no MCP report tool. Run `sling assist report --id ` so the user reviews the draft, then `--github` or `--email`. + - When Context shows a platform exec, load the sling-platform skill. Get state with `sling project execs status `, then logs with `sling project execs log --status error`. Pull the config with `sling project files get `. These commands need SLING_PROJECT_TOKEN, which is already set. + +_objective: |- + Follow the Ask. Load the matching Sling skill (replications, pipelines, build, api-specs, connections, or project). + Complete its "Gather first" checklist before you write files. diff --git a/core/sling/assist/report.go b/core/sling/assist/report.go new file mode 100644 index 000000000..9b85cd22d --- /dev/null +++ b/core/sling/assist/report.go @@ -0,0 +1,574 @@ +package assist + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "net/url" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "github.com/charmbracelet/huh" + "github.com/flarco/g" + "github.com/slingdata-io/sling-cli/core" + "github.com/slingdata-io/sling-cli/core/dbio" + "github.com/slingdata-io/sling-cli/core/env" + "github.com/spf13/cast" +) + +const ( + githubIssueURLMax = 8000 + // Cloudflare rejects request lines over ~16KB. Stay well under it. + contactFormURLMax = 12000 + githubIssueBaseURL = "https://github.com/slingdata-io/sling-cli/issues/new" + contactFormBaseURL = "https://slingdata.io/contact/" + maxLogExcerptBytes = 64 * 1024 + maxSkeletonBytes = 4 * 1024 +) + +// ReportDraft is the composed, redacted report. Both routes consume it. +type ReportDraft struct { + Title string `json:"title"` + Description string `json:"description"` + Config string `json:"config"` + LogExcerpt string `json:"log_excerpt"` + Version string `json:"version"` + OS string `json:"os"` + SourceType string `json:"source_type"` + TargetType string `json:"target_type"` + SignatureID string `json:"signature_id"` + Skeleton string `json:"skeleton"` + ExecID string `json:"exec_id"` + Origin string `json:"origin,omitempty"` + PlatformHost string `json:"platform_host,omitempty"` + JobName string `json:"job_name,omitempty"` + ConnName string `json:"conn_name,omitempty"` + // CustomDescription is caller-supplied context, shown above the error. + CustomDescription string `json:"custom_description,omitempty"` +} + +// ComposeReport builds a redacted report from a local failure snapshot. +func ComposeReport(execID string) (ReportDraft, error) { + le, err := ResolveLocalExec(execID) + if err != nil { + return ReportDraft{}, err + } + + errText := readSnapshotFile(le.LogDir, "error.txt") + runLog := readSnapshotFile(le.LogDir, "stderr.log") + meta := map[string]any{} + if b, err := os.ReadFile(filepath.Join(le.LogDir, "meta.json")); err == nil { + _ = json.Unmarshal(b, &meta) + } + + src := cast.ToString(meta["source_type"]) + tgt := cast.ToString(meta["target_type"]) + label := cast.ToString(meta["error_short_label"]) + configPath := cast.ToString(meta["config_path"]) + if configPath == "" { + configPath = cast.ToString(meta["object"]) // legacy snapshots + } + connName := cast.ToString(meta["conn_name"]) + sigID := strings.ToLower(cast.ToString(meta["error_signature"])) + origin, platformHost, jobName := reportOriginFromMeta(meta) + + skel := Skeleton(errText) + if sigID == "" || len(sigID) != CompositeIDLen { + sig := SignError(errText, SignMeta{SourceType: dbio.Type(src), TargetType: dbio.Type(tgt)}) + sigID = sig.ID + if label == "" { + label = sig.ShortLabel + } + if src == "" { + src = string(sig.Meta.SourceType) + } + if tgt == "" { + tgt = string(sig.Meta.TargetType) + } + } + if label == "" { + label = ShortLabel(skel) + } + + d := ReportDraft{ + Title: reportTitle(label, src, tgt), + Description: reportDescription(redactForReport(errText)), + Config: reportConfig(configPath, connName, le.LogDir), + LogExcerpt: reportLogExcerpt(redactForReport(runLog)), + Version: core.Version, + OS: reportOSName(), + SourceType: src, + TargetType: tgt, + SignatureID: sigID, + Skeleton: skel, + ExecID: le.ID, + Origin: origin, + PlatformHost: platformHost, + JobName: jobName, + ConnName: connName, + } + return d, nil +} + +func reportTitle(label, src, tgt string) string { + if label == "" { + label = "unknown_error" + } + return fmt.Sprintf("%s (%s→%s)", label, typeToken(dbio.Type(src)), typeToken(dbio.Type(tgt))) +} + +func reportDescription(redactedErr string) string { + redactedErr = strings.TrimSpace(redactedErr) + if redactedErr == "" { + return "(no error message captured)" + } + // Keep the debug message stack (caller frames + wrap messages), not a + // collapsed one-line summary. GitHub URL budget trims logs/config first. + return redactedErr +} + +func reportConfig(configPath, connName, logDir string) string { + if connName != "" { + return "" + } + snap := filepath.Join(logDir, "config.snapshot.yaml") + body := "" + if b, err := os.ReadFile(snap); err == nil { + body = strings.TrimSpace(redactForReport(string(b))) + } + // Fall back to the config path on disk (snapshots written before + // config.snapshot.yaml existed). + if body == "" && isConfigFilePath(configPath) { + if b, err := os.ReadFile(configPath); err == nil && int64(len(b)) <= 64*1024 { + body = strings.TrimSpace(redactForReport(string(b))) + } + } + if body != "" { + return body + } + return "(not captured)" +} + +func isConfigFilePath(path string) bool { + return g.In(strings.ToLower(filepath.Ext(path)), ".yaml", ".yml", ".json") +} + +func reportLogExcerpt(redacted string) string { + redacted = strings.TrimRight(redacted, "\n") + if redacted == "" { + return "(no log captured)" + } + if len(redacted) <= maxLogExcerptBytes { + return redacted + } + s := redacted[len(redacted)-maxLogExcerptBytes:] + if i := strings.IndexByte(s, '\n'); i >= 0 && i < 200 { + s = s[i+1:] + } + return "[...truncated...]\n" + s +} + +func reportOSName() string { + switch runtime.GOOS { + case "linux": + return "Linux" + case "darwin": + return "Mac" + case "windows": + return "Windows" + default: + return runtime.GOOS + } +} + +func readSnapshotFile(dir, name string) string { + b, err := os.ReadFile(filepath.Join(dir, name)) + if err != nil { + return "" + } + return string(b) +} + +// redactForReport scrubs secrets, then replaces only URLs and file paths. +// Identifiers, numbers and timestamps stay readable for debugging. +func redactForReport(s string) string { + s = env.ScrubLine(s) + lines := strings.Split(s, "\n") + for i, line := range lines { + line = reQuotedURL.ReplaceAllStringFunc(line, replaceURL) + line = reQuotedPath.ReplaceAllString(line, "") + line = reURL.ReplaceAllStringFunc(line, replaceURL) + line = reVersionBanner.ReplaceAllString(line, "") + line = replacePathKeepLead(reUnixPath, line) + line = replacePathKeepLead(reWinPath, line) + line = replacePathKeepLead(reHomePath, line) + lines[i] = line + } + return strings.Join(lines, "\n") +} + +// BodyMarkdown renders sections that match the GitHub issue template fields. +func (d ReportDraft) BodyMarkdown() string { + var b strings.Builder + b.WriteString("## Description\n\n") + if custom := strings.TrimSpace(d.CustomDescription); custom != "" { + b.WriteString(custom) + b.WriteString("\n\n") + } + b.WriteString("Error:\n\n```\n") + b.WriteString(strings.TrimSpace(d.Description)) + b.WriteString("\n```\n\n") + fmt.Fprintf(&b, "Exec ID: %s\n", d.ExecID) + fmt.Fprintf(&b, "Sling version: %s\n", d.Version) + fmt.Fprintf(&b, "OS: %s\n", d.OS) + if strings.EqualFold(d.Origin, "platform") { + fmt.Fprintf(&b, "Platform execution (host: %s, job: %s)\n", d.PlatformHost, d.JobName) + } + if d.ConnName != "" { + fmt.Fprintf(&b, "Connection: %s\n", d.ConnName) + } + if d.SourceType != "" { + fmt.Fprintf(&b, "Source: %s\n", typeToken(dbio.Type(d.SourceType))) + } + if d.TargetType != "" { + fmt.Fprintf(&b, "Target: %s\n", typeToken(dbio.Type(d.TargetType))) + } + if d.ConnName == "" { + b.WriteString("\n## Replication Configuration\n\n") + b.WriteString("```yaml\n") + b.WriteString(strings.TrimRight(d.Config, "\n")) + b.WriteString("\n```\n") + } + b.WriteString("\n## Log Output\n\n") + b.WriteString("```\n") + b.WriteString(strings.TrimRight(d.LogExcerpt, "\n")) + b.WriteString("\n```\n") + return b.String() +} + +// trimToBudget shrinks logs, then config, until build stays within max bytes. +// Never trims the description. +func (d ReportDraft) trimToBudget(max int, build func(logs, config string) string) string { + logs := d.LogExcerpt + config := d.Config + u := build(logs, config) + for len(u) > max && logsHasMore(logs) { + logs = dropFirstLine(logs) + u = build(logs, config) + } + if len(u) > max && len(logs) > 0 { + // One remaining line still too long: keep the tail. + keep := len(logs) / 2 + for keep > 32 && len(build(logs[len(logs)-keep:], config)) > max { + keep = keep / 2 + } + if keep < len(logs) { + logs = logs[len(logs)-keep:] + } + u = build(logs, config) + } + for len(u) > max && config != "" && config != "(not captured)" && config != "(truncated)" { + next := dropFirstLine(config) + if next == config || next == "" { + if len(config) > 64 { + config = config[len(config)/2:] + } else { + config = "(truncated)" + } + } else { + config = next + } + if strings.TrimSpace(config) == "" { + config = "(truncated)" + } + u = build(logs, config) + } + return u +} + +// GitHubIssueURL builds a prefilled new-issue URL. Issue forms (.yml) ignore +// query params except title, so the full report goes into body against the +// blank issue form. +func (d ReportDraft) GitHubIssueURL() string { + return d.trimToBudget(githubIssueURLMax, func(logs, config string) string { + trimmed := d + trimmed.LogExcerpt = logs + trimmed.Config = config + q := url.Values{} + q.Set("title", d.Title) + q.Set("body", trimmed.BodyMarkdown()) + return githubIssueBaseURL + "?" + q.Encode() + }) +} + +// ContactFormURL builds the contact-page URL with the full report body +// base64 encoded in the issue param. The form collects name/email and +// Turnstile. Never trims logs or config; oversized bodies go to a file +// instead (see deliverContactForm). +func (d ReportDraft) ContactFormURL() string { + return contactFormBaseURL + "?issue=" + base64.RawURLEncoding.EncodeToString([]byte(d.BodyMarkdown())) +} + +// deliverGitHubIssue opens a prefilled GitHub issue when the URL fits. +// When it still exceeds githubIssueURLMax after trimming, it falls back +// to the email contact form (or a temp file if that is also oversize). +func deliverGitHubIssue(d ReportDraft, interactive bool) error { + u := d.GitHubIssueURL() + if len(u) > githubIssueURLMax { + fmt.Fprintln(os.Stdout, "report too long for a GitHub issue URL; sending via email instead.") + return deliverContactForm(d, interactive) + } + fmt.Fprintln(os.Stdout, u) + if interactive { + if err := OpenBrowser(u); err != nil { + g.Warn("could not open browser: %s", err.Error()) + } + } + return nil +} + +// deliverContactForm opens the prefilled contact page when the URL fits. +// When it exceeds the Cloudflare request-line limit, it writes the full +// report to a temp file for manual email instead. +func deliverContactForm(d ReportDraft, interactive bool) error { + u := d.ContactFormURL() + if len(u) > contactFormURLMax { + return writeReportFile(d) + } + fmt.Fprintln(os.Stdout, u) + if interactive { + if err := OpenBrowser(u); err != nil { + g.Warn("could not open browser: %s", err.Error()) + } + fmt.Fprintln(os.Stdout, "complete the name/email fields in the form, then submit.") + } + return nil +} + +// writeReportFile saves the full report body for manual email as attachment. +func writeReportFile(d ReportDraft) error { + f, err := os.CreateTemp("", "sling-report-*.md") + if err != nil { + return g.Error("could not write report file: %s", err.Error()) + } + defer f.Close() + if _, err := f.WriteString(d.BodyMarkdown()); err != nil { + return g.Error("could not write report file: %s", err.Error()) + } + fmt.Fprintln(os.Stdout, "report too long for the web form.") + fmt.Fprintf(os.Stdout, "attach this file and email it to %s:\n %s\n", "support@slingdata.io", f.Name()) + return nil +} + +func logsHasMore(s string) bool { + return strings.Contains(s, "\n") +} + +func dropFirstLine(s string) string { + i := strings.IndexByte(s, '\n') + if i < 0 { + return "" + } + return s[i+1:] +} + +// OpenBrowser opens url in the default browser. Always also print the URL. +func OpenBrowser(rawURL string) error { + if browserEnv := strings.TrimSpace(os.Getenv("BROWSER")); browserEnv != "" { + return exec.Command(browserEnv, rawURL).Start() + } + var cmd string + var args []string + switch runtime.GOOS { + case "windows": + cmd = "rundll32" + args = []string{"url.dll,FileProtocolHandler", rawURL} + case "darwin": + cmd = "open" + args = []string{rawURL} + default: + cmd = "xdg-open" + args = []string{rawURL} + } + return exec.Command(cmd, args...).Start() +} + +func composeReportWithPlatformFallback(execID string, localErr error) (ReportDraft, error) { + pe, err := resolvePlatformFallback(execID, localErr) + if err != nil { + return ReportDraft{}, err + } + if _, err := materializePlatformExec(pe); err != nil { + return ReportDraft{}, err + } + return ComposeReport(execID) +} + +// ReportCmd is the `sling assist report` entry. +type ReportCmd struct { + ExecID string + Title string // optional override + Description string // optional custom context, shown above the error + GitHub bool + Email bool + Submit bool // skip the confirm prompt (for agents) +} + +// RunReport prints the redacted draft, then optionally sends it. +func RunReport(opts ReportCmd) error { + d, err := ComposeReport(opts.ExecID) + if err != nil { + d, err = composeReportWithPlatformFallback(opts.ExecID, err) + if err != nil { + return err + } + } + if opts.Title != "" { + d.Title = opts.Title + } + if opts.Description != "" { + d.CustomDescription = opts.Description + } + fmt.Fprintln(os.Stdout, d.Title) + fmt.Fprintln(os.Stdout, "") + body := d.BodyMarkdown() + fmt.Fprint(os.Stdout, body) + if !strings.HasSuffix(body, "\n") { + fmt.Fprintln(os.Stdout) + } + + if opts.GitHub && opts.Email { + return g.Error("use only one of --github or --email") + } + + route := "" + switch { + case opts.GitHub: + route = "github" + case opts.Email: + route = "email" + } + + if !env.IsInteractiveTerminal() { + if route == "github" { + return deliverGitHubIssue(d, false) + } + if route == "email" { + // Form flow: the user confirms by submitting the contact form. + return deliverContactForm(d, false) + } + return nil + } + + if route == "" { + picked, err := pickReportRoute() + if err != nil { + return err + } + if picked == "cancel" || picked == "" { + return nil + } + route = picked + } + + if !opts.Submit { + ok, err := confirmSendReport() + if err != nil { + return err + } + if !ok { + return nil + } + } + return sendReport(d, route) +} + +func sendReport(d ReportDraft, route string) error { + switch route { + case "github": + return deliverGitHubIssue(d, true) + case "email": + // The contact form's Turnstile check and name/email fields validate + // the submission. + return deliverContactForm(d, true) + default: + return g.Error("unknown report route %q", route) + } +} + +func pickReportRoute() (string, error) { + route := "cancel" + form := huh.NewForm( + huh.NewGroup( + huh.NewSelect[string](). + Title("How do you want to send this report?"). + Options( + huh.NewOption("GitHub issue (public, needs account)", "github"), + huh.NewOption("Email to support", "email"), + huh.NewOption("Cancel", "cancel"), + ). + Value(&route), + ), + ).WithTheme(huh.ThemeCharm()) + if err := form.Run(); err != nil { + return "", err + } + return route, nil +} + +func confirmSendReport() (bool, error) { + ok := false + form := huh.NewForm( + huh.NewGroup( + huh.NewConfirm(). + Title("Send this report? [y/N]"). + Affirmative("Yes"). + Negative("No"). + Value(&ok), + ), + ).WithTheme(huh.ThemeCharm()) + if err := form.Run(); err != nil { + return false, err + } + return ok, nil +} + +// PlatformExec is the platform view of one execution. +type PlatformExec struct { + ExecID string + Status string // success | error | running | ... + Type string // replication | pipeline | query | monitor + JobName string + FileName string + StartTime string // formatted, may be empty + EndTime string + ErrSummary string // first error line from the record, if present + HostLabel string + Rows string + Duration string + Object string + Version string +} + +// resolvePlatformFallback looks up a missed local exec on the Sling Platform. +var resolvePlatformFallback = func(execID string, localErr error) (*PlatformExec, error) { + if localErr == nil { + return nil, g.Error("see ~/.sling/assist/errors/") + } + return nil, g.Error(localErr, "see ~/.sling/assist/errors/") +} + +// materializePlatformExec writes a local snapshot from platform data. +var materializePlatformExec = func(pe *PlatformExec) (LocalExec, error) { + return LocalExec{}, g.Error("use the official release of sling-cli to materialize platform executions") +} + +func reportOriginFromMeta(meta map[string]any) (origin, host, job string) { + if len(meta) == 0 { + return "", "", "" + } + return cast.ToString(meta["origin"]), cast.ToString(meta["platform_host"]), cast.ToString(meta["job_name"]) +} diff --git a/core/sling/assist/report_test.go b/core/sling/assist/report_test.go new file mode 100644 index 000000000..bf7046ae9 --- /dev/null +++ b/core/sling/assist/report_test.go @@ -0,0 +1,358 @@ +package assist + +import ( + "encoding/base64" + "io" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/slingdata-io/sling-cli/core/dbio" +) + +func writeReportFixture(t *testing.T, id string) string { + t.Helper() + withTempHomeDir(t) + errMsg := "could not read /Users/alice/secret/file.csv from host" + WriteFailureSnapshot(FailureSnapshot{ + ExecID: id, + ErrMsg: errMsg, + ConfigPath: "/Users/alice/secret/repl.yaml", + SignMeta: SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbSnowflake}, + RunLog: strings.Join([]string{ + "opened https://db.internal.example.com/sync", + "path=/Users/alice/secret/file.csv", + "failed to copy rows", + }, "\n"), + }) + dir := findLocalExecDir(id) + if dir == "" { + t.Fatal("fixture dir missing") + } + return dir +} + +func TestComposeReportGoldenDraft(t *testing.T) { + writeReportFixture(t, "exec_report1") + d, err := ComposeReport("exec_report1") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(d.Title, " (postgres→snowflake)") { + t.Fatalf("title = %q", d.Title) + } + md := d.BodyMarkdown() + descAt := strings.Index(md, "## Description") + cfgAt := strings.Index(md, "## Replication Configuration") + logAt := strings.Index(md, "## Log Output") + if descAt < 0 || cfgAt < 0 || logAt < 0 || !(descAt < cfgAt && cfgAt < logAt) { + t.Fatalf("markdown section order:\n%s", md) + } + if strings.Contains(d.LogExcerpt, "/Users/") { + t.Fatalf("path leaked in LogExcerpt: %q", d.LogExcerpt) + } + if strings.Contains(d.LogExcerpt, "db.internal.example.com") { + t.Fatalf("hostname leaked in LogExcerpt: %q", d.LogExcerpt) + } + if !strings.Contains(d.LogExcerpt, "") && !strings.Contains(d.LogExcerpt, "") { + t.Fatalf("expected placeholders in LogExcerpt: %q", d.LogExcerpt) + } + if strings.Contains(d.Config, "/Users/") { + t.Fatalf("path leaked in Config: %q", d.Config) + } + if strings.Contains(md, d.SignatureID) { + t.Fatalf("signature must not appear in markdown body") + } +} + +func TestComposeReportKeepsDebugStack(t *testing.T) { + withTempHomeDir(t) + errMsg := strings.Join([]string{ + "~ could not connect", + "--- database.go:123 Connect ---", + "~ failed to ping", + "--- task_run.go:140 Execute ---", + "connection refused", + }, "\n") + WriteFailureSnapshot(FailureSnapshot{ + ExecID: "exec_stack1", + ErrMsg: errMsg, + SignMeta: SignMeta{SourceType: dbio.TypeDbPostgres, TargetType: dbio.TypeDbSnowflake}, + }) + d, err := ComposeReport("exec_stack1") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(d.Description, "--- database.go:123 Connect ---") { + t.Fatalf("debug stack stripped from Description:\n%s", d.Description) + } + if !strings.Contains(d.Description, "--- task_run.go:140 Execute ---") { + t.Fatalf("debug stack stripped from Description:\n%s", d.Description) + } + if strings.Contains(d.Description, "could not connect failed to ping") { + t.Fatalf("debug stack collapsed to one line:\n%s", d.Description) + } + md := d.BodyMarkdown() + if !strings.Contains(md, "--- task_run.go:140 Execute ---") { + t.Fatalf("debug stack stripped from body:\n%s", md) + } +} + +func TestComposeReportRedactsPathAndHost(t *testing.T) { + writeReportFixture(t, "exec_redact1") + d, err := ComposeReport("exec_redact1") + if err != nil { + t.Fatal(err) + } + if strings.Contains(d.LogExcerpt, "/Users/alice") { + t.Fatalf("raw path in excerpt: %q", d.LogExcerpt) + } + if strings.Contains(d.LogExcerpt, "db.internal.example.com") { + t.Fatalf("raw host in excerpt: %q", d.LogExcerpt) + } +} + +func TestComposeReportPrefixID(t *testing.T) { + writeReportFixture(t, "exec_prefix_abc") + if _, err := ComposeReport("exec_pre"); err != nil { + t.Fatal(err) + } +} + +func TestGitHubIssueURLBudget(t *testing.T) { + var b strings.Builder + for i := 0; i < 400; i++ { + b.WriteString("line ") + b.WriteString(strings.Repeat("x", 80)) + b.WriteByte('\n') + } + d := ReportDraft{ + Title: "boom (postgres→snowflake)", + Description: "kept description", + Config: "repl.yaml", + LogExcerpt: b.String(), + Version: "1.4.24", + OS: "Mac", + } + u := d.GitHubIssueURL() + if len(u) > githubIssueURLMax { + t.Fatalf("url len %d > %d", len(u), githubIssueURLMax) + } + parsed, err := url.Parse(u) + if err != nil { + t.Fatal(err) + } + q := parsed.Query() + if q.Get("template") != "" { + t.Fatalf("template param must be absent for body prefill, got %q", q.Get("template")) + } + body := q.Get("body") + if !strings.Contains(body, "kept description") { + t.Fatalf("description trimmed from body") + } + if !strings.Contains(body, "Sling version: 1.4.24") { + t.Fatalf("version missing from body") + } + if !strings.Contains(body, "OS: Mac") { + t.Fatalf("os missing from body") + } + if !strings.Contains(body, "```") { + t.Fatal("logs stripped entirely from body") + } +} + +func TestContactFormURLNoTrim(t *testing.T) { + d := ReportDraft{ + Title: "boom", + Description: "kept", + Config: strings.Repeat("col: value\n", 4000), + LogExcerpt: strings.Repeat("log line\n", 4000), + Version: "1.4.24", + OS: "Mac", + } + u := d.ContactFormURL() + if !strings.HasPrefix(u, contactFormBaseURL+"?issue=") { + t.Fatalf("bad prefix: %q", u[:60]) + } + // Over-limit bodies must stay complete; delivery falls back to a file. + if len(u) <= contactFormURLMax { + t.Fatalf("expected over-limit url, got %d", len(u)) + } + enc := strings.TrimPrefix(u, contactFormBaseURL+"?issue=") + decoded, err := base64.RawURLEncoding.DecodeString(enc) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(decoded), "log line") || !strings.Contains(string(decoded), "col: value") { + t.Fatal("body trimmed") + } +} + +func TestDeliverContactFormOversizedWritesFile(t *testing.T) { + d := ReportDraft{ + Title: "boom", + Description: "kept", + Config: strings.Repeat("col: value\n", 4000), + LogExcerpt: "log", + Version: "1.4.24", + OS: "Mac", + } + // Writes to stdout; failure surfaces as an error return. + if err := deliverContactForm(d, false); err != nil { + t.Fatal(err) + } +} + +func captureStdout(t *testing.T, fn func()) string { + t.Helper() + r, w, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + old := os.Stdout + os.Stdout = w + fn() + _ = w.Close() + os.Stdout = old + b, err := io.ReadAll(r) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestDeliverGitHubIssueOversizedFallsBackToEmail(t *testing.T) { + d := ReportDraft{ + Title: "boom", + Description: "kept", + CustomDescription: strings.Repeat("context ", 4000), + Config: "cfg", + LogExcerpt: "log", + Version: "1.4.24", + OS: "Mac", + } + if n := len(d.GitHubIssueURL()); n <= githubIssueURLMax { + t.Fatalf("fixture not over github budget: %d", n) + } + out := captureStdout(t, func() { + if err := deliverGitHubIssue(d, false); err != nil { + t.Fatal(err) + } + }) + if !strings.Contains(out, "sending via email instead") { + t.Fatalf("missing email fallback:\n%s", out) + } + if strings.Contains(out, githubIssueBaseURL) { + t.Fatalf("still printed a GitHub URL:\n%s", out) + } +} + +func TestDeliverGitHubIssueFitsStaysOnGitHub(t *testing.T) { + d := ReportDraft{ + Title: "boom", + Description: "kept", + Config: "cfg", + LogExcerpt: "log", + Version: "1.4.24", + OS: "Mac", + } + u := d.GitHubIssueURL() + if len(u) > githubIssueURLMax { + t.Fatalf("fixture over github budget: %d", len(u)) + } + out := captureStdout(t, func() { + if err := deliverGitHubIssue(d, false); err != nil { + t.Fatal(err) + } + }) + if !strings.Contains(out, githubIssueBaseURL) { + t.Fatalf("missing GitHub URL:\n%s", out) + } + if strings.Contains(out, "sending via email instead") { + t.Fatalf("should not fall back to email:\n%s", out) + } +} + +func TestGitHubIssueURLDropdownAndEscape(t *testing.T) { + d := ReportDraft{ + Title: "hash # and café", + Description: "line1\nline2 #frag", + Config: "a=b", + LogExcerpt: "ok", + OS: "FreeBSD", + Version: "dev", + } + u := d.GitHubIssueURL() + parsed, err := url.Parse(u) + if err != nil { + t.Fatal(err) + } + q := parsed.Query() + body := q.Get("body") + if !strings.Contains(body, "line1\nline2 #frag") { + t.Fatalf("description unescape = %q", body) + } + if !strings.Contains(body, "OS: FreeBSD") { + t.Fatalf("non-dropdown os should stay in body text, got %q", body) + } + if q.Get("title") != "hash # and café" { + t.Fatalf("title unescape = %q", q.Get("title")) + } +} + +func TestComposeReportConnTestHasNoConfig(t *testing.T) { + withTempHomeDir(t) + WriteFailureSnapshot(FailureSnapshot{ + ExecID: "exec_conn1", + ErrMsg: "could not connect", + ConnName: "MY_PG", + SignMeta: SignMeta{SourceType: dbio.TypeDbPostgres}, + }) + dir := findLocalExecDir("exec_conn1") + if dir == "" { + t.Fatal("fixture dir missing") + } + if _, err := os.Stat(filepath.Join(dir, "config.snapshot.yaml")); !os.IsNotExist(err) { + t.Fatal("conns test must not write config.snapshot.yaml") + } + d, err := ComposeReport("exec_conn1") + if err != nil { + t.Fatal(err) + } + if d.ConnName != "MY_PG" { + t.Fatalf("ConnName=%q", d.ConnName) + } + if d.Config != "" { + t.Fatalf("config text must be empty for conns test, got %q", d.Config) + } + md := d.BodyMarkdown() + if !strings.Contains(md, "Connection: MY_PG") { + t.Fatalf("missing connection line:\n%s", md) + } + if strings.Contains(md, "## Replication Configuration") { + t.Fatalf("conns test must not include config section:\n%s", md) + } + if strings.Contains(md, "Target:") { + t.Fatalf("conns test must not invent a target:\n%s", md) + } +} + +func TestHandleReportComposeRedacts(t *testing.T) { + writeReportFixture(t, "exec_mcp1") + d, err := ComposeReport("exec_mcp1") + if err != nil { + t.Fatal(err) + } + raw, err := os.ReadFile(filepath.Join(findLocalExecDir("exec_mcp1"), "stderr.log")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(raw), "/Users/alice") { + t.Fatal("fixture missing raw path") + } + if strings.Contains(d.LogExcerpt, "/Users/alice") { + t.Fatalf("compose leaked path: %q", d.LogExcerpt) + } +} diff --git a/core/sling/assist/session.go b/core/sling/assist/session.go new file mode 100644 index 000000000..fbc214f82 --- /dev/null +++ b/core/sling/assist/session.go @@ -0,0 +1,820 @@ +package assist + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/charmbracelet/huh" + "github.com/flarco/g" + "github.com/google/uuid" + "github.com/slingdata-io/sling-cli/core" + "github.com/slingdata-io/sling-cli/core/env" + "golang.org/x/term" +) + +// assistOut is stdout for --out - / nested-launch / the open card. Tests swap it. +var assistOut io.Writer = os.Stdout + +const ( + modeAsk = "ask" + modeOpen = "open" +) + +// SessionOptions is the flags-only `sling assist` invocation. +type SessionOptions struct { + Ask string + Name string + Agent string + Model string + Print bool + OutputFile string + Headless bool + ExecID string // --id: investigate this failure + ResumeID string + ResumeSet bool // --resume present (empty id → picker already resolved) + NonInteractive map[string]string +} + +// NestedLaunch reports an already-running CLI agent (or non-TTY stdin). +// When true, print the prompt instead of spawning another agent. +func NestedLaunch() bool { + if os.Getenv("CLAUDECODE") != "" { + return true + } + if os.Getenv("CURSOR_TRACE_ID") != "" { + return true + } + if os.Getenv("OPENCODE") != "" || os.Getenv("OPENCODE_SESSION") != "" { + return true + } + return false +} + +// Session probes local state, renders the five-section prompt, then prints or launches. +func Session(opts SessionOptions) (string, error) { + if opts.ResumeID != "" { + return resumeSession(opts) + } + + if len(opts.NonInteractive) > 0 { + if v := opts.NonInteractive["Intention"]; v != "" && opts.Ask == "" { + opts.Ask = v + } + if v := opts.NonInteractive["intention"]; v != "" && opts.Ask == "" { + opts.Ask = v + } + } + opts.Ask = strings.TrimSpace(opts.Ask) + + if opts.Headless && opts.Ask == "" && opts.ExecID == "" { + return "", g.Error(`no ask given; pass it as an argument: sling assist ""`) + } + + // Failure details enter the prompt only when the ask targets the + // failure: the investigate pick, or the empty-ask fallback. + ctx := Probe(ProbeOptions{ + Ask: opts.Ask, + ExecID: opts.ExecID, + IncludeFailure: opts.Ask == "", + }) + if opts.ExecID != "" && ctx.TargetExec == nil { + _, localErr := ResolveLocalExec(opts.ExecID) + if localErr != nil { + pe, err := resolvePlatformFallback(opts.ExecID, localErr) + if err != nil { + return "", err + } + ctx.PlatformExec = pe + ctx.Route = "platform_failed_run" + } + } + mode := modeAsk + if shouldOpenScreen(opts) { + ask, investigate, ok := runOpenScreen(ctx) + if !ok { + return "", nil + } + opts.Ask = ask + ctx = Probe(ProbeOptions{Ask: opts.Ask, IncludeFailure: investigate}) + mode = modeOpen + } + + prompt, err := RenderPrompt(ctx) + if err != nil { + return "", err + } + + if opts.Print { + fmt.Fprint(assistOut, prompt) + return prompt, nil + } + if opts.OutputFile != "" { + if err := os.WriteFile(opts.OutputFile, []byte(prompt), 0o644); err != nil { + return "", g.Error(err, "write --out %s", opts.OutputFile) + } + return prompt, nil + } + if NestedLaunch() || (!ttyCheck(os.Stdin) && !opts.Headless) { + fmt.Fprint(assistOut, prompt) + return prompt, nil + } + + if err := EnsureAssistReady(); err != nil { + return "", err + } + prof, _, _ := LoadProfile() + resolvedAgent, agentErr := ResolveAgent(opts.Agent, prof) + if agentErr != nil { + return "", agentErr + } + + a := AnswersFile{ + Name: opts.Name, + Task: mode, + SlingVersion: core.Version, + Created: time.Now().UTC(), + Agent: resolvedAgent, + Cwd: mustGetwd(), + Answers: map[string]any{"intention": opts.Ask, "ask": opts.Ask}, + } + if a.Name == "" { + if s := slugify(opts.Ask); s != "" && s != "entry" { + a.Name = s + } else { + a.Name = "assist" + } + } + now := time.Now().UTC() + harnessID := newHarnessSessionID(resolvedAgent) + m := Meta{ + Task: mode, + Agent: resolvedAgent, + Model: opts.Model, + HarnessSessionID: harnessID, + LaunchedAt: &now, + } + id, err := SaveEntry(a, prompt, m) + if err != nil { + return prompt, err + } + if err := AutoTrim(); err != nil { + g.Debug("assist: auto-trim: %s", err.Error()) + } + if resolvedAgent == "" { + return id, nil + } + promptPath := filepath.Join(HistoryDir(), id, "prompt.md") + g.Info("submitting prompt to agent %s: %s", + env.CyanString(resolvedAgent), env.DarkGrayString(collapseHome(promptPath))) + snap := snapshotHarnessFiles(resolvedAgent) + err = LaunchAgent(LaunchOptions{ + Agent: resolvedAgent, + Prompt: prompt, + PromptPath: promptPath, + Model: opts.Model, + SessionID: harnessID, + }) + if harnessID == "" { + if hid := discoverHarnessSessionID(resolvedAgent, snap); hid != "" { + e, lerr := LoadEntry(id) + if lerr == nil { + e.Meta.HarnessSessionID = hid + if serr := e.saveMeta(); serr != nil { + g.Debug("assist: save harness session id: %s", serr.Error()) + } + } + } + } + if err != nil { + var ae *AgentExitError + if errors.As(err, &ae) { + return id, ae + } + return id, g.Error(err, "agent launch failed") + } + return id, nil +} + +func resumeSession(opts SessionOptions) (string, error) { + e, err := LoadEntry(opts.ResumeID) + if err != nil { + return "", g.Error("unknown session %q; run `sling assist --resume` to pick one", opts.ResumeID) + } + + promptPath := filepath.Join(e.Path, "prompt.md") + promptBytes, _ := os.ReadFile(promptPath) + prompt := string(promptBytes) + + if opts.Print { + fmt.Fprint(assistOut, prompt) + return prompt, nil + } + if opts.OutputFile != "" { + if err := os.WriteFile(opts.OutputFile, promptBytes, 0o644); err != nil { + return "", g.Error(err, "write --out %s", opts.OutputFile) + } + return prompt, nil + } + + if err := EnsureAssistReady(); err != nil { + return "", err + } + + agent := e.Answers.Agent + if e.Meta.Agent != "" { + agent = e.Meta.Agent + } + if opts.Agent != "" && opts.Agent != agent { + return "", g.Error("session %q was launched with agent %q; cannot resume with --agent %s", e.ID, agent, opts.Agent) + } + + hid := e.Meta.HarnessSessionID + if hid == "" { + hid = discoverHarnessSessionID(agent, nil) + if hid != "" { + e.Meta.HarnessSessionID = hid + if serr := e.saveMeta(); serr != nil { + g.Debug("assist: save harness session id: %s", serr.Error()) + } + } + } + if hid == "" { + return "", g.Error("session %q has no harness session id; cannot resume", e.ID) + } + + if err := LaunchResume(agent, hid, opts.Model); err != nil { + var ae *AgentExitError + if errors.As(err, &ae) { + return e.ID, ae + } + return e.ID, g.Error(err, "agent resume failed") + } + return e.ID, nil +} + +// assistIn is stdin for the open screen. Tests swap it. +var assistIn io.Reader = os.Stdin + +// ttyCheck is the TTY probe. Tests swap it. +var ttyCheck = isTTY + +func shouldOpenScreen(opts SessionOptions) bool { + if strings.TrimSpace(opts.Ask) != "" || opts.ExecID != "" { + return false + } + if opts.Print || opts.OutputFile != "" || opts.Headless || NestedLaunch() { + return false + } + return ttyCheck(os.Stdin) +} + +func runOpenScreen(p PromptContext) (ask string, investigate, ok bool) { + width := 80 + if w, _, err := term.GetSize(int(os.Stdout.Fd())); err == nil && w > 0 { + width = w + } + opts := p.suggestions() + if assistIn == os.Stdin { + // Real terminal: the huh form draws its own list, no banner. + return pickOpenAsk(opts) + } + fmt.Fprint(assistOut, renderOpenCard(p, width)) + return readOpenAsk(assistIn, assistOut, opts) +} + +// pickOpenAsk is the huh picker: a select over the suggestion rows, then a +// text area when the user picks the describe-it row. +func pickOpenAsk(opts []suggestion) (ask string, investigate, ok bool) { + if len(opts) == 0 { + return "", false, false + } + sel := make([]huh.Option[int], 0, len(opts)) + for i, s := range opts { + sel = append(sel, huh.NewOption(s.Label, i)) + } + pick := 0 + free := "" + isFree := func() bool { return pick >= 0 && pick < len(opts) && strings.TrimSpace(opts[pick].Ask) == "" } + + form := huh.NewForm( + huh.NewGroup( + // No Height: huh sizes the viewport to all options, so a + // wrapped row never scrolls the list. + huh.NewSelect[int](). + Title("What would you like to do?"). + Options(sel...). + Value(&pick), + ), + huh.NewGroup( + huh.NewText(). + Title("Describe what you want"). + Placeholder("e.g. backfill orders from postgres into snowflake"). + Lines(5). + CharLimit(2000). + Value(&free), + ).WithHideFunc(func() bool { return !isFree() }), + ).WithTheme(huh.ThemeCharm()) + + if err := form.Run(); err != nil { + if !errors.Is(err, huh.ErrUserAborted) { + g.Debug("assist: open picker: %s", err.Error()) + } + fmt.Fprintln(assistOut, `No ask given. Run: sling assist ""`) + return "", false, false + } + + if pick < 0 || pick >= len(opts) { + return "", false, false + } + s := opts[pick] + if strings.TrimSpace(s.Ask) != "" { + return s.Ask, s.Investigate, true + } + free = strings.TrimSpace(free) + if free == "" { + fmt.Fprintln(assistOut, `No ask given. Run: sling assist ""`) + return "", false, false + } + return free, false, true +} + +func renderOpenCard(p PromptContext, width int) string { + if width <= 0 { + width = 80 + } + var b strings.Builder + b.WriteString(openSummary(p)) + b.WriteByte('\n') + for i, s := range p.suggestions() { + fmt.Fprintf(&b, " %d. %s\n", i+1, s.Label) + } + b.WriteByte('\n') + return wrapToWidth(strings.TrimRight(b.String(), "\n")+"\n", width) +} + +func openSummary(p PromptContext) string { + n := userConnectionCount(p.Connections) + noun := "connections" + if n == 1 { + noun = "connection" + } + var line1 string + switch { + case p.HasProject && p.ProjectName != "": + line1 = fmt.Sprintf("On project %s · %d %s", p.ProjectName, n, noun) + case p.HasProject: + line1 = fmt.Sprintf("On this project · %d %s", n, noun) + default: + line1 = fmt.Sprintf("No project in this folder · %d %s", n, noun) + } + if run, ok := latestRun(p.RecentRuns); ok { + label := run.ID + if obj := run.displayObject(); obj != "" { + label += " " + obj + } + s := fmt.Sprintf("%s [%s]", label, run.Status) + if !run.When.IsZero() { + rt := relTime(run.When) + if rt == "just now" { + s += " just now" + } else { + s += " " + rt + " ago" + } + } + return line1 + "\n last run: " + s + "\n" + } + return line1 + "\n" +} + +func readOpenAsk(r io.Reader, w io.Writer, opts []suggestion) (ask string, investigate, ok bool) { + br := bufio.NewReader(r) + empty := 0 + fmt.Fprint(w, "> ") + for { + line, err := br.ReadString('\n') + if err != nil && strings.TrimSpace(line) == "" { + fmt.Fprintln(w, `No ask given. Run: sling assist ""`) + return "", false, false + } + line = strings.TrimSpace(line) + if line == "" { + empty++ + if empty >= 2 { + fmt.Fprintln(w, `No ask given. Run: sling assist ""`) + return "", false, false + } + fmt.Fprintln(w, `Type a number or describe what you want.`) + fmt.Fprint(w, "> ") + continue + } + empty = 0 + if n, convErr := strconv.Atoi(line); convErr == nil && n >= 1 && n <= len(opts) { + s := opts[n-1] + if strings.TrimSpace(s.Ask) != "" { + return s.Ask, s.Investigate, true + } + fmt.Fprintln(w, "Describe it:") + fmt.Fprint(w, "> ") + continue + } + return line, false, true + } +} + +// isTTY reports whether the file descriptor is connected to a terminal. +func isTTY(f *os.File) bool { + if f == nil { + return false + } + info, err := f.Stat() + if err != nil { + return false + } + return (info.Mode() & os.ModeCharDevice) != 0 +} + +// ResolveAgent picks the agent to launch, in order: +// 1. --agent override (must be a known CLI agent and detected on disk). +// 2. profile.Agent (when not "auto"). +// 3. profile.Agent == "auto" → first detected CLI agent on $PATH. +func ResolveAgent(override string, prof Profile) (string, error) { + if override != "" { + c := LookupClient(override) + if c == nil { + return "", g.Error("unknown agent %q", override) + } + if c.Kind() != KindCLIAgent { + return "", g.Error("agent %q is not a launchable CLI agent (it's an install target)", override) + } + if !c.Detect() { + return "", g.Error("agent %q not detected; run `sling assist setup` to set it up", override) + } + return override, nil + } + if prof.Agent != "" && prof.Agent != "auto" { + c := LookupClient(prof.Agent) + if c == nil { + return "", g.Error("profile names unknown agent %q; run `sling assist setup`", prof.Agent) + } + if c.Kind() != KindCLIAgent { + return "", g.Error("profile names non-launchable agent %q; run `sling assist setup` or use --agent", prof.Agent) + } + return prof.Agent, nil + } + for _, c := range CLIAgents() { + if c.Detect() && commandOnPath(agentBinary(c.Name())) { + return c.Name(), nil + } + } + return "", g.Error("no AI agent on $PATH; run `sling assist setup` or pass --agent") +} + +type agentLaunchPlan struct { + Args []string + UseStdin bool +} + +func agentBinary(agent string) string { + if agent == "cursor" { + return "cursor-agent" + } + return agent +} + +func assignsHarnessSessionID(agent string) bool { + return agent == "claude" || agent == "grok" +} + +func newHarnessSessionID(agent string) string { + if !assignsHarnessSessionID(agent) { + return "" + } + return uuid.NewString() +} + +func agentLaunchArgs(agent, promptPath, model, sessionID string) agentLaunchPlan { + var p agentLaunchPlan + switch agent { + case "codex": + p = agentLaunchPlan{Args: []string{"exec", "-"}, UseStdin: true} + case "gemini": + p = agentLaunchPlan{Args: []string{"-p", "-"}, UseStdin: true} + case "grok": + // -p/--prompt-file are single-turn. Seed the interactive session + // with a positional prompt that @-references the file. + p = agentLaunchPlan{ + Args: []string{fmt.Sprintf("Read and execute the task in @%s", promptPath)}, + UseStdin: false, + } + case "pi": + p = agentLaunchPlan{Args: []string{"-p"}, UseStdin: true} + case "opencode": + p = agentLaunchPlan{ + Args: []string{"run", "--file", promptPath, "Read and execute the attached task"}, + UseStdin: false, + } + case "claude": + p = agentLaunchPlan{ + Args: []string{fmt.Sprintf("Read and execute the task in @%s", promptPath)}, + UseStdin: false, + } + case "cursor": + p = agentLaunchPlan{Args: []string{promptPath}, UseStdin: false} + default: + p = agentLaunchPlan{UseStdin: true} + } + p.Args = withModelAndSession(agent, p.Args, model, sessionID) + return p +} + +func agentResumeArgs(agent, harnessID, model string) agentLaunchPlan { + var p agentLaunchPlan + switch agent { + case "claude": + p = agentLaunchPlan{Args: []string{"--resume", harnessID}} + case "grok": + p = agentLaunchPlan{Args: []string{"--resume", harnessID}} + case "codex": + p = agentLaunchPlan{Args: []string{"resume", harnessID}} + case "gemini": + p = agentLaunchPlan{Args: []string{"--resume", harnessID}} + case "cursor": + p = agentLaunchPlan{Args: []string{"--resume=" + harnessID}} + case "opencode": + p = agentLaunchPlan{Args: []string{"--session", harnessID}} + case "pi": + p = agentLaunchPlan{Args: []string{"--session", harnessID}} + default: + p = agentLaunchPlan{Args: []string{"--resume", harnessID}} + } + p.Args = withModelAndSession(agent, p.Args, model, "") + return p +} + +func withModelAndSession(agent string, args []string, model, sessionID string) []string { + flags := []string{} + switch agent { + case "claude": + if sessionID != "" { + flags = append(flags, "--session-id", sessionID) + } + if model != "" { + flags = append(flags, "--model", model) + } + return append(flags, args...) + case "grok": + if sessionID != "" { + flags = append(flags, "--session-id", sessionID) + } + if model != "" { + flags = append(flags, "--model", model) + } + return append(flags, args...) + case "codex": + if model == "" { + return args + } + if len(args) == 0 { + return []string{"-m", model} + } + out := make([]string, 0, len(args)+2) + out = append(out, args[0], "-m", model) + out = append(out, args[1:]...) + return out + default: + if model != "" { + flags = append(flags, "--model", model) + } + return append(flags, args...) + } +} + +// LaunchOptions is one agent exec (first run or resume). +type LaunchOptions struct { + Agent string + Prompt string + PromptPath string + Model string + SessionID string // pre-assigned harness id (claude, grok) +} + +// LaunchAgent execs the given CLI agent with the prompt. +func LaunchAgent(opts LaunchOptions) error { + plan := agentLaunchArgs(opts.Agent, opts.PromptPath, opts.Model, opts.SessionID) + return startAgent(opts.Agent, plan, opts.Prompt) +} + +// LaunchResume execs the harness resume command. No prompt is sent. +func LaunchResume(agent, harnessSessionID, model string) error { + if strings.TrimSpace(harnessSessionID) == "" { + return g.Error("missing harness session id") + } + plan := agentResumeArgs(agent, harnessSessionID, model) + return startAgent(agent, plan, "") +} + +func startAgent(agent string, plan agentLaunchPlan, prompt string) error { + binary, err := lookPath(agent) + if err != nil { + return err + } + + args := plan.Args + useStdin := plan.UseStdin + + procAttr := &os.ProcAttr{ + Files: []*os.File{os.Stdin, os.Stdout, os.Stderr}, + Env: os.Environ(), + } + if useStdin { + pr, pw, err := os.Pipe() + if err != nil { + return err + } + go func() { + defer pw.Close() + _, _ = pw.Write([]byte(prompt)) + }() + procAttr.Files = []*os.File{pr, os.Stdout, os.Stderr} + argv := append([]string{binary}, args...) + proc, err := os.StartProcess(binary, argv, procAttr) + if err != nil { + return err + } + state, err := proc.Wait() + pr.Close() + if err != nil { + return err + } + return finishAgentExit(state, agent) + } + + argv := append([]string{binary}, args...) + proc, err := os.StartProcess(binary, argv, procAttr) + if err != nil { + return err + } + state, err := proc.Wait() + if err != nil { + return err + } + return finishAgentExit(state, agent) +} + +// AgentExitError is returned when a launched CLI agent exits non-zero. +type AgentExitError struct { + ExitCode int + Agent string +} + +func (e *AgentExitError) Error() string { + if e == nil { + return "agent exited with error" + } + if e.Agent != "" { + return fmt.Sprintf("agent %q exited with code %d", e.Agent, e.ExitCode) + } + return fmt.Sprintf("agent exited with code %d", e.ExitCode) +} + +// ExitCodeOf returns (code, true) when err is or wraps an AgentExitError. +func ExitCodeOf(err error) (int, bool) { + var ae *AgentExitError + if errors.As(err, &ae) && ae != nil { + return ae.ExitCode, true + } + return 0, false +} + +func finishAgentExit(state *os.ProcessState, agent string) error { + if state.Success() { + return nil + } + code := 1 + if ws, ok := state.Sys().(syscall.WaitStatus); ok { + code = ws.ExitStatus() + } + return &AgentExitError{ExitCode: code, Agent: agent} +} + +func lookPath(name string) (string, error) { + bin := agentBinary(name) + if bin == "opencode" { + p, err := EnsureBinOpenCode() + if err != nil { + return "", g.Error(err, "agent %q not found on $PATH", name) + } + return p, nil + } + p, err := exec.LookPath(bin) + if err != nil { + return "", g.Error(err, "agent %q not found on $PATH", name) + } + return p, nil +} + +func harnessSessionRoot(agent string) string { + home := userHome() + switch agent { + case "claude": + return filepath.Join(home, ".claude", "projects") + case "codex": + return filepath.Join(home, ".codex", "sessions") + case "gemini": + return filepath.Join(home, ".gemini", "tmp") + case "cursor": + return filepath.Join(home, ".cursor") + case "opencode": + dataHome := os.Getenv("XDG_DATA_HOME") + if dataHome == "" { + dataHome = filepath.Join(home, ".local", "share") + } + return filepath.Join(dataHome, "opencode") + case "pi": + return filepath.Join(home, ".pi", "agent", "sessions") + case "grok": + return filepath.Join(home, ".grok", "sessions") + default: + return "" + } +} + +func snapshotHarnessFiles(agent string) map[string]time.Time { + root := harnessSessionRoot(agent) + out := map[string]time.Time{} + if root == "" { + return out + } + _ = filepath.Walk(root, func(p string, info os.FileInfo, err error) error { + if err != nil || info == nil || info.IsDir() { + return nil + } + rel, rerr := filepath.Rel(root, p) + if rerr != nil { + rel = p + } + out[rel] = info.ModTime() + return nil + }) + return out +} + +func discoverHarnessSessionID(agent string, before map[string]time.Time) string { + root := harnessSessionRoot(agent) + if root == "" { + return "" + } + if before == nil { + before = map[string]time.Time{} + } + var bestPath string + var bestTime time.Time + _ = filepath.Walk(root, func(p string, info os.FileInfo, err error) error { + if err != nil || info == nil || info.IsDir() { + return nil + } + rel, rerr := filepath.Rel(root, p) + if rerr != nil { + rel = p + } + prev, known := before[rel] + if known && !info.ModTime().After(prev) { + return nil + } + id := idFromSessionPath(p) + if id == "" { + return nil + } + if bestPath == "" || info.ModTime().After(bestTime) { + bestPath = p + bestTime = info.ModTime() + } + return nil + }) + if bestPath == "" { + return "" + } + return idFromSessionPath(bestPath) +} + +func idFromSessionPath(p string) string { + base := filepath.Base(p) + base = strings.TrimSuffix(base, filepath.Ext(base)) + if base == "" || strings.HasPrefix(base, ".") { + return "" + } + switch strings.ToLower(base) { + case "meta", "index", "config", "settings": + return "" + } + return base +} diff --git a/core/sling/assist/skills/agent-browser/CORE.md b/core/sling/assist/skills/agent-browser/CORE.md new file mode 100644 index 000000000..cf4433981 --- /dev/null +++ b/core/sling/assist/skills/agent-browser/CORE.md @@ -0,0 +1,516 @@ +--- +name: core +description: Core agent-browser usage guide. Read this before running any agent-browser commands. Covers the snapshot-and-ref workflow, navigating pages, interacting with elements (click, fill, type, select), extracting text and data, taking screenshots, managing tabs, handling forms and auth, waiting for content, running multiple browser sessions in parallel, and troubleshooting common failures. Use when the user asks to interact with a website, fill a form, click something, extract data, take a screenshot, log into a site, test a web app, or automate any browser task. +allowed-tools: Bash(agent-browser:*), Bash(npx agent-browser:*) +--- + +# agent-browser core + +Fast browser automation CLI for AI agents. Chrome/Chromium via CDP, no Playwright or Puppeteer dependency. Accessibility-tree snapshots with compact `@eN` refs let agents interact with pages in ~200-400 tokens instead of parsing raw HTML. + +Most normal web tasks (navigate, read, click, fill, extract, screenshot) are covered here. Load a specialized skill when the task falls outside browser web pages — see [When to load another skill](#when-to-load-another-skill). + +## The core loop + +```bash +agent-browser open # 1. Open a page +agent-browser snapshot -i # 2. See what's on it (interactive elements only) +agent-browser click @e3 # 3. Act on refs from the snapshot +agent-browser snapshot -i # 4. Re-snapshot after any page change +``` + +Refs (`@e1`, `@e2`, ...) are assigned fresh on every snapshot. They become **stale the moment the page changes** — after clicks that navigate, form submits, dynamic re-renders, dialog opens. Always re-snapshot before your next ref interaction. + +## Always use your own session + +Before your first command, set a named session for the whole task: + +```bash +export AGENT_BROWSER_SESSION="$(agent-browser session id --scope worktree --prefix task)" +``` + +The default (unnamed) session is a single shared browser: it is shared with every other agent on the machine and it persists across conversations, so working in it can hijack another agent's page mid-task or navigate away from something the human left open. Every example below assumes a named session is active. See [Run multiple browsers in parallel](#run-multiple-browsers-in-parallel) and `references/session-management.md`. + +## Quickstart + +```bash +# Install once +npm i -g agent-browser && agent-browser install + +# Linux hosts can install required browser libraries too +agent-browser install --with-deps + +# Take a screenshot of a page +agent-browser open https://example.com +agent-browser screenshot home.png +agent-browser close + +# Search, click a result, and capture it +agent-browser open https://duckduckgo.com +agent-browser snapshot -i # find the search box ref +agent-browser fill @e1 "agent-browser cli" +agent-browser press Enter +agent-browser wait --load networkidle +agent-browser snapshot -i # refs now reflect results +agent-browser click @e5 # click a result +agent-browser screenshot result.png +``` + +The browser stays running across commands so these feel like a single session. By default, an inactive daemon saves configured restore state, closes its headless browser, and exits after one hour; the next command starts it again. Without `--restore` or another restore key, shutdown discards transient browser state and open tabs. Dashboard mouse, keyboard, and touch input count as activity. Headed browsers, Safari and iOS WebDriver sessions, and user-attached browsers are exempt from the default; provider-owned cloud browsers are not. Use `--idle-timeout