Repository navigation
Expand file tree
/
Copy pathinit.core.template.sql
More file actions
3065 lines (2927 loc) · 182 KB
/
Copy pathinit.core.template.sql
File metadata and controls
3065 lines (2927 loc) · 182 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
-- ─────────────────────────────────────────────────────────────────────────────
-- init.core.sql — the read-only half: CDC and snapshots, and nothing that writes.
--
-- Runnable on its own. That is the point: a deployment that only *reads* PostgreSQL
-- should never meet RLS, tombstone guards or tenant scoping, and a newcomer reading this
-- file should not have to skip past them to find out how a change becomes an event.
--
-- envsubst < init.core.template.sql | psql -v ON_ERROR_STOP=1 …
--
-- The write path is `init.write.template.sql`, appended after this one:
--
-- cat init.core.template.sql init.write.template.sql | envsubst | psql …
--
-- ⚠️ It is appended, never merged. The read-only setup is this file *unmodified*, so the
-- two profiles cannot drift: there is no commented-out block here that the other profile
-- uncomments, and no second copy of the publication or the DDL triggers.
--
-- ⚠️ Nothing here may reference ${POSTGRES_WRITER_USER}. That role is created by the
-- write half, so a reference from this file breaks the read-only profile — which is the
-- one invariant that keeps the split honest.
-- ─────────────────────────────────────────────────────────────────────────────
-- Every object below carries a `-- <what it is> — see <doc> §<n>` line above it.
--
-- The reference points at the section that explains *why* the object exists, because this
-- file is where someone is standing when they need that: reading a trigger and wondering
-- whether it can be dropped. The docs describe behaviour a DBA has to preserve;
-- `zebridge_gc_watermark` in particular carries a 🔴 DBA callout in PROTOCOL.md §7.5,
-- because dropping it or unpublishing it removes a client guarantee without failing
-- anything.
--
-- Keep the reference when moving or rewriting an object. A comment that says what the code
-- already says is noise; one that says where the reasoning lives is the only link between
-- the two files.
--
-- The link runs both ways: **PROTOCOL.md §8b** lists every object created here, what it is,
-- and what breaks if it is removed. Read this file and you can find the reasoning; read the
-- docs and you can find the code. Adding an object means adding a row there too — a
-- one-directional link is the one that goes stale.
-- ─────────────────────────────────────────────────────────────────────────────
DO $$
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${POSTGRES_READER_USER}') THEN
CREATE USER ${POSTGRES_READER_USER} WITH PASSWORD '${POSTGRES_READER_PASSWORD}';
ALTER USER ${POSTGRES_READER_USER} WITH REPLICATION;
END IF;
-- The reader's connection BUDGET, DB-enforced (outside the IF: applies to
-- pre-existing roles on re-apply). Steady state is ~4 backends (producer,
-- snapshot worker, boot passes) — the replication stream rides a walsender
-- slot, not a regular backend. 10 leaves boot-burst headroom while making it
-- impossible for a bridge bug to eat the cluster's default 100.
ALTER ROLE ${POSTGRES_READER_USER} CONNECTION LIMIT 10;
IF true THEN
END IF;
END;
$$;
GRANT CONNECT ON DATABASE ${TARGET_DB} TO ${POSTGRES_READER_USER};
GRANT USAGE ON SCHEMA public TO ${POSTGRES_READER_USER};
-- Grant SELECT on existing tables
GRANT SELECT ON ALL TABLES IN SCHEMA public TO ${POSTGRES_READER_USER};
-- Ensure bridge user gets SELECT permissions on all FUTURE tables created by the DB owner
ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT ON TABLES TO ${POSTGRES_READER_USER};
-- Put the tenant column inside the table's REPLICA IDENTITY: a unique index on
-- (tenant, primary key), made the identity. Returns the column list it set (or, with
-- dry_run, would set), NULL when the identity already covers the tenant.
--
-- ⚠️ Every tenant table needs it, read-only or writable. A DELETE carries only the
-- identity columns — by default the primary key alone — so without the tenant it cannot
-- be routed to a tenant subject, and PostgreSQL refuses UPDATE and DELETE once a
-- publication filters on the tenant. The bridge refuses such a table at boot
-- (`tenant_not_in_replica_identity`). Lives in core, not init.write, because a
-- read-only-profile database needs it too; it used to exist only on the write path,
-- and a read-only tenant table came out of zebridge_enable refused (2026-09-25).
-- The caller checks that the tenant column exists and is NOT NULL.
CREATE OR REPLACE FUNCTION public.zebridge_tenant_replica_identity(
tbl regclass, tenant_col name, dry_run boolean DEFAULT false
) RETURNS text AS $$
DECLARE
ri_kind "char";
covered boolean;
pk_cols name[];
idx_name name;
col_list text;
BEGIN
SELECT relreplident INTO ri_kind FROM pg_class WHERE oid = tbl;
IF ri_kind = 'f' THEN
RETURN NULL; -- FULL carries every column
END IF;
SELECT coalesce(bool_or(a.attname = tenant_col), false) INTO covered
FROM pg_index i
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = ANY(i.indkey)
WHERE i.indrelid = tbl
AND ((ri_kind = 'd' AND i.indisprimary) OR (ri_kind = 'i' AND i.indisreplident));
IF covered THEN
RETURN NULL;
END IF;
SELECT array_agg(a.attname ORDER BY k.ord) INTO pk_cols
FROM pg_index i
JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS k(attnum, ord) ON true
JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum
WHERE i.indrelid = tbl AND i.indisprimary;
IF pk_cols IS NULL THEN
RAISE EXCEPTION '% has no primary key, so no replica identity can cover %', tbl, tenant_col;
END IF;
idx_name := (SELECT relname FROM pg_class WHERE oid = tbl) || '_zb_ri';
col_list := quote_ident(tenant_col) || ', ' ||
(SELECT string_agg(quote_ident(c), ', ') FROM unnest(pk_cols) c);
IF NOT dry_run THEN
-- `name` keeps 63 bytes, so the suffix of a 63-character table is cut off and the
-- "index" is the table itself: CREATE ... IF NOT EXISTS skips it, and the ALTER below
-- fails with '"<table>" is not an index', which names nothing to do (measured
-- 2026-10-26). Say what is wrong while there is still nothing to undo.
IF idx_name = (SELECT relname FROM pg_class WHERE oid = tbl) THEN
RAISE EXCEPTION 'table name % is % bytes: the replica-identity index name derived from it (<table>_zb_ri) is cut to the table''s own name. Rename the table to at most 62 bytes, or make the tenant part of the replica identity yourself (a unique index on (%, primary key) and ALTER TABLE ... REPLICA IDENTITY USING INDEX)',
tbl, octet_length(idx_name), tenant_col;
END IF;
EXECUTE format('CREATE UNIQUE INDEX IF NOT EXISTS %I ON %s (%s)', idx_name, tbl, col_list);
EXECUTE format('ALTER TABLE %s REPLICA IDENTITY USING INDEX %I', tbl, idx_name);
RAISE NOTICE 'replica identity of % moved to (%) so its rows route by tenant', tbl, col_list;
END IF;
RETURN col_list;
END;
$$ LANGUAGE plpgsql;
-- Scope the reader's SELECT by tenant, for the snapshot path — see PROTOCOL.md "The
-- Connection Flow" Step 0, NOTES.md §1.12 part 1
--
-- ⚠️ RLS is evaluated against a query, and logical decoding has no query — structurally
-- invisible to CDC (§1.8: a policy returned 1 row to a SELECT while the WAL still carried
-- 2). CDC's tenant correctness is the subject's job and stays that way; this function
-- only ever affects a real SELECT, which today means the snapshot listener's own
-- connection setting `zb.tenant` before it queries.
--
-- Lives in core, not write: it needs nothing from `zebridge_user_tenants` (that table
-- resolves a WRITER's own tenant when omitted from a write — a write-path concern only)
-- and nothing from the writer role. A read-only deployment (`init.core.template.sql`
-- alone, `ZB_PROFILE=readonly`) can scope its snapshots by tenant with no write profile
-- installed at all.
--
-- ⚠️ `zb_reader_all` cannot simply gain a second, more restrictive policy alongside it:
-- PostgreSQL ORs every applicable permissive policy for one role/command together, so a
-- second policy next to an existing `USING (true)` changes nothing — the permissive one
-- always wins. The fix is making the ONE policy conditional instead: wholesale when
-- `zb.tenant` is unset, filtered when it is. The replication connection never calls
-- `set_config('zb.tenant', ...)` — and RLS does not apply to it regardless — so this is a
-- no-op for CDC and preserves exactly what `zb_reader_all USING (true)` already gave every
-- caller that never sets it: the reader sees everything by default, and only narrows on
-- an explicit opt-in.
--
-- ⚠️ `coalesce(..., '') = ''`, NOT `IS NULL` — the same trap `zb_sweeper`'s policy already
-- documents for `zb.principal`, measured again building this one: `current_setting(...,
-- true)` returns an empty string, not SQL NULL, for a GUC that has never been set on this
-- role/session (not only after a `SET LOCAL` resets at COMMIT). `IS NULL` never matched —
-- the "wholesale when unset" branch was dead code, and every reader saw zero rows instead
-- of everything until this was measured directly against the running database.
--
-- ⚠️ The bridge sets `zb.tenant` from the AUTHENTICATED REQUEST SUBJECT, never resolves it
-- through `zebridge_user_tenants` for this purpose — two derivations of one fact can
-- disagree (subject says acme, mapping says globex), and only one of them cannot disagree
-- with itself.
CREATE OR REPLACE FUNCTION public.zebridge_scope_reads_by_tenant(tbl regclass, tenant_col name)
RETURNS void AS $$
DECLARE
t text := tbl::text;
BEGIN
IF NOT EXISTS (SELECT 1 FROM pg_attribute
WHERE attrelid = tbl AND attname = tenant_col
AND attnum > 0 AND NOT attisdropped AND attnotnull) THEN
RAISE EXCEPTION 'tenant column %.% must exist and be NOT NULL', tbl, tenant_col;
END IF;
-- A tenant table's CDC routes by the tenant, reads scoped or not.
PERFORM public.zebridge_tenant_replica_identity(tbl, tenant_col);
EXECUTE format('ALTER TABLE %s ENABLE ROW LEVEL SECURITY', tbl);
EXECUTE format('DROP POLICY IF EXISTS zb_reader_all ON %s', tbl);
EXECUTE format(
'CREATE POLICY zb_reader_all ON %s FOR SELECT TO %I'
-- Same open-tenant carve-out zb_tenant_write already has (init.write.template.sql):
-- a row mapped to the OPEN tenant is readable by every principal, not just whoever
-- has zb.tenant set to it. Without this, CDC (subject-routed, RLS-blind) delivers
-- an open row to every client while a snapshot never grants the SELECT that would
-- have put it there in the first place — audience says everyone, contents says only
-- the open tenant itself, and a client's local copy of the row depends on whether it
-- happened to be connected for a live CDC write, not on what its snapshot returned.
' USING (coalesce((SELECT current_setting(''zb.tenant'', true)), '''') = '''' OR %I::text = (SELECT current_setting(''zb.tenant'', true)) OR %I::text = ''${OPEN_TENANT}'')',
tbl, '${POSTGRES_READER_USER}', tenant_col, tenant_col);
RAISE NOTICE 'reads on % now filtered by % when zb.tenant is set — CDC is unaffected (RLS does not apply to replication); a client-scoped connection sets zb.tenant to scope it',
tbl, tenant_col;
END;
$$ LANGUAGE plpgsql;
-- The widest row actually stored, in the units CDC packs — see PROTOCOL.md §9
--
-- ⚠️ `octet_length`, never `pg_column_size`. The latter reports the *stored, compressed*
-- size: a 40 KB run of one character compresses to almost nothing, so a check built on it
-- reports a table as safe while CDC cannot carry its widest row. Wrong direction for a
-- safety bound.
--
-- ⚠️ Cheap for the types that matter: `octet_length` on text/varchar/bytea reads the length
-- out of the TOAST pointer without fetching the value — measured at 2 shared buffers for a
-- 50 MB table. `jsonb`/arrays need `::text`, which does materialise; they are usually small.
CREATE OR REPLACE FUNCTION public.zebridge_widest_row(tbl regclass)
RETURNS bigint AS $$
DECLARE
expr text;
result bigint;
BEGIN
SELECT coalesce(string_agg(
CASE
WHEN a.atttypid IN ('text'::regtype, 'bytea'::regtype, 'varchar'::regtype)
THEN format('coalesce(octet_length(%I),0)', a.attname)
WHEN t.typcategory = 'A'
OR a.atttypid IN ('json'::regtype, 'jsonb'::regtype, 'xml'::regtype)
THEN format('coalesce(octet_length(%I::text),0)', a.attname)
-- Fixed-width columns: a flat allowance rather than a per-type table. The
-- result is a floor on the row's size, which is what a guard wants.
ELSE '8'
END, ' + '), '0')
INTO expr
FROM pg_attribute a
JOIN pg_type t ON t.oid = a.atttypid
WHERE a.attrelid = tbl AND a.attnum > 0 AND NOT a.attisdropped;
EXECUTE format('SELECT coalesce(max(%s),0)::bigint FROM %s', expr, tbl) INTO result;
RETURN result;
END;
$$ LANGUAGE plpgsql STABLE;
-- Column DEFAULTs that alone exceed a byte budget — see PROTOCOL.md §9
--
-- ⚠️ The case `zebridge_widest_row` cannot see: a table with a huge column DEFAULT and no
-- rows yet measures as perfectly safe, and then the *first* insert — a 258-byte mutation
-- that passed every ingress guard — stores a 50 KB row and suspends the table for every
-- client. Measured. The default is the hazard; the row is only where it shows up.
--
-- ⚠️ Two guards make evaluating a default safe:
--
-- * only **variable-width** columns are evaluated. A `nextval()` default lives on an
-- integer column, so it is never reached and no sequence is burned;
-- * the function is **STABLE**, which makes PostgreSQL refuse any write the expression
-- attempts — a side-effecting default raises, and the handler below skips it rather
-- than failing the check.
CREATE OR REPLACE FUNCTION public.zebridge_oversized_defaults(tbl regclass, budget bigint)
RETURNS TABLE(col text, bytes bigint) AS $$
DECLARE
r record;
n bigint;
BEGIN
FOR r IN
SELECT a.attname AS name, pg_get_expr(ad.adbin, ad.adrelid) AS expr
FROM pg_attribute a
JOIN pg_attrdef ad ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum
JOIN pg_type t ON t.oid = a.atttypid
WHERE a.attrelid = tbl AND a.attnum > 0 AND NOT a.attisdropped
AND (a.atttypid IN ('text'::regtype, 'bytea'::regtype, 'varchar'::regtype)
OR t.typcategory = 'A'
OR a.atttypid IN ('json'::regtype, 'jsonb'::regtype, 'xml'::regtype))
LOOP
BEGIN
EXECUTE format('SELECT coalesce(octet_length((%s)::text),0)::bigint', r.expr) INTO n;
EXCEPTION WHEN OTHERS THEN
-- A default that cannot be evaluated read-only is not a finding. Saying
-- nothing beats refusing to start over an expression we declined to run.
CONTINUE;
END;
IF n >= budget THEN
col := r.name;
bytes := n;
RETURN NEXT;
END IF;
END LOOP;
END;
$$ LANGUAGE plpgsql STABLE;
-- keeps the tracker's own rows out of the DDL feed — see PROTOCOL.md §5. Defined before
-- any table it names is created: on an upgrade the DDL trigger is already installed, and
-- a table created before its name is listed enters the feed as an application table.
CREATE OR REPLACE FUNCTION public.zebridge_is_internal_table(tbl text)
RETURNS boolean AS $$
-- The namespace, not a list (OPERATIONS "Reserved names"; zebridge_enable refuses the prefix): every
-- zebridge_ table is the bridge's and is never client-replicated, except the GC floor, which every client
-- reads. That includes the principal->tenant roster, zebridge_user_tenants: BRIDGE INPUT (its CDC feeds
-- $KV.tenants), never client-replicated, since replicating it would disclose the roster the per-key
-- $KV.tenants grant exists to protect (NOTES.md §1.12 part 3). The bridge's own Zig rule
-- (preflight.isInternalTable) asks a different question and checks the roster like a user table.
-- A name list here had to be remembered at each new table and drifted; enable_scoping check 11 and
-- init_sql.zig test this against every table the init SQL creates.
-- schema_migrations is Ecto's / ActiveRecord's migration bookkeeping.
SELECT (left(tbl, 9) = 'zebridge_' AND tbl <> 'zebridge_gc_watermark') OR tbl = 'schema_migrations';
$$ LANGUAGE sql IMMUTABLE;
-- THE CATALOGUE comes first: the guard below and every registration in this file
-- read it. Public tables used to live in a separate `zebridge_public_tables`; that
-- was a strict projection of this relation (`tenant_col IS NULL`) and is gone — the
-- deliberate "published to everyone" decision is the `public_reason` column, which
-- the CHECK makes mandatory exactly when a row is public.
-- THE CATALOGUE — one row per replicated table, the disjoint union of public and
-- tenant-scoped as one nullable column (NOTES.md, the static-residue endgame).
-- Written by `zebridge_enable` in the SAME transaction as the guards it installs
-- with these very columns, so nothing can disagree: the bridge and the sweeper read
-- their per-table rules only from here (no env override), and the generation
-- producer reads it per tick.
-- The CHECK absorbs the publication guard's core law into the schema itself: a
-- row is tenant-scoped or publicly justified — never silently neither.
CREATE TABLE IF NOT EXISTS public.zebridge_catalogue (
tbl text PRIMARY KEY,
tenant_col name, -- NULL = public; NOT NULL = tenant-scoped
public_reason text,
version_col name NOT NULL DEFAULT 'updated_at',
tombstone_col name,
tiebreak_col name,
generations boolean NOT NULL DEFAULT true,
-- The re-seed lever (NOTES §10df): bumped by zebridge_reseed(), carried in the
-- schema descriptor and the chain manifest; a client whose stored epoch is lower
-- forgets its watermark and seeds a fresh full. The producer builds that full.
seed_epoch integer NOT NULL DEFAULT 0,
-- The operator's acceptance of physical deletes on a writable table with no
-- tombstone (zebridge_enable's allow_physical_deletes), kept so zebridge_check
-- reads it too: the catalogue is the intent, nothing has to be repeated.
allow_physical_deletes boolean NOT NULL DEFAULT false,
CHECK ((tenant_col IS NULL) <> (public_reason IS NULL))
);
ALTER TABLE public.zebridge_catalogue ADD COLUMN IF NOT EXISTS allow_physical_deletes boolean NOT NULL DEFAULT false;
-- A table's incarnations (one lifetime under its name, from a CREATE to the DROP that
-- ends it): `incarnation_relid` the live table's OID, NULL while there is none;
-- `incarnation` how many lifetimes this name has had. Kept by zebridge_observe_incarnation
-- alone. The OID is the evidence of a replacement, the counter is the identity — and the
-- identity must never repeat (chain objects are named with it), so it lives in its OWN
-- table: a catalogue row can be deleted and inserted again, and a counter kept on it
-- started over at 1. Rows here are never deleted.
-- `seed_epoch` the highest seed epoch the name has had: the catalogue's own column is
-- the published copy, and a catalogue row inserted again starts from this floor (the
-- trigger below), never from 0 — a replica compares epochs, so an epoch that came back
-- to its own value told it nothing of the replacement in between.
CREATE TABLE IF NOT EXISTS public.zebridge_incarnations (
tbl text PRIMARY KEY,
incarnation_relid oid,
incarnation bigint NOT NULL DEFAULT 0,
seed_epoch integer NOT NULL DEFAULT 0
);
ALTER TABLE public.zebridge_incarnations ADD COLUMN IF NOT EXISTS seed_epoch integer NOT NULL DEFAULT 0;
GRANT SELECT ON public.zebridge_incarnations TO ${POSTGRES_READER_USER};
-- The catalogue carried these two columns for a day (dev): their values move here once.
DO $$ BEGIN
IF EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = 'zebridge_catalogue' AND column_name = 'incarnation') THEN
INSERT INTO public.zebridge_incarnations (tbl, incarnation_relid, incarnation)
SELECT tbl, incarnation_relid, incarnation FROM public.zebridge_catalogue WHERE incarnation > 0
ON CONFLICT (tbl) DO NOTHING;
ALTER TABLE public.zebridge_catalogue DROP COLUMN incarnation, DROP COLUMN incarnation_relid;
END IF;
END $$;
GRANT SELECT ON public.zebridge_catalogue TO ${POSTGRES_READER_USER};
-- The bridge's runtime verdicts, made queryable (NOTES §10cf). A row_too_large
-- suspension lived only in the bridge's MEMORY plus the client-facing KV descriptor —
-- an operator who missed the log line had no psql answer to "which tables are
-- refused?". This table is that answer:
--
-- SELECT tbl, reason, since FROM zebridge_suspensions;
--
-- Written by the bridge through the SECURITY DEFINER setters below (the
-- zebridge_register_limits pattern — the reader role executes a narrow function
-- rather than holding write privilege on the table). Semantics mirror the in-memory
-- registry exactly: a row per refused table, set on every refusal transition, deleted
-- on every lift — and emptied WHOLESALE at boot, because the registry is memory and
-- a restart forgets (§13/§10bw: the quarantine is lazily re-earned on the next touch,
-- so a surviving row would lie in the opposite direction).
--
-- Its OWN table, deliberately NOT in the publication (NOTES §10cy). It used to be two
-- columns on zebridge_catalogue, and the catalogue rides the publication so the bridge
-- reloads on every catalogue row: each refuse and each lift therefore came back to the
-- bridge as a catalogue move and cost a reload that changed nothing — three reloads
-- per live table birth, one useful. A mirror write is not a declaration; it must not
-- look like one.
CREATE TABLE IF NOT EXISTS public.zebridge_suspensions (
tbl name PRIMARY KEY,
reason text NOT NULL,
since timestamptz NOT NULL DEFAULT now()
);
GRANT SELECT ON public.zebridge_suspensions TO ${POSTGRES_READER_USER};
CREATE OR REPLACE FUNCTION public.zebridge_set_suspended(p_tbl text, p_reason text)
RETURNS void AS $$
-- p_reason NULL = lifted. EVERY refused table records a row, catalogued or not
-- (§10dk): `no_cdc_subject` IS the table with no catalogue row, and a projection
-- that omitted it answered "which tables are refused?" with one missing.
DELETE FROM public.zebridge_suspensions WHERE tbl = p_tbl AND p_reason IS NULL;
INSERT INTO public.zebridge_suspensions (tbl, reason)
SELECT p_tbl, p_reason
WHERE p_reason IS NOT NULL
ON CONFLICT (tbl) DO UPDATE SET reason = EXCLUDED.reason, since = now();
$$ LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, public;
CREATE OR REPLACE FUNCTION public.zebridge_clear_suspensions()
RETURNS void AS $$
DELETE FROM public.zebridge_suspensions;
$$ LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, public;
GRANT EXECUTE ON FUNCTION public.zebridge_set_suspended(text, text) TO ${POSTGRES_READER_USER};
GRANT EXECUTE ON FUNCTION public.zebridge_clear_suspensions() TO ${POSTGRES_READER_USER};
-- Where the read side's WAL is, primary or standby (NOTES §10cz). DATABASE_READER_URL
-- may point at a hot standby (PostgreSQL ≥ 16 decodes logically there): on it
-- pg_current_wal_lsn() raises "recovery is in progress", and the honest head is what
-- has been REPLAYED. One function, replicated with the rest of the schema, so the
-- bridge asks the same question on either and carries no mode flag for it.
CREATE OR REPLACE FUNCTION public.zebridge_wal_head() RETURNS pg_lsn AS $$
SELECT CASE WHEN pg_is_in_recovery() THEN pg_last_wal_replay_lsn()
ELSE pg_current_wal_lsn() END;
$$ LANGUAGE sql VOLATILE;
GRANT EXECUTE ON FUNCTION public.zebridge_wal_head() TO ${POSTGRES_READER_USER};
-- Force every client to re-seed a table — and every table that REFERENCES it,
-- transitively (NOTES §10df). The one lever for what CDC cannot carry: a volatile
-- column default (now(), gen_random_uuid()), a primary-key re-key, a hand-fixed
-- dataset. A parent re-keyed invalidates the foreign-key values its children hold,
-- and PostgreSQL's rewrite of those children is just as invisible to decoding, so
-- the closure is taken here, once, rather than remembered by hand. Returns what it
-- bumped. SECURITY DEFINER on the zebridge_set_suspended pattern (§10dh): the
-- bridge's reader role pulls it when a suspension that DROPPED events lifts — the
-- rows written meanwhile reached no replica, and only a fresh full brings them —
-- and the DDL trigger pulls it on a re-key whatever role migrates. EXECUTE is
-- revoked from PUBLIC and granted to the reader only; the writer has no use for it.
CREATE OR REPLACE FUNCTION public.zebridge_reseed(p_table regclass)
RETURNS TABLE (tbl text, seed_epoch integer) AS $$
WITH RECURSIVE closure(oid) AS (
SELECT p_table::oid
UNION
SELECT con.conrelid FROM pg_constraint con JOIN closure ON con.confrelid = closure.oid
WHERE con.contype = 'f'
)
-- Once per table per TRANSACTION, closure members included: a transaction-local
-- setting marks every table this bumps, and a later call in the same transaction
-- (a second parent re-keyed, the child's own re-type) skips the marked ones. A
-- call outside a transaction marks nothing that outlives it. Measured before the
-- marker: a child of two re-keyed parents was bumped three times in one migration.
, bumped AS (
UPDATE public.zebridge_catalogue c
SET seed_epoch = c.seed_epoch + 1
FROM closure JOIN pg_class r ON r.oid = closure.oid
WHERE c.tbl = r.relname
AND current_setting('zebridge.reseeded_' || md5(c.tbl), true) IS DISTINCT FROM '1'
RETURNING c.tbl, c.seed_epoch
)
SELECT b.tbl, b.seed_epoch
FROM bumped b, LATERAL set_config('zebridge.reseeded_' || md5(b.tbl), '1', true) AS marked;
$$ LANGUAGE sql SECURITY DEFINER SET search_path = pg_catalog, public;
REVOKE ALL ON FUNCTION public.zebridge_reseed(regclass) FROM PUBLIC;
GRANT EXECUTE ON FUNCTION public.zebridge_reseed(regclass) TO ${POSTGRES_READER_USER};
-- A table's incarnation, observed (the recovery model's §1): ONE operation, called by the
-- DDL triggers inside the DDL's own transaction (prompt), by zebridge_enable, and by the
-- generation producer when a pair finds the OID moved (the backstop for whatever the
-- triggers did not see). "Already handled" is a comparison, never a count: observing the
-- same OID twice changes nothing, so every caller may call it.
-- p_relid NULL the table is gone: the incarnation ends (incarnation_relid NULL), nothing re-seeds;
-- same OID nothing;
-- another OID a new incarnation: the counter moves, and — when a previous one existed
-- — the seed epoch moves ONCE (the per-transaction marker zebridge_reseed
-- uses), so every replica of the old table re-seeds. Not once per tenant
-- pair: the producer used to bump it from each pair's build (0 -> 3 for
-- three pairs, recreate_recovery.py check 1).
-- A table not catalogued is not observed. The first observation of a table records it
-- without a re-seed (no replica can hold another incarnation of it).
-- Fresh observations only: an OID is applied only while it IS the live table's, read
-- after the row lock (so a recreate that held the lock is committed and visible). A
-- caller that read the OID earlier — the producer reads it, then calls — and lost a race
-- with another recreate supplies a stale OID: it changes nothing. Without this a delayed
-- observation of an earlier OID moved the incarnation again, bumped the epoch again and
-- recorded an OID that was no longer the table's. Likewise NULL (a DROP) applies only
-- while no table of the name exists.
DROP FUNCTION IF EXISTS public.zebridge_observe_incarnation(text, oid);
CREATE FUNCTION public.zebridge_observe_incarnation(p_tbl text, p_relid oid)
RETURNS TABLE (incarnation bigint, status text) AS $$
-- status: 'new' (a new incarnation recorded), 'current' (the OID is already the
-- recorded one), 'ended' (a DROP ended it), 'stale' (the OID — or NULL — does not
-- describe the live table: nothing changed; a caller that read it must read again).
DECLARE
prev_relid oid;
prev_inc bigint;
now_inc bigint;
reseed boolean;
live oid;
BEGIN
-- All identity writers take the catalogue lock first, including epoch recording.
PERFORM 1 FROM public.zebridge_catalogue c WHERE c.tbl = p_tbl FOR UPDATE;
IF NOT FOUND THEN
RETURN;
END IF;
INSERT INTO public.zebridge_incarnations (tbl) VALUES (p_tbl) ON CONFLICT (tbl) DO NOTHING;
SELECT i.incarnation_relid, i.incarnation INTO prev_relid, prev_inc
FROM public.zebridge_incarnations i WHERE i.tbl = p_tbl FOR UPDATE;
live := to_regclass(format('%I.%I', 'public', p_tbl))::oid;
IF p_relid IS DISTINCT FROM live THEN
RETURN QUERY SELECT prev_inc, 'stale'::text;
RETURN;
END IF;
IF p_relid IS NULL THEN
IF prev_relid IS NOT NULL THEN
UPDATE public.zebridge_incarnations i SET incarnation_relid = NULL WHERE i.tbl = p_tbl;
END IF;
RETURN QUERY SELECT prev_inc, 'ended'::text;
RETURN;
END IF;
IF prev_relid = p_relid THEN
RETURN QUERY SELECT prev_inc, 'current'::text;
RETURN;
END IF;
UPDATE public.zebridge_incarnations i
SET incarnation_relid = p_relid, incarnation = i.incarnation + 1
WHERE i.tbl = p_tbl
RETURNING i.incarnation INTO now_inc;
reseed := prev_inc > 0 AND current_setting('zebridge.reseeded_' || md5(p_tbl), true) IS DISTINCT FROM '1';
IF reseed THEN
UPDATE public.zebridge_catalogue c SET seed_epoch = c.seed_epoch + 1 WHERE c.tbl = p_tbl;
PERFORM set_config('zebridge.reseeded_' || md5(p_tbl), '1', true);
END IF;
RETURN QUERY SELECT now_inc, 'new'::text;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public;
REVOKE ALL ON FUNCTION public.zebridge_observe_incarnation(text, oid) FROM PUBLIC;
GRANT EXECUTE ON FUNCTION public.zebridge_observe_incarnation(text, oid) TO ${POSTGRES_READER_USER};
-- Every catalogued table that exists has an incarnation. Without this, a table catalogued
-- before the columns existed sat at 0, and its FIRST recreate counted as a first
-- observation: incarnation 1, no re-seed — its replicas kept the old table's rows. Only
-- tables never observed are touched (a row the epoch floor alone created is at 0), so
-- re-applying this file changes nothing.
INSERT INTO public.zebridge_incarnations AS i (tbl, incarnation_relid, incarnation)
SELECT c.tbl, to_regclass(format('%I.%I', 'public', c.tbl))::oid, 1
FROM public.zebridge_catalogue c
WHERE to_regclass(format('%I.%I', 'public', c.tbl)) IS NOT NULL
ON CONFLICT (tbl) DO UPDATE SET incarnation_relid = EXCLUDED.incarnation_relid, incarnation = 1
WHERE i.incarnation = 0;
-- The seed epoch never goes back for a name. Every write of a catalogue row's epoch — an
-- enable inserting it, zebridge_reseed, an observed replacement — passes through this
-- floor: an insert or a lowering is raised to the highest epoch the name has had, and
-- that highest is recorded. Without it a removed and re-inserted catalogue row started
-- at 0, and the replacement's single move brought it back to the epoch its replicas
-- already held (1 -> 0 -> 1): a replica that missed the DROP kept the old table's rows.
CREATE OR REPLACE FUNCTION public.zebridge_catalogue_epoch_floor()
RETURNS trigger AS $$
BEGIN
-- BEFORE INSERT also runs before an ON CONFLICT row is locked: only read here.
NEW.seed_epoch := greatest(NEW.seed_epoch, coalesce(
(SELECT i.seed_epoch FROM public.zebridge_incarnations i WHERE i.tbl = NEW.tbl), 0));
RETURN NEW;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public;
REVOKE ALL ON FUNCTION public.zebridge_catalogue_epoch_floor() FROM PUBLIC;
CREATE OR REPLACE FUNCTION public.zebridge_catalogue_epoch_record()
RETURNS trigger AS $$
DECLARE
floor_epoch bigint;
BEGIN
-- AFTER INSERT/UPDATE: the catalogue row is now held before the identity row.
INSERT INTO public.zebridge_incarnations AS i (tbl, seed_epoch) VALUES (NEW.tbl, NEW.seed_epoch)
ON CONFLICT (tbl) DO UPDATE SET seed_epoch = greatest(i.seed_epoch, EXCLUDED.seed_epoch)
RETURNING i.seed_epoch INTO floor_epoch;
-- An INSERT can wait behind a concurrent DELETE after its BEFORE trigger read
-- the floor. Recheck after locking the identity; the catalogue row is ours now.
IF NEW.seed_epoch < floor_epoch THEN
UPDATE public.zebridge_catalogue SET seed_epoch = floor_epoch
WHERE tbl = NEW.tbl AND seed_epoch < floor_epoch;
END IF;
RETURN NEW;
END;
$$ LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, public;
REVOKE ALL ON FUNCTION public.zebridge_catalogue_epoch_record() FROM PUBLIC;
-- An upgrade: the floor starts at the epochs the catalogue holds now.
INSERT INTO public.zebridge_incarnations AS i (tbl, seed_epoch)
SELECT c.tbl, c.seed_epoch FROM public.zebridge_catalogue c
ON CONFLICT (tbl) DO UPDATE SET seed_epoch = greatest(i.seed_epoch, EXCLUDED.seed_epoch);
DROP TRIGGER IF EXISTS zebridge_catalogue_epoch_floor_t ON public.zebridge_catalogue;
CREATE TRIGGER zebridge_catalogue_epoch_floor_t
BEFORE INSERT OR UPDATE OF tbl, seed_epoch ON public.zebridge_catalogue
FOR EACH ROW EXECUTE FUNCTION public.zebridge_catalogue_epoch_floor();
DROP TRIGGER IF EXISTS zebridge_catalogue_epoch_record_t ON public.zebridge_catalogue;
CREATE TRIGGER zebridge_catalogue_epoch_record_t
AFTER INSERT OR UPDATE OF tbl, seed_epoch ON public.zebridge_catalogue
FOR EACH ROW EXECUTE FUNCTION public.zebridge_catalogue_epoch_record();
-- The writer's grant lives in init.write.template.sql, NOT here: roles are
-- cluster-wide, so this line passed silently on any cluster where the write half
-- had ever run — and failed with `role does not exist` only on a fresh cluster
-- running the read-only profile, which is precisely the profile it breaks. See
-- this file's header: nothing here may reference the writer role.
-- Refuses `ALTER PUBLICATION ... ADD TABLE` for a table that is neither tenant-scoped nor
-- recorded as public. The bridge cannot check this for itself — it is a pass-through, and
-- an unscoped table looks exactly like a scoped one from the WAL — so the check belongs
-- here, where the mistake is actually made.
--
-- ⚠️ Only the table the command touched is checked. Validating the whole publication would
-- make a pre-existing unscoped table block every unrelated change, and a guard that blocks
-- unrelated work is a guard people drop.
--
-- `zebridge_scope_publication_to_one_tenant()` passes this because it creates the policies and the
-- row filter *before* adding to the publication. That ordering is load-bearing, not
-- incidental.
-- refuses an unscoped ALTER PUBLICATION — see NOTES.md §1.8
CREATE OR REPLACE FUNCTION public.zebridge_publication_guard()
RETURNS event_trigger
LANGUAGE plpgsql
AS $$
DECLARE
r record;
relid oid;
scoped boolean;
rls boolean;
BEGIN
FOR r IN SELECT * FROM pg_event_trigger_ddl_commands()
WHERE command_tag = 'ALTER PUBLICATION' AND object_type = 'publication relation'
LOOP
-- Only the table this command touched: a pre-existing unscoped table must not
-- block an unrelated change, or the guard becomes something people disable.
SELECT pr.prrelid, pr.prqual IS NOT NULL, c.relrowsecurity
INTO relid, scoped, rls
FROM pg_publication_rel pr JOIN pg_class c ON c.oid = pr.prrelid
WHERE pr.oid = r.objid;
CONTINUE WHEN relid IS NULL;
CONTINUE WHEN relid::regclass::text LIKE '%zebridge_ddl_events';
-- Same reasoning as zebridge_ddl_events: this table's rows are never published as
-- ordinary CDC (the bridge special-cases its relation name and diverts every row
-- into $KV.tenants.<principal> instead — NOTES.md §1.12 part 3). It carries the
-- full principal→tenant roster, which the whole point of that KV bucket's
-- principal-first key order is to avoid ever exposing wholesale; publishing it as
-- `cdc.zebridge_user_tenants.*` would hand every client with a broad CDC grant
-- exactly that roster.
CONTINUE WHEN relid::regclass::text LIKE '%zebridge_user_tenants';
-- A control table like the two above: the bridge diverts its rows into a
-- catalogue reload and never publishes them (NOTES §10bj).
CONTINUE WHEN relid::regclass::text LIKE '%zebridge_catalogue';
CONTINUE WHEN EXISTS (SELECT 1 FROM public.zebridge_catalogue cat
WHERE cat.tbl = (SELECT relname FROM pg_class WHERE oid = relid)
AND cat.tenant_col IS NULL);
-- Either model counts as scoped:
-- A a publication row filter → PostgreSQL bounds CDC itself
-- B RLS enabled → RLS bounds writes and snapshots, and the
-- bridge bounds CDC via the catalogue's tenant_col
-- Requiring both would refuse every model-B table; requiring neither would let an
-- unscoped table through. So: at least one deliberate act of scoping.
CONTINUE WHEN scoped OR rls;
RAISE EXCEPTION
'refusing to publish % unscoped: row filter=%, RLS=%',
relid::regclass, scoped, rls
USING HINT =
'Use SELECT zebridge_scope_publication_to_one_tenant(''' || relid::regclass ||
''', ''<tenant_col>'', ''<tenant>'', ''<publication>''), or record the decision '
'with SELECT zebridge_enable(''' || relid::regclass ||
''', public_reason => ''why everyone may read this'', dry_run => false).';
END LOOP;
END $$;
DROP EVENT TRIGGER IF EXISTS zebridge_publication_guard_t;
CREATE EVENT TRIGGER zebridge_publication_guard_t ON ddl_command_end
WHEN TAG IN ('ALTER PUBLICATION') EXECUTE FUNCTION public.zebridge_publication_guard();
-- Answers "is anything published without being scoped?" — the invariant a pass-through
-- bridge cannot check for itself. Run it after any publication change.
-- is anything published without being scoped? — see NOTES.md §1.8
CREATE OR REPLACE FUNCTION public.zebridge_audit_publications()
RETURNS TABLE (publication name, tbl text, row_filter text, rls boolean, verdict text) AS $$
SELECT p.pubname,
pr.prrelid::regclass::text,
coalesce(pg_get_expr(pr.prqual, pr.prrelid), '(none)'),
c.relrowsecurity,
CASE
WHEN pr.prrelid::regclass::text LIKE '%zebridge_ddl_events' THEN 'internal — expected'
WHEN pr.prrelid::regclass::text LIKE '%zebridge_catalogue' THEN 'internal — expected'
WHEN pr.prqual IS NULL AND NOT c.relrowsecurity THEN
'PASS-THROUGH — every row reaches every subscriber of this publication'
WHEN pr.prqual IS NULL AND c.relrowsecurity THEN
'CDC UNSCOPED — RLS bounds the snapshot, pgoutput ignores it'
WHEN pr.prqual IS NOT NULL AND NOT c.relrowsecurity THEN
'reads scoped, WRITES UNSCOPED — no policy bounds what a principal may write'
ELSE 'scoped'
END
FROM pg_publication_rel pr
JOIN pg_publication p ON p.oid = pr.prpubid
JOIN pg_class c ON c.oid = pr.prrelid
ORDER BY p.pubname, 2;
$$ LANGUAGE sql;
-- ── making a publication is an OPERATION, not a line in a template ──────────
--
-- Nothing in these files creates a publication any more. This one used to:
-- init.core rendered `CREATE PUBLICATION` with BRIDGE_CDC_PUBLICATION substituted in from the
-- environment, so the name a bridge streams was spelled in two places — the
-- template's env and the bridge's own `--pub` — with nothing to make them agree.
--
-- And that block was not the only reader of the variable. THREE internal tables
-- get attached to the publication, and every consuming bridge needs all three:
--
-- zebridge_ddl_events the DDL transport. Without it a bridge never learns a
-- schema changed.
-- zebridge_gc_watermark the offline-window watermark every client reads.
-- zebridge_user_tenants the principal->tenant roster, diverted to $KV.tenants.
-- Without it PROTOCOL Step 0 fails for every client of
-- that bridge. (Write profile only.)
--
-- So the obvious recipe for a second bridge — `CREATE PUBLICATION pub_x` by hand,
-- then enables — produced a QUIET CRIPPLE: user tables replicate, DDL
-- propagation, the GC watermark and tenant resolution silently do not, and every
-- check stays green because the publication is not empty.
--
-- One fix for both: the publication is created HERE, by name, as a deliberate
-- act, and this is the only supported way to make one — so the three cannot be
-- forgotten. Idempotent, and worth re-running after init.write.template.sql to
-- pick up zebridge_user_tenants.
CREATE OR REPLACE FUNCTION public.zebridge_create_publication(p_name name)
RETURNS TABLE (step text, status text, detail text) AS $$
DECLARE
t text;
-- Existence-checked one by one rather than assumed: zebridge_user_tenants
-- lives in init.write, which a readonly-profile database never applies.
-- zebridge_catalogue rides along so a running bridge sees zebridge_enable's row
-- the way it sees DDL — through the WAL — and reloads its routing without a
-- restart (NOTES §10bj). Its rows are never published as CDC.
internal text[] := ARRAY['zebridge_ddl_events', 'zebridge_gc_watermark', 'zebridge_user_tenants', 'zebridge_catalogue'];
BEGIN
IF p_name IS NULL OR p_name = '' THEN
RAISE EXCEPTION 'zebridge_create_publication: no name given'
USING HINT = 'Name it: SELECT * FROM zebridge_create_publication(''my_pub''). '
'The same name the bridge is given as --pub / BRIDGE_CDC_PUBLICATION.';
END IF;
IF EXISTS (SELECT 1 FROM pg_publication WHERE pubname = p_name) THEN
RETURN QUERY SELECT 'publication', 'already', format('%I exists', p_name);
ELSE
EXECUTE format('CREATE PUBLICATION %I', p_name);
RETURN QUERY SELECT 'publication', 'created',
format('CREATE PUBLICATION %I — carries only the internal tables until the '
'first zebridge_enable', p_name);
END IF;
FOREACH t IN ARRAY internal LOOP
IF to_regclass('public.' || quote_ident(t)) IS NULL THEN
RETURN QUERY SELECT 'internal', 'ABSENT',
format('%I does not exist in this database. If this is a write-enabled '
'deployment, apply init.write.template.sql and re-run this function — '
'a bridge on %I loses what that table carries.', t, p_name);
ELSIF EXISTS (SELECT 1 FROM pg_publication_tables
WHERE pubname = p_name AND schemaname = 'public' AND tablename = t) THEN
RETURN QUERY SELECT 'internal', 'already', format('%I already carries %I', p_name, t);
ELSE
EXECUTE format('ALTER PUBLICATION %I ADD TABLE public.%I', p_name, t);
RETURN QUERY SELECT 'internal', 'added',
format('ALTER PUBLICATION %I ADD TABLE %I', p_name, t);
END IF;
END LOOP;
END $$ LANGUAGE plpgsql;
-- ---------------------------------------------------------
-- ZeBridge DDL Event Tracking
-- Guarantees strict ordering of schema changes with CDC data
-- ---------------------------------------------------------
-- DDL transport: the INSERT is what reaches the bridge — see PROTOCOL.md §5
CREATE TABLE IF NOT EXISTS public.zebridge_ddl_events (
id SERIAL PRIMARY KEY,
schema_name TEXT NOT NULL,
table_name TEXT NOT NULL,
command_tag TEXT NOT NULL,
-- Schema captured INSIDE the DDL transaction, so it is the schema as of this
-- LSN. The bridge must not re-query the catalog at processing time: that reads
-- "now" rather than "then", which reorders bursts and corrupts WAL replay after
-- a restart. Shape: {"columns":[{"name":..,"type":..}], "pk":".."|null}
schema_def JSONB,
emitted_at TIMESTAMPTZ DEFAULT NOW()
);
-- A DROP TABLE row: the incarnation that DROP ended (NULL: not catalogued). The bridge's
-- cleanup removes only chains of that incarnation or older, never a later one's.
ALTER TABLE public.zebridge_ddl_events ADD COLUMN IF NOT EXISTS incarnation bigint;
-- Backfill for databases initialised before schema_def existed
ALTER TABLE public.zebridge_ddl_events ADD COLUMN IF NOT EXISTS schema_def JSONB;
-- Tables whose DDL is infrastructure, not application data. Publishing their schemas
-- would tell every client to build a local replica of bookkeeping it has no use for.
-- Defined once and used by BOTH triggers: keeping two copies of this list in sync by
-- hand is exactly how one of them ends up stale.
--
-- Add your migration tool's table here if it differs: flyway_schema_history,
-- alembic_version, __diesel_schema_migrations, goose_db_version, ...
-- ---------------------------------------------------------
-- GC watermark — the oldest tombstone still standing
-- ---------------------------------------------------------
--
-- `GC_THRESHOLD_MS` is a promise to clients: the maximum offline window this deployment
-- supports. A client offline for longer can resurrect a deleted row, because the tombstone
-- that would have overruled its queued edit has been reaped. Until now a client had no way
-- to know where that line sits — it could only assume the sweeper was running on schedule.
--
-- ⚠️ Carried as a **replicated table**, not published to KV by the sweeper. The sweeper is
-- a PostgreSQL client and nothing else: giving it a NATS identity would widen what a
-- compromised sweeper can reach, in order to publish one number the existing CDC pipeline
-- already carries for free. It writes a row; every client that consumes CDC gets it.
--
-- A singleton (`id = 1`, CHECKed), because there is one watermark per deployment and a
-- table that can grow to two rows will eventually have two disagreeing answers.
--
-- ⚠️ Deliberately NOT in `zebridge_is_internal_table`: internal tables are hidden from
-- clients, and this one exists precisely to reach them.
-- the GC watermark clients read before flushing an outbox — see PROTOCOL.md §7.5
CREATE TABLE IF NOT EXISTS public.zebridge_gc_watermark (
id smallint PRIMARY KEY DEFAULT 1 CHECK (id = 1),
-- Nothing soft-deleted before this is guaranteed to still exist. A client whose oldest
-- queued write predates it must re-snapshot rather than flush.
watermark timestamptz NOT NULL,
-- What the sweeper was configured with, so a client can tell a *moving* watermark from
-- a sweeper that has stopped: if `swept_at` is old, the watermark is stale whatever it
-- says.
threshold_ms bigint NOT NULL,
swept_at timestamptz NOT NULL DEFAULT now(),
-- Rows removed in the last pass. Zero is the normal steady state; a large number after
-- a long quiet period usually means the sweeper was down.
reaped bigint NOT NULL DEFAULT 0,
-- The version column, so this table satisfies the same contract as any other CDC
-- table. It is never written from the edge — see the grant below.
updated_at timestamptz NOT NULL DEFAULT now()
);
-- The sweeper writes it; everyone else reads it through CDC. ⚠️ SELECT for the reader is
-- what puts it in snapshots; UPDATE/INSERT for the writer is what lets the sweeper stamp
-- it. No DELETE: a singleton is never removed, and the row must survive a sweeper bug.
GRANT SELECT ON public.zebridge_gc_watermark TO ${POSTGRES_READER_USER};
INSERT INTO public.zebridge_gc_watermark (id, watermark, threshold_ms)
VALUES (1, now(), 0)
ON CONFLICT (id) DO NOTHING;
-- ⚠️ The floor only moves forward, and the DATABASE says so (NOTES "One floor, one sweeper").
-- The watermark is a fact about the database's history: once a tombstone is reaped, no writer
-- can un-reap it, so no value below the floor can ever be true again. A second sweeper with a
-- shorter or longer retention, a bridge, a restored script, a forged mutation (§10aq dragged it
-- back six years with one) or a re-applied init file must not be able to lower it. The sweeper's
-- own SQL uses GREATEST, but that is the sweeper behaving; this is what holds when something else
-- does not. A raise, not a silent clamp: whoever tried should know.
--
-- Break glass, for an operator who has restored the database from a backup taken BEFORE the
-- reaping, or a test that stages a floor: ALTER TABLE public.zebridge_gc_watermark DISABLE TRIGGER
-- zebridge_gc_watermark_monotone; (and the same for _no_delete), the UPDATE, then ENABLE TRIGGER.
CREATE OR REPLACE FUNCTION public.zebridge_gc_watermark_guard() RETURNS trigger
LANGUAGE plpgsql AS $$
BEGIN
IF TG_OP = 'UPDATE' THEN
IF NEW.watermark < OLD.watermark THEN
RAISE EXCEPTION 'zebridge_gc_watermark: the floor only moves forward (% -> % refused)', OLD.watermark, NEW.watermark
USING ERRCODE = 'check_violation',
HINT = 'Tombstones older than the floor are gone; no value below it can be true again. A longer retention takes effect as time catches up.';
END IF;
RETURN NEW;
END IF;
RAISE EXCEPTION 'zebridge_gc_watermark: the floor is never removed (% refused)', TG_OP
USING ERRCODE = 'check_violation';
END $$;
DROP TRIGGER IF EXISTS zebridge_gc_watermark_monotone ON public.zebridge_gc_watermark;
CREATE TRIGGER zebridge_gc_watermark_monotone BEFORE UPDATE ON public.zebridge_gc_watermark
FOR EACH ROW EXECUTE FUNCTION public.zebridge_gc_watermark_guard();
DROP TRIGGER IF EXISTS zebridge_gc_watermark_no_delete ON public.zebridge_gc_watermark;
CREATE TRIGGER zebridge_gc_watermark_no_delete BEFORE DELETE OR TRUNCATE ON public.zebridge_gc_watermark
FOR EACH STATEMENT EXECUTE FUNCTION public.zebridge_gc_watermark_guard();
-- ⚠️ Published, or the whole design is inert: the row reaches clients through CDC and
-- nothing else, so a table nobody publishes is a watermark nobody can read.
--
-- Published **here**, hardcoded, rather than through `zebridge_scope_publication_to_one_tenant()`.
-- That is the rule for bridge-owned tables and it is worth stating, because there are now
-- two publication paths and the difference is not obvious:
--
-- this file the bridge's own tables (zebridge_ddl_events, zebridge_gc_watermark).
-- They exist before any migration runs, they have no tenant, and their
-- shape is fixed by the bridge rather than chosen by a DBA. Publishing
-- them anywhere else would mean the bridge could boot without them.
-- a migration application tables. Tenant-scoped ones go through
-- `zebridge_scope_publication_to_one_tenant()`; genuinely public ones are recorded
-- in `zebridge_catalogue` (public_reason) and added directly.
--
-- Both paths still pass through `zebridge_publication_guard`, which is why the registration
-- below is not optional even here: the guard refuses an unscoped ADD TABLE regardless of who
-- issues it, and this table genuinely is public — one row, no tenant, identical for every
-- consumer.
DO $$
BEGIN
INSERT INTO public.zebridge_catalogue (tbl, public_reason)
VALUES ('zebridge_gc_watermark',
'GC watermark: one row, no tenant, every client needs it')
ON CONFLICT (tbl) DO NOTHING;
-- The catalogue row is written here and the publication attachment is NOT:
-- `zebridge_create_publication` attaches this table to whichever publication
-- is being made. The row must exist first either way — it is what carries this
-- table past zebridge_publication_guard (tenant_col IS NULL == public), and
-- the guard fires on the ALTER, not here.
END $$;
-- ---------------------------------------------------------
-- Generation bookkeeping — the producer's own memory (NOTES.md §1.13, delta generations)
-- ---------------------------------------------------------
--
-- One row per built generation of a (tenant, table) pair: which cutoff it used and at
-- which LSN it was taken. The generation producer reads its LAST row to compute the
-- next delta and appends its new one — from PostgreSQL, never from NATS, because "the
-- bridge never reads its own output back" is a founding invariant. Append-only with
-- pruning past the chain depth; the rows double as the audit trail ("what did G17
-- cover and when").
--
-- ⚠️ `cutoff_lsn` is stamped by the producer itself: `SELECT pg_current_wal_lsn()`
-- taken BEFORE the REPEATABLE READ snapshot that reads the content, and this row
-- committed in that same transaction — overlap-never-gap; see NOTES.md §1.13 for why
-- the reverse order loses rows.
--
-- ⚠️ NOT published and listed in `zebridge_is_internal_table`: clients must never build
-- a local replica of producer bookkeeping, and its DDL must not reach `$KV.schemas`.
CREATE TABLE IF NOT EXISTS public.zebridge_generations (
tenant text NOT NULL,
tbl text NOT NULL,
gen bigint NOT NULL,
-- The version watermark this generation's content query cut at. The next delta
-- selects `version > cutoff_version - clamp_tolerance` (the margin rule).
cutoff_version timestamptz NOT NULL,
-- Where CDC resumes after applying this generation's chain: clients skip events
-- with lsn <= cutoff_lsn (every CDC event carries lsn).
cutoff_lsn pg_lsn NOT NULL,
-- The previous generation's cutoff — the delta's lower bound (minus the margin).
-- Stored, not derived: after pruning, the oldest kept delta's lower bound refers
-- to a generation whose row is gone. NULL on a chain's first generation.
prev_cutoff timestamptz,
-- Whether this generation also shipped a full object (`<tbl>-g<N>-full`): the
-- chain's jump-in point, refreshed before it can age out of the kept window.
has_full boolean NOT NULL DEFAULT false,
built_at timestamptz NOT NULL DEFAULT now(),
-- The catalogue's seed_epoch this generation was built under (§10df): a later
-- bump forces the next build to be a FULL, whatever the counts say.
seed_epoch integer NOT NULL DEFAULT 0,
-- The table's column shape (`name:type,…` in attnum order) at this build (§10dg):
-- a later DROP/RENAME/re-type/ADD forces a FULL, so a chain object never names a
-- column the replica no longer has.
col_shape text,
-- The table's OID at this build (§10dl): a table dropped and re-created under the
-- same name within one tick is a DIFFERENT table with the same chain name — the
-- producer sweeps the old chain, bumps the epoch, and starts it at g1.
relid oid,
PRIMARY KEY (tenant, tbl, gen)
);
-- The incarnation this generation was built under: chain objects are named with it
-- (`<table>-i<incarnation>-g<gen>-<kind>`), so a recreated table's g1 never shares a
-- name with its predecessor's, and whoever deletes an old chain cannot hit the new one.
ALTER TABLE public.zebridge_generations ADD COLUMN IF NOT EXISTS incarnation bigint NOT NULL DEFAULT 0;
-- 2026-09-24: the per-era compression dictionary is gone (NOTES §10iy). Measured on the
-- 668 MB test_types full: 53% smaller at 4 KB, 3% from 64 KB up — for a training pass,
-- five columns here, an object per era, and a decoder a phone did not have. Every
-- object is plain zstd, level 3. The eras cut with a dictionary are retired here in one
-- go, so the next cut is a plain full (numbered above every row, retired included);
-- their objects leave with the retirement grace, the dictionary objects among them.
DO $$ BEGIN
IF EXISTS (SELECT 1 FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = 'zebridge_generations' AND column_name = 'dict_object') THEN
UPDATE public.zebridge_generations SET retired_at = now()
WHERE retired_at IS NULL AND (tenant, tbl) IN (
SELECT tenant, tbl FROM public.zebridge_generations
WHERE dict_object IS NOT NULL OR full_dict_object IS NOT NULL OR ckpt_dict_object IS NOT NULL);
END IF;
END $$;
ALTER TABLE public.zebridge_generations
DROP COLUMN IF EXISTS dict,
DROP COLUMN IF EXISTS dict_object,
DROP COLUMN IF EXISTS dict_ratio,
DROP COLUMN IF EXISTS full_dict_object,
DROP COLUMN IF EXISTS ckpt_dict_object;
-- §10gl: the lower bound of the NEXT delta. `now()` is a transaction's start, so a row
-- committed after this cut can carry a version older than the cutoff; it came from a
-- transaction already open when the cut was read. The floor is the start of the oldest
-- of those (or the cut itself), and the next delta reads `version >= floor`.
ALTER TABLE public.zebridge_generations ADD COLUMN IF NOT EXISTS open_xact_floor timestamptz;
-- §10gt: the incremental chain's middle level. A checkpoint carries every row whose
-- version moved inside its window — tombstoned rows included, which is what lets a
-- returning client learn about a delete without reloading the table — and attaches to an
-- existing generation, like a background full. `ckpt_lower` is the window's lower bound:
-- the previous checkpoint's cutoff, or the base's.
ALTER TABLE public.zebridge_generations ADD COLUMN IF NOT EXISTS has_checkpoint boolean NOT NULL DEFAULT false;
ALTER TABLE public.zebridge_generations ADD COLUMN IF NOT EXISTS ckpt_lower timestamptz;
-- §10ja: whether each object this row carries was cut in primary-key order. The manifest
-- says `"sorted": true` for those, and a client applies them without staging or sorting.
-- False for everything cut before 2026-09-25 — those objects stay readable as they were.