diff --git a/console/src/components/Table/TablePagination.tsx b/console/src/components/Table/TablePagination.tsx index b5abfb1c95f7e..963027c570aa1 100644 --- a/console/src/components/Table/TablePagination.tsx +++ b/console/src/components/Table/TablePagination.tsx @@ -15,26 +15,39 @@ import { MaterializeTheme } from "~/theme"; import { TablePaginationProps } from "./tableTypes"; -export const TablePagination = ({ - table, +export interface PaginationControlsProps { + /** Zero-based. */ + pageIndex: number; + pageSize: number; + pageCount: number; + totalItems: number; + itemLabel?: React.ReactNode; + onPrevious: () => void; + onNext: () => void; +} + +/** "Showing 1-30 of 120" with previous and next buttons. Hidden for one page. */ +export const PaginationControls = ({ + pageIndex, + pageSize, + pageCount, + totalItems, itemLabel = "results", -}: TablePaginationProps) => { + onPrevious, + onNext, +}: PaginationControlsProps) => { const { colors } = useTheme(); - const pageCount = table.getPageCount(); - const totalRows = table.getFilteredRowModel().rows.length; - const { pageIndex, pageSize } = table.getState().pagination; - if (pageCount <= 1) return null; const from = pageIndex * pageSize + 1; - const to = Math.min((pageIndex + 1) * pageSize, totalRows); + const to = Math.min((pageIndex + 1) * pageSize, totalItems); return ( Showing {from.toLocaleString("en-US")}-{to.toLocaleString("en-US")} of{" "} - {totalRows.toLocaleString("en-US")} {itemLabel} + {totalItems.toLocaleString("en-US")} {itemLabel} ({ variant="secondary" minW="8" height="8" - isDisabled={!table.getCanPreviousPage()} - onClick={() => table.previousPage()} + isDisabled={pageIndex === 0} + onClick={onPrevious} /> page {pageIndex + 1} of {pageCount} @@ -55,10 +68,28 @@ export const TablePagination = ({ variant="secondary" minW="8" height="8" - isDisabled={!table.getCanNextPage()} - onClick={() => table.nextPage()} + isDisabled={pageIndex >= pageCount - 1} + onClick={onNext} /> ); }; + +export const TablePagination = ({ + table, + itemLabel, +}: TablePaginationProps) => { + const { pageIndex, pageSize } = table.getState().pagination; + return ( + table.previousPage()} + onNext={() => table.nextPage()} + /> + ); +}; diff --git a/console/src/platform/clusters/ClusterDetail.test.tsx b/console/src/platform/clusters/ClusterDetail.test.tsx index 2c0ebb160bbc2..ae22fe2a9651a 100644 --- a/console/src/platform/clusters/ClusterDetail.test.tsx +++ b/console/src/platform/clusters/ClusterDetail.test.tsx @@ -60,6 +60,9 @@ vi.mock("~/platform/clusters/ClusterPage/ClusterWorkers", async () => { ), }; }); +vi.mock("~/platform/clusters/ClusterPage/Hydration/ClusterHydration", () => ({ + ClusterHydration: () =>
ClusterHydration component
, +})); vi.mock("~/platform/clusters/ClusterReplicas", () => ({ default: () =>
ClusterReplicas component
, })); @@ -192,9 +195,11 @@ describe("ClusterDetail redesign preview", () => { .getAllByRole("link") .map((link) => link.textContent) .filter((name) => - ["Metrics", "Workers", "Objects", "Replicas"].includes(name ?? ""), + ["Metrics", "Workers", "Objects", "Hydration", "Replicas"].includes( + name ?? "", + ), ), - ).toEqual(["Metrics", "Workers", "Objects"]); + ).toEqual(["Metrics", "Workers", "Objects", "Hydration"]); expect( screen.getByRole("button", { name: "Configure cluster" }), ).toBeVisible(); @@ -270,6 +275,18 @@ describe("ClusterDetail redesign preview", () => { ); }); + it("opens the Hydration tab from its link", async () => { + const user = userEvent.setup(); + await renderClusterDetail("/u1/default"); + + await user.click(await screen.findByRole("link", { name: "Hydration" })); + + await expectPathname("/u1/default/hydration"); + expect( + await screen.findByText("ClusterHydration component"), + ).toBeVisible(); + }); + it("opens the Objects tab on its first object type", async () => { const user = userEvent.setup(); await renderClusterDetail("/u1/default"); @@ -473,5 +490,14 @@ describe("ClusterDetail redesign preview", () => { await expectPathname("/u1/default/sinks"); expect(await screen.findByText("Sinks component")).toBeVisible(); }); + + it("sends a redesigned Hydration URL to the classic overview", async () => { + await renderClusterDetail("/u1/default/hydration"); + + await expectPathname("/u1/default"); + expect( + await screen.findByText("ClusterOverview component"), + ).toBeVisible(); + }); }); }); diff --git a/console/src/platform/clusters/ClusterDetail.tsx b/console/src/platform/clusters/ClusterDetail.tsx index 4584866aab57c..1cc38e607aa4d 100644 --- a/console/src/platform/clusters/ClusterDetail.tsx +++ b/console/src/platform/clusters/ClusterDetail.tsx @@ -25,7 +25,7 @@ import { ClusterDetailBreadcrumbs } from "./ClusterDetailBreadcrumbs"; import ClusterOverview from "./ClusterOverview"; import { CLUSTER_OBJECT_TYPES } from "./ClusterPage/clusterObjectTypes"; import ClusterPage, { ClusterPageHeader } from "./ClusterPage/ClusterPage"; -import { WORKERS_TAB_PATH } from "./ClusterPage/tabLinks"; +import { HYDRATION_TAB_PATH, WORKERS_TAB_PATH } from "./ClusterPage/tabLinks"; import ClusterReplicas from "./ClusterReplicas"; import IndexList from "./IndexList"; import MaterializedViewsList from "./MaterializedViewsList"; @@ -72,8 +72,9 @@ const ClassicClusterDetailPage = () => { } /> } /> } /> - {/* The redesigned Workers tab has no classic counterpart. */} + {/* The redesigned Workers and Hydration tabs have no classic counterpart. */} } /> + } /> ); }; diff --git a/console/src/platform/clusters/ClusterPage/ClusterPage.tsx b/console/src/platform/clusters/ClusterPage/ClusterPage.tsx index c5d7a7a3d5a8c..60611f48b1fcd 100644 --- a/console/src/platform/clusters/ClusterPage/ClusterPage.tsx +++ b/console/src/platform/clusters/ClusterPage/ClusterPage.tsx @@ -28,10 +28,16 @@ import { CONFIGURATION_SECTIONS, ConfigurationSection, } from "./Configuration/clusterConfigurationModel"; -import { replicaSearch, WORKERS_TAB_PATH } from "./tabLinks"; +import { ClusterHydration } from "./Hydration/ClusterHydration"; +import { + HYDRATION_TAB_PATH, + replicaSearch, + WORKERS_TAB_PATH, +} from "./tabLinks"; import { REPLICA_SEARCH_PARAM } from "./useSelectedReplica"; -// The Metrics and Workers tabs share a replica choice, so their links carry it. +// The Metrics, Workers and Hydration tabs share a replica choice, so their +// links carry it. const tabs = (replicaName: string | undefined): Tab[] => [ { label: "Metrics", href: `..${replicaSearch(replicaName)}`, end: true }, { @@ -39,6 +45,10 @@ const tabs = (replicaName: string | undefined): Tab[] => [ href: `../${WORKERS_TAB_PATH}${replicaSearch(replicaName)}`, }, { label: "Objects", href: "../objects" }, + { + label: "Hydration", + href: `../${HYDRATION_TAB_PATH}${replicaSearch(replicaName)}`, + }, ]; // The configuration drawer's open section, in the URL so it survives a reload @@ -174,6 +184,10 @@ const ClusterPage = () => { path="objects/:objectType" element={} /> + } + /> ({ + ...(await importOriginal()), + useParentSize: () => ({ + parentRef: () => {}, + width: 900, + height: 0, + resize: () => {}, + }), +})); + +const ENVIRONMENT = { + ...healthyEnvironment, + status: { + ...healthyEnvironment.status, + version: parseDbVersion("v26.46.0 (0000000000)"), + }, +}; +const ENVIRONMENT_WITHOUT_HISTORY = { + ...healthyEnvironment, + status: { + ...healthyEnvironment.status, + version: parseDbVersion("v26.42.0 (0000000000)"), + }, +}; + +const MINUTE_MS = 60_000; +const DAY_MS = 24 * 60 * MINUTE_MS; +const NOW_MS = Date.now(); +const TIME_PERIOD_MINUTES = 7 * 24 * 60; +const GIB = 2 ** 30; + +const buildCluster = (offlineReplicaIds: string[] = []) => + buildClusterServerResponse( + { id: "u9", name: "analytics" }, + ["u1", "u2"].map((id, index) => ({ + id, + name: `r${index + 1}`, + size: "100cc", + disk: false, + statuses: offlineReplicaIds.includes(id) + ? [ + { + replica_id: id, + process_id: "0", + reason: null, + status: "offline", + updated_at: new Date(NOW_MS).toISOString(), + }, + ] + : [], + })), + ); +const NO_REPLICAS_CLUSTER = buildClusterServerResponse( + { id: "u9", name: "analytics" }, + [], +); +const BOOTED_AT_MS = NOW_MS - 3 * DAY_MS; + +const catalogObject = ( + id: string, + name: string, + objectType: string, +): DatabaseObject => ({ + databaseName: "materialize", + databaseId: "u1", + name, + schemaName: "public", + schemaId: "u2", + id, + objectType, + sourceType: null, + isWebhookTable: null, + clusterId: "u9", + clusterName: "analytics", +}); + +const timestamptz = (name: string) => + buildColumn({ name, type_oid: MzDataType.timestamptz }); +const float8 = (name: string) => + buildColumn({ name, type_oid: MzDataType.float8 }); + +const episode = ( + replicaId: string, + startedAtMs: number, + durationMs: number, + objectCount: number, +) => ({ + replicaId, + startedAt: String(startedAtMs), + finishedAt: String(startedAtMs + durationMs), + objectCount: String(objectCount), + peakMemoryBytes: String(6 * GIB), + peakDiskBytes: String(GIB / 4), + peakHeapBytes: String(6.4 * GIB), +}); + +// r2's full hydration, the latest one. +const LATEST_FULL = { startedAtMs: BOOTED_AT_MS + 21_000, durationMs: 280_000 }; +// r1's additive hydration, the last to finish. +const LAST_FINISHED = { startedAtMs: NOW_MS - DAY_MS, durationMs: 26_000 }; +const EPISODES = [ + episode("u1", BOOTED_AT_MS + 20_000, 271_000, 8), + episode("u2", LATEST_FULL.startedAtMs, LATEST_FULL.durationMs, 8), + episode("u1", LAST_FINISHED.startedAtMs, LAST_FINISHED.durationMs, 1), +]; + +const online = (replicaId: string, atMs: number) => ({ + replicaId, + processId: "0", + occurredAt: String(atMs), + status: "online", + reason: null, +}); +const offline = (replicaId: string, atMs: number) => ({ + replicaId, + processId: "0", + occurredAt: String(atMs), + status: "offline", + reason: null, +}); +const outOfMemory = (replicaId: string, atMs: number) => ({ + replicaId, + processId: "0", + occurredAt: String(atMs), + status: "offline", + reason: "oom-killed", +}); + +interface Scenario { + environment?: typeof ENVIRONMENT; + statusRows?: object[]; + /** Hydrated and total objects per replica. */ + progress?: Record; + /** Live progress rows, in place of the ones `progress` makes up. */ + progressRows?: unknown[][]; + /** When the cluster's objects were created. */ + creationRows?: object[]; + hasReplicas?: boolean; + /** Replicas whose process reports offline. */ + offlineReplicaIds?: string[]; + /** Replicas dropped in the window. */ + droppedReplicas?: Array<{ + replicaId: string; + name: string; + createdAtMs: number; + droppedAtMs: number; + }>; + /** The CPU peak of the last finished hydration fails to load. */ + isCpuPeakError?: boolean; + /** Objects in the last finished hydration, beyond its first. */ + extraObjectCount?: number; + liveObjects?: unknown[][]; + /** The live progress subscribe never delivers a snapshot. */ + isProgressStuck?: boolean; + isEpisodesError?: boolean; + /** The current replicas' status history fails to load. */ + isStatusError?: boolean; + search?: string; +} + +const renderHydrationTab = async ({ + environment = ENVIRONMENT, + statusRows = [online("u1", BOOTED_AT_MS), online("u2", BOOTED_AT_MS)], + progress = { u1: [9, 9], u2: [9, 9] }, + progressRows, + creationRows = [ + { + id: "u11", + objectType: "materialized-view", + occurredAt: String(NOW_MS - 2 * DAY_MS), + }, + ], + hasReplicas = true, + offlineReplicaIds = [], + droppedReplicas = [], + isCpuPeakError = false, + extraObjectCount = 0, + liveObjects = [], + isProgressStuck = false, + isEpisodesError = false, + isStatusError = false, + search = "", +}: Scenario = {}) => { + // Query keys embed the environment version, so set it before the handlers. + await setFakeEnvironment(getStore().set, "aws/us-east-1", environment); + const replicaIds = hasReplicas ? ["u1", "u2"] : []; + const cluster = hasReplicas + ? buildCluster(offlineReplicaIds) + : NO_REPLICAS_CLUSTER; + getStore().set(allClusters, mockSubscribeState({ data: [cluster] })); + const lastFinishedStart = new Date(LAST_FINISHED.startedAtMs).toISOString(); + const lastFinishedEnd = new Date( + LAST_FINISHED.startedAtMs + LAST_FINISHED.durationMs, + ).toISOString(); + const latestStart = new Date(LATEST_FULL.startedAtMs).toISOString(); + const latestEnd = new Date( + LATEST_FULL.startedAtMs + LATEST_FULL.durationMs, + ).toISOString(); + server.use( + buildSqlQueryHandlerV2({ + queryKey: clusterQueryKeys.clusterReplicaHistory({ + clusterId: "u9", + timePeriodMinutes: TIME_PERIOD_MINUTES, + currentReplicaIdsKey: replicaIds.join(","), + }), + results: mapKyselyToTabular({ + rows: [ + ...replicaIds.map((replicaId, index) => ({ + replicaId, + name: `r${index + 1}`, + size: "100cc", + createdAt: String(BOOTED_AT_MS), + droppedAt: null, + })), + ...droppedReplicas.map( + ({ replicaId, name, createdAtMs, droppedAtMs }) => ({ + replicaId, + name, + size: "100cc", + createdAt: String(createdAtMs), + droppedAt: String(droppedAtMs), + }), + ), + ], + columns: [timestamptz("createdAt"), timestamptz("droppedAt")], + }), + }), + buildSqlQueryHandlerV2({ + queryKey: clusterQueryKeys.replicaStatusHistory({ + replicaIds: droppedReplicas.map(({ replicaId }) => replicaId), + statusesUpdatedAt: undefined, + }), + results: mapKyselyToTabular({ + rows: [], + columns: [timestamptz("occurredAt")], + }), + }), + buildSqlQueryHandlerV2({ + queryKey: clusterQueryKeys.replicaStatusHistory({ + replicaIds: ["u1", "u2"], + statusesUpdatedAt: replicaStatusesUpdatedAt(cluster.replicas), + }), + results: isStatusError + ? { + notices: [], + error: { + message: "Something went wrong", + code: ErrorCode.INTERNAL_ERROR, + }, + } + : mapKyselyToTabular({ + rows: statusRows, + columns: [timestamptz("occurredAt")], + }), + }), + buildSqlQueryHandlerV2({ + queryKey: clusterQueryKeys.clusterHydrationEpisodes({ + clusterId: "u9", + timePeriodMinutes: TIME_PERIOD_MINUTES, + }), + results: isEpisodesError + ? { + notices: [], + error: { + message: "Something went wrong", + code: ErrorCode.INTERNAL_ERROR, + }, + } + : mapKyselyToTabular({ + rows: EPISODES, + columns: [ + timestamptz("startedAt"), + timestamptz("finishedAt"), + float8("objectCount"), + float8("peakMemoryBytes"), + float8("peakDiskBytes"), + float8("peakHeapBytes"), + ], + }), + }), + buildSqlQueryHandlerV2({ + queryKey: clusterQueryKeys.replicaHeapLimits({ + replicaIds: ["u1", "u2"], + }), + results: mapKyselyToTabular({ + rows: ["u1", "u2"].map((replicaId) => ({ + replicaId, + processId: "0", + heapLimit: String(10 * GIB), + memoryBytes: String(GIB), + })), + columns: buildColumns(["replicaId", "processId", "heapLimit"]), + }), + }), + buildSqlQueryHandlerV2({ + queryKey: clusterQueryKeys.replicaSizeAllocations({ sizes: ["100cc"] }), + results: mapKyselyToTabular({ + rows: [ + { + size: "100cc", + processes: "1", + workers: "1", + cpuNanoCores: "1000000000", + memoryBytes: String(8 * GIB), + diskBytes: String(2 * GIB), + }, + ], + columns: [ + float8("processes"), + float8("workers"), + float8("cpuNanoCores"), + float8("memoryBytes"), + float8("diskBytes"), + ], + }), + }), + buildSqlQueryHandlerV2({ + queryKey: clusterQueryKeys.objectCreationTimes({ + objectIds: ["u10", "u11"], + }), + results: mapKyselyToTabular({ + rows: creationRows, + columns: [timestamptz("occurredAt")], + }), + }), + buildSqlQueryHandlerV2({ + queryKey: clusterQueryKeys.episodeObjectHydrations({ + replicaId: "u2", + startDate: latestStart, + endDate: latestEnd, + }), + results: mapKyselyToTabular({ + rows: [ + { + globalId: "u10", + objectId: "u10", + installedAt: String(LATEST_FULL.startedAtMs), + startedAt: String(LATEST_FULL.startedAtMs), + hydratedAt: String(LATEST_FULL.startedAtMs + 38_000), + }, + { + globalId: "u11", + objectId: "u11", + installedAt: String(LATEST_FULL.startedAtMs), + startedAt: String(LATEST_FULL.startedAtMs + 10_000), + hydratedAt: String( + LATEST_FULL.startedAtMs + LATEST_FULL.durationMs, + ), + }, + ], + columns: [ + timestamptz("installedAt"), + timestamptz("startedAt"), + timestamptz("hydratedAt"), + ], + }), + }), + buildSqlQueryHandlerV2({ + queryKey: clusterQueryKeys.episodeObjectHydrations({ + replicaId: "u1", + startDate: lastFinishedStart, + endDate: lastFinishedEnd, + }), + results: mapKyselyToTabular({ + rows: [ + { + globalId: "u11", + objectId: "u11", + installedAt: String(LAST_FINISHED.startedAtMs), + startedAt: String(LAST_FINISHED.startedAtMs), + hydratedAt: String( + LAST_FINISHED.startedAtMs + LAST_FINISHED.durationMs, + ), + }, + // Dropped objects' dataflows, named by their ids, starting in turn. + ...Array.from({ length: extraObjectCount }, (_, index) => ({ + globalId: `u${200 + index}`, + objectId: null, + installedAt: String(LAST_FINISHED.startedAtMs + index + 1), + startedAt: String(LAST_FINISHED.startedAtMs + index + 1), + hydratedAt: String(LAST_FINISHED.startedAtMs + 1_000), + })), + ], + columns: [ + timestamptz("installedAt"), + timestamptz("startedAt"), + timestamptz("hydratedAt"), + ], + }), + }), + buildSqlQueryHandlerV2({ + queryKey: clusterQueryKeys.replicaCpuPeak({ + replicaId: "u1", + startDate: lastFinishedStart, + endDate: lastFinishedEnd, + }), + results: isCpuPeakError + ? { + notices: [], + error: { + message: "Something went wrong", + code: ErrorCode.INTERNAL_ERROR, + }, + } + : mapKyselyToTabular({ + rows: [{ peakCpuNanoCores: "300000000" }], + columns: [float8("peakCpuNanoCores")], + }), + }), + buildSqlQueryHandlerV2({ + queryKey: clusterQueryKeys.replicaCpuPeak({ + replicaId: "u2", + startDate: latestStart, + endDate: latestEnd, + }), + results: mapKyselyToTabular({ + rows: [{ peakCpuNanoCores: "900000000" }], + columns: [float8("peakCpuNanoCores")], + }), + }), + mockSubscribeSocket((sql) => { + if (sql.includes("mz_compute_hydration_times_per_worker")) { + return { + columns: [ + buildColumn({ name: "globalId" }), + buildColumn({ name: "objectId" }), + timestamptz("installedAt"), + timestamptz("startedAt"), + timestamptz("hydratedAt"), + ], + rows: liveObjects, + }; + } + if (sql.includes("mz_compute_hydration_times")) { + if (isProgressStuck) return undefined; + return { + columns: [ + buildColumn({ name: "replicaId" }), + buildColumn({ name: "objectId" }), + buildColumn({ name: "isHydrated", type_oid: MzDataType.bool }), + ], + rows: + progressRows ?? + Object.entries(progress).flatMap(([replicaId, [hydrated, total]]) => + Array.from({ length: total }, (_, objectIndex) => [ + replicaId, + `u${100 + objectIndex}`, + objectIndex < hydrated, + ]), + ), + }; + } + return undefined; + }), + ); + return renderComponent( + + } /> + , + { + initialRouterEntries: [`/u9/analytics${search}`], + initializeState: ({ set }) => + setFakeEnvironment(set, "aws/us-east-1", environment), + }, + ); +}; + +describe("ClusterHydration", () => { + beforeEach(async () => { + localStorage.clear(); + // The collections outlive a test, which would serve its rows to the next. + const scope = await getStore().get(syncEngineCacheScopeAtom); + assert(scope); + hydrationStatesByCluster.get(scope, "u9").reset(); + liveObjectHydrationsByReplica.get(scope, "u2").reset(); + allObjectsCollection.applySnapshot({ + data: [ + catalogObject("u10", "orders_idx", "index"), + catalogObject("u11", "customer_ltv", "materialized-view"), + ], + snapshotComplete: true, + error: undefined, + }); + }); + + it("opens the last successful hydration, and any other from the table", async () => { + await renderHydrationTab(); + + expect( + await screen.findByRole("heading", { name: "Last Successful Hydration" }), + ).toBeVisible(); + expect(screen.getByText(/^Additive hydration on r1, /)).toBeVisible(); + // Every replica is hydrated, so there's nothing to call out. + expect(screen.queryByRole("status")).not.toBeInTheDocument(); + expect( + screen.queryByRole("heading", { name: "Hydration in Progress" }), + ).not.toBeInTheDocument(); + const table = screen.getByRole("table"); + const rows = within(table).getAllByRole("row").slice(1); + expect( + rows.map((row) => within(row).getAllByRole("cell")[2].textContent), + ).toEqual(["Additive", "Full", "Full"]); + await userEvent.click(within(rows[1]).getByRole("button")); + expect( + await screen.findByRole("heading", { name: "Full Hydration on r2" }), + ).toBeVisible(); + const peakHeap = screen.getByRole("group", { name: "Peak heap" }); + expect(within(peakHeap).getByText("64%")).toBeVisible(); + expect(within(peakHeap).getByText("6.4 GB")).toBeVisible(); + expect(screen.getByText("90%")).toBeVisible(); + }); + + it("orders a hydration's objects by how long they worked", async () => { + await renderHydrationTab(); + const table = await screen.findByRole("table"); + const rows = within(table).getAllByRole("row").slice(1); + await userEvent.click(within(rows[1]).getByRole("button")); + const waterfall = await screen.findByRole("img", { + name: "Object hydration times", + }); + const names = () => + within(waterfall) + .getAllByText(/^(orders_idx|customer_ltv)$/, { selector: "title" }) + .map(({ textContent }) => textContent); + expect(names()).toEqual(["orders_idx", "customer_ltv"]); + expect(screen.getByText("Waiting")).toBeVisible(); + + await userEvent.click( + screen.getByRole("button", { name: "Slowest first" }), + ); + + expect(names()).toEqual(["customer_ltv", "orders_idx"]); + // Working times alone: one bar per object, and no waiting in the legend. + expect(screen.queryByText("Waiting")).not.toBeInTheDocument(); + expect(waterfall.querySelectorAll("rect")).toHaveLength(2); + }); + + it("pages through a hydration's objects", async () => { + await renderHydrationTab({ extraObjectCount: 30 }); + + expect(await screen.findByText("Showing 1-30 of 31 objects")).toBeVisible(); + const waterfall = screen.getByRole("img", { + name: "Object hydration times", + }); + expect( + within(waterfall).queryByText("u229", { selector: "title" }), + ).not.toBeInTheDocument(); + + await userEvent.click(screen.getByRole("button", { name: "Next page" })); + + expect(screen.getByText("Showing 31-31 of 31 objects")).toBeVisible(); + expect( + within(waterfall).getByText("u229", { selector: "title" }), + ).toBeInTheDocument(); + }); + + it("keeps a hydration's object timings when its CPU sample fails to load", async () => { + await renderHydrationTab({ isCpuPeakError: true }); + + expect( + await screen.findByRole("img", { name: "Object hydration times" }), + ).toBeVisible(); + expect(screen.getByText("Couldn't be loaded")).toBeVisible(); + }); + + it("keeps the history up while a new replica's or object's data loads", async () => { + await renderHydrationTab(); + expect(await screen.findByRole("table")).toBeVisible(); + + // A third replica and a new index each change the keys the history reads. + act(() => + getStore().set( + allClusters, + mockSubscribeState({ + data: [ + buildClusterServerResponse( + { id: "u9", name: "analytics" }, + ["u1", "u2", "u3"].map((id, index) => ({ + id, + name: `r${index + 1}`, + size: "100cc", + disk: false, + statuses: [], + })), + ), + ], + }), + ), + ); + act(() => + allObjectsCollection.applySnapshot({ + data: [ + catalogObject("u10", "orders_idx", "index"), + catalogObject("u11", "customer_ltv", "materialized-view"), + catalogObject("u12", "orders_by_day_idx", "index"), + ], + snapshotComplete: true, + error: undefined, + }), + ); + + expect(screen.getByRole("table")).toBeVisible(); + expect( + screen.getByRole("heading", { name: "Last Successful Hydration" }), + ).toBeVisible(); + }); + + it("moves a hydrating replica's elapsed time on", async () => { + // Relative to this test's start, as the fixtures' `NOW_MS` is not. + const bootedAtMs = Date.now() - 2 * MINUTE_MS; + await renderHydrationTab({ + statusRows: [online("u1", BOOTED_AT_MS), online("u2", bootedAtMs)], + progress: { u1: [9, 9], u2: [1, 2] }, + }); + // Followed by how long its last full hydration took. + expect(await screen.findByText(/^2m \d+s so far · /)).toBeVisible(); + + expect( + await screen.findByText(/^2m [2-9]s so far · /, {}, { timeout: 4_000 }), + ).toBeVisible(); + }); + + it("says restarts that went offline went offline", async () => { + await renderHydrationTab({ + statusRows: [ + online("u1", BOOTED_AT_MS), + online("u2", BOOTED_AT_MS), + offline("u1", NOW_MS - 50 * MINUTE_MS), + online("u1", NOW_MS - 49 * MINUTE_MS), + offline("u1", NOW_MS - 30 * MINUTE_MS), + online("u1", NOW_MS - 29 * MINUTE_MS), + offline("u1", NOW_MS - 10 * MINUTE_MS), + online("u1", NOW_MS - 9 * MINUTE_MS), + ], + progress: { u1: [3, 9], u2: [9, 9] }, + }); + + expect( + await screen.findByRole("button", { + name: /^r1 restarted 2 times before finishing, Each after going offline/, + }), + ).toBeInTheDocument(); + }); + + it("calls out a replica that keeps restarting before it finishes", async () => { + await renderHydrationTab({ + statusRows: [ + online("u1", BOOTED_AT_MS), + online("u2", BOOTED_AT_MS), + outOfMemory("u1", NOW_MS - 50 * MINUTE_MS), + online("u1", NOW_MS - 49 * MINUTE_MS), + outOfMemory("u1", NOW_MS - 30 * MINUTE_MS), + online("u1", NOW_MS - 29 * MINUTE_MS), + outOfMemory("u1", NOW_MS - 10 * MINUTE_MS), + online("u1", NOW_MS - 9 * MINUTE_MS), + ], + progress: { u1: [3, 9], u2: [9, 9] }, + }); + + expect( + await screen.findByText(/^r1 has no finished hydration recorded since /), + ).toBeVisible(); + expect( + screen.getByText( + /^It restarted 2 times, each after running out of memory\. See our/, + ), + ).toBeVisible(); + expect( + screen.getByRole("link", { + name: "docs on troubleshooting hydration failures", + }), + ).toBeVisible(); + expect(screen.getByText("Hydrated so far: 3 of 9 objects")).toBeVisible(); + // The alert already counts the restarts. + expect(screen.queryByText(/without finishing/)).not.toBeInTheDocument(); + // Close together, the two restarts are one mark, counted in its label. + const restarts = screen.getByRole("button", { + name: /^r1 restarted 2 times before finishing, Each after running out of memory/, + }); + expect(restarts).not.toHaveTextContent("2"); + const table = await screen.findByRole("table"); + expect( + within(table).getAllByText("Restarted before finishing"), + ).toHaveLength(2); + }); + + it("shows what a hydrating replica is working on", async () => { + await renderHydrationTab({ + statusRows: [ + online("u1", BOOTED_AT_MS), + online("u2", NOW_MS - 2 * MINUTE_MS), + ], + progress: { u1: [9, 9], u2: [1, 2] }, + liveObjects: [ + [ + "u10", + "u10", + String(NOW_MS - 2 * MINUTE_MS), + String(NOW_MS - MINUTE_MS), + null, + ], + ["u11", "u11", String(NOW_MS - 2 * MINUTE_MS), null, null], + ], + }); + + expect( + await screen.findByText("Hydrated so far: 1 of 2 objects"), + ).toBeVisible(); + // The progress card says it, so there's no alert and no Hydrating label. + expect(screen.queryByRole("status")).not.toBeInTheDocument(); + expect(screen.queryByText("Hydrating")).not.toBeInTheDocument(); + const working = await screen.findByRole("list", { + name: "Objects r2 is hydrating", + }); + expect( + within(working).getByText( + (_, element) => + element?.tagName === "LI" && + /^Working on orders_idx for 1m \d+s$/.test(element.textContent ?? ""), + ), + ).toBeVisible(); + expect(within(working).queryByText(/waiting/)).not.toBeInTheDocument(); + }); + + it("dates a new object's hydration from its creation, not the replica's boot", async () => { + // Relative to this test's start: the fixtures' `NOW_MS` is taken when the + // file loads, and the tests before this one move the clock on. + const createdAtMs = Date.now() - 90_000; + await renderHydrationTab({ + // Both replicas booted days ago, and an index created a minute and a + // half ago is hydrating on r1. + progressRows: [ + ["u1", "u10", true], + ["u1", "u11", false], + ["u2", "u10", true], + ["u2", "u11", true], + ], + creationRows: [ + { + id: "u11", + objectType: "materialized-view", + occurredAt: String(createdAtMs), + }, + ], + }); + + expect(await screen.findByText(/^1m 3\ds so far/)).toBeVisible(); + expect( + screen.getByRole("img", { name: /^r1 is hydrating, 1m 3\ds so far$/ }), + ).toBeInTheDocument(); + }); + + it("lists past hydrations with each one's peak memory", async () => { + await renderHydrationTab(); + + expect( + await screen.findByRole("heading", { name: "Past Hydrations" }), + ).toBeVisible(); + // Every kind shows, with no filter over them. + expect( + screen.queryByRole("list", { name: "Show hydrations" }), + ).not.toBeInTheDocument(); + const table = screen.getByRole("table"); + expect( + within(table) + .getAllByRole("columnheader") + .map(({ textContent }) => textContent), + ).toEqual( + expect.arrayContaining(["Peak memory", "Peak disk", "Peak heap"]), + ); + // 6 GiB of the replica's 8 GiB. + expect(within(table).getAllByText("75.0%")).toHaveLength(3); + }); + + it("names each replica once in the legend and dates the time axis", async () => { + await renderHydrationTab({ + droppedReplicas: [ + { + replicaId: "u0", + name: "r1", + createdAtMs: NOW_MS - 6 * DAY_MS, + droppedAtMs: NOW_MS - 4 * DAY_MS, + }, + ], + }); + + const legend = await screen.findByRole("list", { name: "Legend" }); + expect(within(legend).getAllByText("r1")).toHaveLength(1); + expect( + screen.queryByText("From Materialize's hydration history"), + ).not.toBeInTheDocument(); + expect( + screen.getAllByText(/^[A-Z][a-z]{2} \d{1,2}$/).length, + ).toBeGreaterThan(0); + }); + + it("keeps a replica offline between restarts off the clock", async () => { + await renderHydrationTab({ + offlineReplicaIds: ["u1"], + statusRows: [ + online("u1", BOOTED_AT_MS), + online("u2", BOOTED_AT_MS), + outOfMemory("u1", NOW_MS - 30 * MINUTE_MS), + online("u1", NOW_MS - 29 * MINUTE_MS), + outOfMemory("u1", NOW_MS - 10 * MINUTE_MS), + ], + progress: { u1: [3, 9], u2: [9, 9] }, + }); + + expect( + await screen.findByText(/^r1 has no finished hydration recorded since /), + ).toBeVisible(); + expect(screen.getByText("Restarting")).toBeVisible(); + // Its last progress is from before it went offline. + expect( + screen.queryByRole("progressbar", { name: "r1 objects hydrated" }), + ).not.toBeInTheDocument(); + expect(screen.queryByText(/^\d.* so far/)).not.toBeInTheDocument(); + expect( + screen.queryByRole("img", { name: /^r1 is hydrating/ }), + ).not.toBeInTheDocument(); + }); + + it("infers no restarts on environments without hydration history", async () => { + await renderHydrationTab({ + environment: ENVIRONMENT_WITHOUT_HISTORY, + statusRows: [ + online("u1", BOOTED_AT_MS), + online("u2", BOOTED_AT_MS), + outOfMemory("u1", NOW_MS - 36 * 60 * MINUTE_MS), + online("u1", NOW_MS - 36 * 60 * MINUTE_MS + MINUTE_MS), + ], + progress: { u1: [8, 9], u2: [9, 9] }, + }); + + expect( + await screen.findByRole("progressbar", { name: "r1 objects hydrated" }), + ).toBeVisible(); + expect(screen.queryByText(/no finished hydration/)).not.toBeInTheDocument(); + expect(screen.queryByText("Restarting")).not.toBeInTheDocument(); + }); + + it("says the history failed when the replicas' status fails to load", async () => { + await renderHydrationTab({ isStatusError: true }); + + expect( + await screen.findByText("Hydration history couldn't be loaded."), + ).toBeVisible(); + expect(screen.queryByRole("table")).not.toBeInTheDocument(); + }); + + it("leaves out a new object's hydration time until its creation time loads", async () => { + // Relative to this test's start, as the fixtures' `NOW_MS` is not. + const createdAtMs = Date.now() - 30_000; + await renderHydrationTab({ + progressRows: [ + ["u1", "u10", true], + ["u1", "u11", true], + ["u1", "u12", false], + ["u2", "u10", true], + ["u2", "u11", true], + ["u2", "u12", true], + ], + }); + expect(await screen.findByRole("table")).toBeVisible(); + server.use( + buildSqlQueryHandlerV2( + { + queryKey: clusterQueryKeys.objectCreationTimes({ + objectIds: ["u10", "u11", "u12"], + }), + results: mapKyselyToTabular({ + rows: [ + { + id: "u12", + objectType: "index", + occurredAt: String(createdAtMs), + }, + ], + columns: [timestamptz("occurredAt")], + }), + }, + { waitTimeMs: 200 }, + ), + ); + + // The catalog has the new index before its creation time loads. + act(() => + allObjectsCollection.applySnapshot({ + data: [ + catalogObject("u10", "orders_idx", "index"), + catalogObject("u11", "customer_ltv", "materialized-view"), + catalogObject("u12", "orders_by_day_idx", "index"), + ], + snapshotComplete: true, + error: undefined, + }), + ); + + expect( + screen.getByRole("progressbar", { name: "r1 objects hydrated" }), + ).toBeVisible(); + expect(screen.queryByText(/^\d.* so far/)).not.toBeInTheDocument(); + expect(await screen.findByText(/^3\d(\.\d+)?s so far/)).toBeVisible(); + }); + + it("calls out an offline replica rather than its last progress", async () => { + await renderHydrationTab({ offlineReplicaIds: ["u2"] }); + + expect(await screen.findByText("r2 is offline.")).toBeVisible(); + const inProgress = screen.getByRole("heading", { + name: "Hydration in Progress", + }); + expect(inProgress).toBeVisible(); + expect(screen.getByText("Offline")).toBeVisible(); + }); + + it("says a cluster with no replicas has nothing hydrated", async () => { + await renderHydrationTab({ hasReplicas: false }); + + expect( + await screen.findByText( + "This cluster has no replicas, so nothing on it is hydrated.", + ), + ).toBeVisible(); + expect( + screen.queryByRole("heading", { name: "Hydration in Progress" }), + ).not.toBeInTheDocument(); + }); + + it("says a cluster without indexes or materialized views has nothing to hydrate", async () => { + allObjectsCollection.applySnapshot({ + data: [], + snapshotComplete: true, + error: undefined, + }); + await renderHydrationTab({ progress: { u1: [0, 0], u2: [0, 0] } }); + + expect( + await screen.findByText( + "This cluster has no indexes or materialized views, so it has nothing to hydrate.", + ), + ).toBeVisible(); + expect(await screen.findByRole("table")).toBeVisible(); + expect(screen.queryByRole("status")).not.toBeInTheDocument(); + expect( + screen.queryByRole("heading", { name: "Hydration in Progress" }), + ).not.toBeInTheDocument(); + }); + + it("explains each kind of hydration from a tip that opens on keyboard focus", async () => { + await renderHydrationTab(); + + const tip = await screen.findByRole("img", { + name: "About Additive hydration", + }); + act(() => tip.focus()); + + expect(await screen.findByRole("tooltip")).toHaveTextContent( + HYDRATION_HELP.additive, + ); + }); + + it("shows the history without waiting for live progress", async () => { + await renderHydrationTab({ isProgressStuck: true }); + + expect(await screen.findByRole("table")).toBeVisible(); + expect( + screen.getByRole("heading", { name: "Last Successful Hydration" }), + ).toBeVisible(); + }); + + it("narrows to the replica chosen on the cluster page", async () => { + await renderHydrationTab({ search: "?replica=r1" }); + + const table = await screen.findByRole("table"); + expect( + within(table) + .getAllByRole("row") + .slice(1) + .map((row) => within(row).getAllByRole("cell")[1].textContent), + ).toEqual(["r1", "r1"]); + expect(screen.getByLabelText("Replica")).toHaveValue("r1"); + }); + + it("offers a retired replica while its history is in the window", async () => { + await renderHydrationTab({ + droppedReplicas: [ + { + replicaId: "u0", + name: "r1-burst", + createdAtMs: NOW_MS - 3 * DAY_MS, + droppedAtMs: NOW_MS - 2 * DAY_MS, + }, + ], + search: "?replica=r1-burst", + }); + + expect( + await screen.findByRole("option", { name: "r1-burst · 100cc · retired" }), + ).toBeInTheDocument(); + expect(screen.getByLabelText("Replica")).toHaveValue("r1-burst"); + expect( + await screen.findAllByText("No hydrations in this period."), + ).toHaveLength(2); + // Its view is its history, with no alert about the current replicas. + expect(screen.queryByRole("status")).not.toBeInTheDocument(); + }); + + it("keeps live progress when the history fails to load", async () => { + await renderHydrationTab({ + isEpisodesError: true, + progress: { u1: [3, 9], u2: [9, 9] }, + }); + + expect( + await screen.findByText("Hydration history couldn't be loaded."), + ).toBeVisible(); + expect( + await screen.findByRole("progressbar", { name: "r1 objects hydrated" }), + ).toBeVisible(); + }); + + it("waits for the catalog before classifying hydrations and counting new objects", async () => { + allObjectsCollection.reset(); + await renderHydrationTab({ progress: { u1: [3, 9], u2: [9, 9] } }); + + expect( + await screen.findByRole("progressbar", { name: "r1 objects hydrated" }), + ).toBeVisible(); + expect(screen.queryByRole("table")).not.toBeInTheDocument(); + + act(() => + allObjectsCollection.applySnapshot({ + data: [ + catalogObject("u10", "orders_idx", "index"), + catalogObject("u11", "customer_ltv", "materialized-view"), + ], + snapshotComplete: true, + error: undefined, + }), + ); + + expect(await screen.findByRole("table")).toBeVisible(); + const table = screen.getByRole("table"); + expect( + within(table) + .getAllByRole("row") + .slice(1) + .map((row) => within(row).getAllByRole("cell")[2].textContent), + ).toEqual(["Additive", "Full", "Full"]); + }); + + it("shows the history and object timings when the catalog fails to load", async () => { + allObjectsCollection.reset(); + await renderHydrationTab(); + act(() => + allObjectsCollection.applySnapshot({ + data: [], + snapshotComplete: false, + error: { + message: "Something went wrong", + code: ErrorCode.INTERNAL_ERROR, + }, + }), + ); + + expect(await screen.findByRole("table")).toBeVisible(); + // Without the catalog, objects go by their ids. + const waterfall = await screen.findByRole("img", { + name: "Object hydration times", + }); + expect( + within(waterfall).getByText("u11", { selector: "title" }), + ).toBeInTheDocument(); + }); + + it("keeps live progress on environments without hydration history", async () => { + await renderHydrationTab({ + environment: ENVIRONMENT_WITHOUT_HISTORY, + progress: { u1: [3, 9], u2: [9, 9] }, + }); + + expect( + await screen.findByText( + "Hydration history needs Materialize v26.43 or later.", + ), + ).toBeVisible(); + expect( + await screen.findByRole("progressbar", { name: "r1 objects hydrated" }), + ).toBeVisible(); + expect(screen.queryByRole("table")).not.toBeInTheDocument(); + }); +}); diff --git a/console/src/platform/clusters/ClusterPage/Hydration/ClusterHydration.tsx b/console/src/platform/clusters/ClusterPage/Hydration/ClusterHydration.tsx new file mode 100644 index 0000000000000..5ccc7f3c4820e --- /dev/null +++ b/console/src/platform/clusters/ClusterPage/Hydration/ClusterHydration.tsx @@ -0,0 +1,639 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { + Box, + Card, + Flex, + HStack, + Text, + useInterval, + useTheme, + VStack, +} from "@chakra-ui/react"; +import { greatest } from "d3"; +import { millisecondsInSecond } from "date-fns/constants"; +import React from "react"; +import { useParams } from "react-router-dom"; + +import { Cluster } from "~/api/materialize/cluster/clusterList"; +import Alert from "~/components/Alert"; +import ErrorBox from "~/components/ErrorBox"; +import { TooltipColorSwatch } from "~/components/graphComponents"; +import { LoadingContainer } from "~/components/LoadingContainer"; +import TextLink from "~/components/TextLink"; +import TimePeriodSelect from "~/components/TimePeriodSelect"; +import { useTimePeriodMinutes } from "~/hooks/useTimePeriodSelect"; +import { MainContentContainer } from "~/layouts/BaseLayout"; +import docUrls from "~/mz-doc-urls.json"; +import { ClusterParams } from "~/platform/clusters/ClusterRoutes"; +import { CLUSTERS_FETCH_ERROR_MESSAGE } from "~/platform/clusters/constants"; +import { REPLICA_HYDRATION_HISTORY_MIN_VERSION } from "~/platform/clusters/queries"; +import { useAllClusters } from "~/store/allClusters"; +import { MaterializeTheme } from "~/theme"; +import { assert, pluralize } from "~/util"; +import { + formatDate, + FRIENDLY_DATETIME_FORMAT_NO_SECONDS, +} from "~/utils/dateFormat"; +import { formatDurationExact } from "~/utils/format"; + +import { replicaStatus } from "../Configuration/clusterConfigurationModel"; +import { ReplicaSelect } from "../ReplicaSelect"; +import { + assignReplicaColors, + groupReplicasByName, +} from "../ResourceUsage/resourceUsageModel"; +import { useStatusStyles } from "../ResourceUsage/resourceUsageStyles"; +import { useSelectedReplica } from "../useSelectedReplica"; +import { InfoTip } from "./hydrationComponents"; +import { HydrationDetail } from "./HydrationDetail"; +import { + HydrationHistoryChart, + HydrationHistoryLegend, +} from "./HydrationHistoryChart"; +import { HydrationReplica } from "./hydrationModel"; +import { HydrationTable } from "./HydrationTable"; +import { + HYDRATION_HELP, + HydrationVerdict, + hydrationVerdict, + ReplicaNow, + ROW_KIND_NAMES, + rowKind, + rowTime, + summarizeReplicasNow, + toHydrationRows, +} from "./hydrationView"; +import { useClusterHydrationData } from "./useClusterHydrationData"; +import { useLiveObjectHydrations } from "./useLiveHydration"; + +// Status history, which unfinished hydrations come from, is kept 30 days. +const HYDRATION_TIME_PERIODS = { + "1440": "Last 24 hours", + "10080": "Last 7 days", + "43200": "Last 30 days", +}; +const MAX_LIVE_OBJECTS = 8; + +const SectionCard = ({ + title, + subtitle, + children, +}: { + title: string; + subtitle?: string; + children: React.ReactNode; +}) => { + const { colors } = useTheme(); + return ( + + + + + {title} + + {subtitle && ( + + {subtitle} + + )} + + {children} + + + ); +}; + +/** Each replica keeps the color the Metrics tab gives it. */ +const useReplicaColors = (clusterId: string, replicas: HydrationReplica[]) => { + const { colors } = useTheme(); + return React.useMemo(() => { + const withCluster = replicas.map((replica) => ({ ...replica, clusterId })); + const groupIds = groupReplicasByName( + withCluster, + new Map( + replicas.map(({ id, createdAt }) => [id, createdAt?.getTime() ?? 0]), + ), + ); + const groupColors = assignReplicaColors( + withCluster.filter(({ id }) => groupIds.get(id) === id), + colors.lineGraph, + ); + return new Map( + replicas.map(({ id }) => [ + id, + groupColors.get(groupIds.get(id) ?? id) ?? colors.lineGraph[0], + ]), + ); + }, [clusterId, replicas, colors.lineGraph]); +}; + +// "26.43.0" reads "v26.43". +const HISTORY_MIN_VERSION_TEXT = `v${REPLICA_HYDRATION_HISTORY_MIN_VERSION.split(".").slice(0, 2).join(".")}`; + +const ClusterHydrationContent = ({ cluster }: { cluster: Cluster }) => { + const { colors } = useTheme(); + const { selectedReplicaName, setSelectedReplicaName } = + useSelectedReplica(cluster); + const [timePeriodMinutes, setTimePeriodMinutes] = useTimePeriodMinutes({ + localStorageKey: "mz-cluster-hydration-time-period", + defaultValue: "10080", + timePeriodOptions: HYDRATION_TIME_PERIODS, + }); + const data = useClusterHydrationData({ cluster, timePeriodMinutes }); + const colorById = useReplicaColors(cluster.id, data.replicas); + const [selectedRowId, setSelectedRowId] = React.useState(); + + // A retired replica can be selected while its history is in the window. + // Narrowing keeps every replica of the selected name, such as the ones a + // resize replaced. + const selectedName = data.replicas.some( + ({ name }) => name === selectedReplicaName, + ) + ? selectedReplicaName + : undefined; + const replicas = React.useMemo( + () => + selectedName + ? data.replicas.filter(({ name }) => name === selectedName) + : data.replicas, + [data.replicas, selectedName], + ); + const currentNames = new Set(cluster.replicas.map(({ name }) => name)); + const retired = data.replicas.filter( + ({ isCurrent, name }) => !isCurrent && !currentNames.has(name), + ); + const replicaOptions = [ + ...cluster.replicas, + // The newest retired replica of each name. + ...retired + .filter( + ({ name }, index) => + retired.findIndex((other) => other.name === name) === index, + ) + .map(({ id, name, size }) => ({ + id, + name, + size: size ?? null, + isRetired: true, + })), + ]; + // Held across renders, which come every second while a replica hydrates, so + // the table's and chart's memos keyed on them hold too. + const { replicaIds, hydrations, unfinished, rows } = React.useMemo(() => { + const ids = new Set(replicas.map(({ id }) => id)); + const shownHydrations = data.hydrations.filter(({ replicaId }) => + ids.has(replicaId), + ); + const shownUnfinished = data.unfinished.filter(({ replicaId }) => + ids.has(replicaId), + ); + return { + replicaIds: replicas.map(({ id }) => id), + hydrations: shownHydrations, + unfinished: shownUnfinished, + rows: toHydrationRows(shownHydrations, shownUnfinished), + }; + }, [replicas, data.hydrations, data.unfinished]); + const nowMs = Date.now(); + // Only to rerender, so elapsed times move while a replica hydrates. + const [, setTickMs] = React.useState(0); + const isHistoryReady = !data.isHistoryLoading; + const replicasNow = summarizeReplicasNow({ + replicas, + statusById: new Map( + cluster.replicas.map(({ id, statuses }) => [id, replicaStatus(statuses)]), + ), + clusterObjectCount: data.objectCount, + progressByReplicaId: data.progressByReplicaId, + bootTimes: data.bootTimes, + hydrations, + // Restarts are only unfinished once the history shows nothing finished + // after them, so none count before it loads. + unfinished: isHistoryReady ? unfinished : [], + createdObjects: data.createdObjects, + nowMs, + }); + const notHydrated = replicasNow.filter(({ state }) => state !== "hydrated"); + // A retired replica's view is its history alone, so it gets no alert about + // the cluster's current replicas. + const isRetiredSelected = + selectedName !== undefined && !currentNames.has(selectedName); + const verdict = isRetiredSelected + ? undefined + : hydrationVerdict(replicasNow, nowMs); + const nameById = new Map(data.replicas.map(({ id, name }) => [id, name])); + const lastFinished = greatest(hydrations, ({ finishedAt }) => + finishedAt.getTime(), + ); + const explicitRow = rows.find(({ id }) => id === selectedRowId); + const selectedRow = + explicitRow ?? rows.find(({ id }) => id === lastFinished?.id) ?? rows[0]; + const isLastFinishedShown = + !explicitRow && lastFinished !== undefined && selectedRow !== undefined; + // A replica's boot comes from its status, so its start waits for it too. + const inProgress = replicasNow.flatMap(({ replica, isWorking, startedAt }) => + isWorking && startedAt && !data.isProgressLoading + ? [ + { + replicaId: replica.id, + startedAt, + elapsedMs: nowMs - startedAt.getTime(), + }, + ] + : [], + ); + const nameOf = (replicaId: string) => nameById.get(replicaId) ?? replicaId; + useInterval( + () => setTickMs(Date.now()), + inProgress.length > 0 ? millisecondsInSecond : null, + ); + + return ( + + + + + + + {data.objectCount === 0 && cluster.replicas.length > 0 && ( + + This cluster has no indexes or materialized views, so it has nothing + to hydrate. + + )} + {verdict && + !data.isProgressLoading && + isHistoryReady && + !data.isProgressError && + !data.isHistoryError && } + {data.isProgressError ? ( + + + + ) : ( + !data.isProgressLoading && + notHydrated.length > 0 && ( + + + + ) + )} + {data.hasHistory === false ? ( + + Hydration history needs Materialize {HISTORY_MIN_VERSION_TEXT} or + later. + + ) : data.isHistoryError ? ( + + ) : !isHistoryReady ? ( + + ) : ( + <> + + 0} + /> + {rows.length === 0 && inProgress.length === 0 ? ( + + No hydrations in this period. + + ) : ( + + )} + + {selectedRow && ( + + + + )} + + + + + )} + + + ); +}; + +const Verdict = ({ verdict }: { verdict: HydrationVerdict }) => ( + + {verdict.message} See our{" "} + + docs on troubleshooting hydration failures + + . + + ) : ( + verdict.message + ) + } + /> +); + +const ReplicasNow = ({ + clusterName, + replicasNow, + colorById, + nowMs, + isHistoryReady, +}: { + clusterName: string; + replicasNow: ReplicaNow[]; + colorById: Map; + nowMs: number; + /** Until the history loads, nothing is said about past hydrations. */ + isHistoryReady: boolean; +}) => { + const { colors } = useTheme(); + const statusStyles = useStatusStyles(); + const stateStyles = { + hydrated: { label: "Hydrated", color: statusStyles.running.color }, + hydrating: { label: "Hydrating", color: statusStyles.hydrating.color }, + restarting: { label: "Restarting", color: colors.accent.red }, + offline: { label: "Offline", color: statusStyles.offline.color }, + starting: { label: "Starting", color: statusStyles.hydrating.color }, + }; + const stateHelp: Partial> = { + restarting: HYDRATION_HELP.restarting, + offline: HYDRATION_HELP.offline, + starting: HYDRATION_HELP.starting, + }; + return ( + + {replicasNow.map((replicaNow) => { + const { replica, state, progress, isWorking, startedAt, lastFull } = + replicaNow; + const { label, color } = stateStyles[state]; + const help = stateHelp[state]; + const share = + progress.objectCount === 0 + ? 1 + : progress.hydratedCount / progress.objectCount; + const lastFullDuration = + isHistoryReady && lastFull + ? formatDurationExact(lastFull.durationMs) + : undefined; + const detail = + isWorking && startedAt + ? `${formatDurationExact(nowMs - startedAt.getTime())} so far${ + lastFullDuration + ? ` · last full hydration took ${lastFullDuration}` + : "" + }` + : ""; + return ( + + + + + {replica.name} + {replica.size && ( + + {replica.size} + + )} + {/* The progress bar already says a replica is hydrating. */} + {state !== "hydrating" && ( + + + {label} + {help && } + + )} + {isWorking && ( + + Hydrated so far: {progress.hydratedCount} of{" "} + {progress.objectCount}{" "} + {pluralize(progress.objectCount, "object", "objects")} + + )} + + {detail && ( + + {detail} + + )} + + {isWorking && ( + <> + + + + + + )} + + ); + })} + + ); +}; + +/** The objects a hydrating replica is working on, read from the replica. */ +const LiveObjects = ({ + clusterName, + replicaId, + replicaName, + nowMs, +}: { + clusterName: string; + replicaId: string; + replicaName: string; + nowMs: number; +}) => { + const { colors } = useTheme(); + const { objects, snapshotComplete, isError } = useLiveObjectHydrations({ + clusterName, + replicaId, + replicaName, + }); + // The progress above already says how far the replica has got. + if (isError || !snapshotComplete) return null; + const working = objects.filter(({ state }) => state === "working"); + if (working.length === 0) return null; + return ( + + {working.slice(0, MAX_LIVE_OBJECTS).map((object) => ( + + Working on{" "} + + {object.name} + + {object.startedAt && + ` for ${formatDurationExact(nowMs - object.startedAt.getTime())}`} + + ))} + {working.length > MAX_LIVE_OBJECTS && ( + + and {working.length - MAX_LIVE_OBJECTS} more working + + )} + + ); +}; + +/** The Hydration tab of the redesigned cluster page. */ +export const ClusterHydration = () => { + const { clusterId } = useParams(); + assert(clusterId); + const { getClusterById, isError } = useAllClusters(); + const cluster = getClusterById(clusterId); + + // `ClusterRoutes` redirects away from a cluster that doesn't exist, so a + // missing one is still loading, unless the subscribe failed first. + if (!cluster) { + return isError ? ( + + ) : ( + + ); + } + return ; +}; diff --git a/console/src/platform/clusters/ClusterPage/Hydration/HydrationDetail.tsx b/console/src/platform/clusters/ClusterPage/Hydration/HydrationDetail.tsx new file mode 100644 index 0000000000000..d981da1db15e1 --- /dev/null +++ b/console/src/platform/clusters/ClusterPage/Hydration/HydrationDetail.tsx @@ -0,0 +1,455 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { + Box, + Flex, + HStack, + SimpleGrid, + Spinner, + Text, + useTheme, + VStack, +} from "@chakra-ui/react"; +import { useParentSize } from "@visx/responsive"; +import { scaleLinear } from "@visx/scale"; +import { max, sort } from "d3"; +import React from "react"; + +import { SizeAllocation } from "~/api/materialize/cluster/replicaProcessMetrics"; +import { OOM_REASON } from "~/api/materialize/cluster/replicaStatusTimeline"; +import Alert from "~/components/Alert"; +import { PaginationControls } from "~/components/Table/TablePagination"; +import TextLink from "~/components/TextLink"; +import docUrls from "~/mz-doc-urls.json"; +import { MaterializeTheme } from "~/theme"; +import { pluralize } from "~/util"; +import { formatDurationExact, formatPercentage } from "~/utils/format"; + +import { percentOf } from "../ResourceUsage/processReadingsModel"; +import { InfoTip } from "./hydrationComponents"; +import { Hydration, ObjectHydrationSpan } from "./hydrationModel"; +import { formatPeakBytes, HYDRATION_HELP, HydrationRow } from "./hydrationView"; +import { useHydrationDetail } from "./useClusterHydrationData"; + +// A page of objects, enough to see the ones that set an episode's length +// without mounting a row per object on a cluster with hundreds. +const WATERFALL_PAGE_SIZE = 30; +const ROW_HEIGHT_PX = 26; +// The waterfall's legend row and pagination around its rows. +const WATERFALL_CHROME_HEIGHT_PX = 72; +const LABEL_WIDTH_PX = 200; +const DURATION_LABEL_WIDTH_PX = 64; +// A peak this close to the limit is close to running out of memory. +const NEAR_LIMIT_PERCENT = 90; + +export interface HydrationDetailProps { + clusterId: string; + row: HydrationRow; + replicaName: string; + /** The replica's per-process heap limit. */ + heapLimitBytes: number | undefined; + /** The replica's per-process CPU, memory and disk. */ + allocation: SizeAllocation | undefined; +} + +/** What one hydration took, or why it never finished. */ +export const HydrationDetail = (props: HydrationDetailProps) => + props.row.type === "unfinished" ? ( + + ) : ( + + ); + +const UnfinishedDetail = ({ + row, + replicaName, +}: { + row: Extract; + replicaName: string; +}) => { + const { colors } = useTheme(); + return ( + + + {row.reason === OOM_REASON + ? `${replicaName} ran out of memory and restarted with no finished hydration recorded.` + : `${replicaName} went offline with no finished hydration recorded.`} + + + Materialize records object timings only for hydrations that finish. + + + See our{" "} + + docs on troubleshooting hydration failures + + . + + + ); +}; + +const Stat = ({ + label, + value, + note, + help, + isNearLimit = false, +}: { + label: string; + value: string; + note?: string; + help: string; + isNearLimit?: boolean; +}) => { + const { colors } = useTheme(); + const labelId = React.useId(); + return ( + + + + {label} + + + + + {value} + + {note && ( + + {note} + + )} + + ); +}; + +const formatShare = (percent: number | null) => + percent === null ? "-" : formatPercentage(percent / 100, 0); + +/** A peak as a share of its limit, with the bytes it reached. */ +const peakStat = (peakBytes: number | null, limitBytes: number | undefined) => { + const percent = percentOf(peakBytes, limitBytes ?? null); + return { + value: + percent === null + ? (formatPeakBytes(peakBytes) ?? "-") + : formatShare(percent), + note: percent === null ? undefined : formatPeakBytes(peakBytes), + percent, + }; +}; + +const FinishedDetail = ({ + clusterId, + hydration, + heapLimitBytes, + allocation, +}: HydrationDetailProps & { hydration: Hydration }) => { + const { colors } = useTheme(); + const detail = useHydrationDetail({ clusterId, hydration }); + const memory = peakStat(hydration.peakMemoryBytes, allocation?.memoryBytes); + const disk = peakStat( + hydration.peakDiskBytes, + allocation?.diskBytes ?? undefined, + ); + const heap = peakStat(hydration.peakHeapBytes, heapLimitBytes); + const cpuPercent = + detail.peakCpuNanoCores === undefined + ? null + : percentOf(detail.peakCpuNanoCores, allocation?.cpuNanoCores ?? null); + return ( + + + + + + = NEAR_LIMIT_PERCENT + } + /> + + + {detail.isSpansError ? ( + + ) : detail.isSpansLoading || !detail.spans ? ( + // Holds about the waterfall's height, so picking another hydration + // doesn't move the table below. + + + + ) : detail.spans.length === 0 ? ( + + No object timings for this hydration. Materialize keeps them for 30 + days. + + ) : ( + + )} + + ); +}; + +type NamedSpan = ObjectHydrationSpan & { name: string }; + +type WaterfallOrder = "started" | "slowest"; + +const WATERFALL_ORDERS: Record = { + started: "By start", + slowest: "Slowest first", +}; + +const ObjectWaterfall = ({ + spans, + durationMs, +}: { + spans: NamedSpan[]; + durationMs: number; +}) => { + const { colors, fonts } = useTheme(); + const { parentRef, width } = useParentSize({ debounceTime: 10 }); + const [order, setOrder] = React.useState("started"); + const [pageIndex, setPageIndex] = React.useState(0); + // Slowest first compares working times, so each bar starts at zero and the + // time spent waiting is left out. + const isSlowest = order === "slowest"; + const ordered = React.useMemo( + () => + isSlowest + ? sort(spans, (a, b) => (b.workingMs ?? 0) - (a.workingMs ?? 0)) + : spans, + [spans, isSlowest], + ); + const workingEndOf = (span: NamedSpan) => span.hydratedMs ?? durationMs; + const domainMs = isSlowest + ? (max(spans, (span) => workingEndOf(span) - span.startedMs) ?? 0) + : durationMs; + const pageCount = Math.ceil(ordered.length / WATERFALL_PAGE_SIZE); + const shown = ordered.slice( + pageIndex * WATERFALL_PAGE_SIZE, + (pageIndex + 1) * WATERFALL_PAGE_SIZE, + ); + const height = shown.length * ROW_HEIGHT_PX; + const xScale = React.useMemo( + () => + scaleLinear({ + domain: [0, Math.max(domainMs, 1)], + range: [ + LABEL_WIDTH_PX, + Math.max(LABEL_WIDTH_PX + 1, width - DURATION_LABEL_WIDTH_PX), + ], + clamp: true, + }), + [domainMs, width], + ); + return ( + + + + + + + Working + + + {!isSlowest && ( + + + + Waiting + + + + )} + + + {(Object.keys(WATERFALL_ORDERS) as WaterfallOrder[]).map((each) => ( +
  • + { + setOrder(each); + setPageIndex(0); + }} + > + {WATERFALL_ORDERS[each]} + +
    • + ))} +     +     + + {width > 0 && ( + + {shown.map((span, rowIndex) => { + const y = rowIndex * ROW_HEIGHT_PX; + const barY = y + 6; + const barStart = isSlowest ? 0 : span.startedMs; + const barEnd = isSlowest + ? workingEndOf(span) - span.startedMs + : workingEndOf(span); + return ( + + + {span.name.length > 26 + ? `${span.name.slice(0, 25)}…` + : span.name} + {span.name} + + {!isSlowest && span.queuedMs > 0 && ( + + )} + + + {span.workingMs === undefined + ? "-" + : formatDurationExact(span.workingMs)} + + + ); + })} + + )} + + setPageIndex((index) => Math.max(0, index - 1))} + onNext={() => + setPageIndex((index) => Math.min(pageCount - 1, index + 1)) + } + /> +     + ); +}; diff --git a/console/src/platform/clusters/ClusterPage/Hydration/HydrationHistoryChart.tsx b/console/src/platform/clusters/ClusterPage/Hydration/HydrationHistoryChart.tsx new file mode 100644 index 0000000000000..a371a092376c7 --- /dev/null +++ b/console/src/platform/clusters/ClusterPage/Hydration/HydrationHistoryChart.tsx @@ -0,0 +1,579 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { Box, HStack, Text, useTheme, VStack } from "@chakra-ui/react"; +import { useParentSize } from "@visx/responsive"; +import { scaleLinear, scaleTime } from "@visx/scale"; +import { useTooltip, useTooltipInPortal } from "@visx/tooltip"; +import { max } from "d3"; +import { millisecondsInDay } from "date-fns/constants"; +import React from "react"; + +import { OOM_REASON } from "~/api/materialize/cluster/replicaStatusTimeline"; +import { GraphTooltip, TooltipColorSwatch } from "~/components/graphComponents"; +import { MaterializeTheme } from "~/theme"; +import { pluralize } from "~/util"; +import { + formatDate, + FRIENDLY_DATETIME_FORMAT_NO_SECONDS, + TIME_FORMAT_NO_SECONDS, +} from "~/utils/dateFormat"; +import { formatDurationExact } from "~/utils/format"; + +import { TimeAxis } from "../ResourceUsage/ResourceUsageCharts"; +import { CHART_MARGIN } from "../ResourceUsage/resourceUsageStyles"; +import { InfoTip } from "./hydrationComponents"; +import { + HYDRATION_HELP, + HydrationRow, + ROW_KIND_NAMES, + rowTime, +} from "./hydrationView"; + +const PLOT_HEIGHT_PX = 180; +// The lane under the plot for hydrations that never finished. +const LANE_HEIGHT_PX = 22; +const SVG_HEIGHT_PX = + CHART_MARGIN.top + PLOT_HEIGHT_PX + LANE_HEIGHT_PX + CHART_MARGIN.bottom; +const DURATION_TICK_COUNT = 4; +// Durations the axis ticks at, so each label is a round duration. +const DURATION_STEPS_MS = [ + 100, 200, 500, 1_000, 2_000, 5_000, 10_000, 15_000, 30_000, 60_000, 120_000, + 300_000, 600_000, 900_000, 1_800_000, 3_600_000, 7_200_000, 21_600_000, + 43_200_000, 86_400_000, +]; +// Replicas often hydrate at the same moment, so each is drawn a dot's width +// and a gap from the next. Within 12px, the hit area of the dot drawn on top +// covers the other's center and takes its clicks. +const REPLICA_OFFSET_PX = 16; +// A replica restarting in a loop restarts too often to draw each restart, so +// restarts closer than this are one mark, counted in its tooltip. +const RESTART_GROUP_PX = 12; + +// A week or a month reads in dates, and a day in times of day. +const dateTick = (startMs: number, endMs: number) => { + const format = + endMs - startMs > millisecondsInDay ? "MMM d" : TIME_FORMAT_NO_SECONDS; + return (ms: number) => formatDate(new Date(ms), format); +}; + +/** Ticks from zero at a round duration, at most `DURATION_TICK_COUNT` steps. */ +function durationTicks(maxMs: number) { + const stepMs = + DURATION_STEPS_MS.find( + (step) => Math.ceil(maxMs / step) <= DURATION_TICK_COUNT, + ) ?? DURATION_STEPS_MS[DURATION_STEPS_MS.length - 1]; + const count = Math.max(1, Math.ceil(maxMs / stepMs)); + return Array.from({ length: count + 1 }, (_, index) => index * stepMs); +} + +type UnfinishedRow = Extract; + +export interface InProgressHydration { + replicaId: string; + startedAt: Date; + elapsedMs: number; +} + +export interface HydrationHistoryChartProps { + rows: HydrationRow[]; + inProgress: InProgressHydration[]; + /** The replicas to draw, in display order. */ + replicaIds: string[]; + colorById: Map; + nameById: Map; + startMs: number; + endMs: number; + selectedId: string | undefined; + onSelect: (rowId: string) => void; +} + +type TooltipDatum = + | { type: "row"; row: HydrationRow } + | { type: "restarts"; rows: UnfinishedRow[] } + | { type: "inProgress"; hydration: InProgressHydration }; + +const describeRow = (row: HydrationRow, name: string) => { + if (row.type === "unfinished") { + return { + title: `${name} restarted before finishing`, + detail: + row.reason === OOM_REASON + ? "Out of memory" + : "The replica went offline", + time: row.endedAt, + }; + } + return { + title: `${name} · ${ROW_KIND_NAMES[row.kind].label}`, + detail: `${formatDurationExact(row.durationMs)} · ${row.objectCount} ${pluralize(row.objectCount, "object", "objects")}`, + time: row.startedAt, + }; +}; + +const describeRestarts = (rows: UnfinishedRow[], name: string) => { + const oomCount = rows.filter(({ reason }) => reason === OOM_REASON).length; + return { + title: `${name} restarted ${rows.length} times before finishing`, + detail: + oomCount === rows.length + ? "Each after running out of memory" + : oomCount === 0 + ? "Each after going offline" + : `${oomCount} after running out of memory`, + time: rows[rows.length - 1].endedAt, + }; +}; + +/** + * How long each hydration of the cluster's replicas took over the window, + * with restarts that never finished in a lane below. Each mark selects its + * hydration. + */ +export const HydrationHistoryChart = (props: HydrationHistoryChartProps) => { + const { parentRef, width } = useParentSize({ debounceTime: 10 }); + return ( + + {width > 0 && } + + ); +}; + +const HistoryPlot = ({ + width, + rows, + inProgress, + replicaIds, + colorById, + nameById, + startMs, + endMs, + selectedId, + onSelect, +}: HydrationHistoryChartProps & { width: number }) => { + const { colors } = useTheme(); + const { tooltipData, tooltipLeft, tooltipTop, showTooltip, hideTooltip } = + useTooltip(); + const { containerRef, TooltipInPortal } = useTooltipInPortal({ + detectBounds: true, + scroll: true, + }); + + const plotRight = width - CHART_MARGIN.right; + const plotBottom = CHART_MARGIN.top + PLOT_HEIGHT_PX; + const xScale = React.useMemo( + () => + scaleTime({ + domain: [new Date(startMs), new Date(endMs)], + range: [CHART_MARGIN.left, plotRight], + // A hydration that began before the window and finished in it sits + // at the window's start. + clamp: true, + }), + [startMs, endMs, plotRight], + ); + const ticksMs = React.useMemo( + () => + durationTicks( + max([ + ...rows.map((row) => (row.type === "hydration" ? row.durationMs : 0)), + ...inProgress.map(({ elapsedMs }) => elapsedMs), + // An empty or near-instant window still gets a readable axis. + 1000, + ]) ?? 1000, + ), + [rows, inProgress], + ); + const yScale = React.useMemo( + () => + scaleLinear({ + domain: [0, ticksMs[ticksMs.length - 1]], + range: [plotBottom, CHART_MARGIN.top], + }), + [ticksMs, plotBottom], + ); + + const offsetOf = (replicaId: string) => { + const index = replicaIds.indexOf(replicaId); + return (index - (replicaIds.length - 1) / 2) * REPLICA_OFFSET_PX; + }; + const laneY = plotBottom + LANE_HEIGHT_PX / 2; + const restartGroups: Array<{ x: number; rows: UnfinishedRow[] }> = []; + for (const row of rows + .filter((each): each is UnfinishedRow => each.type === "unfinished") + .sort((a, b) => a.endedAt.getTime() - b.endedAt.getTime())) { + const x = xScale(row.endedAt) + offsetOf(row.replicaId); + const group = restartGroups.findLast( + (each) => each.rows[0].replicaId === row.replicaId, + ); + if (group && x - group.x < RESTART_GROUP_PX) { + group.rows.push(row); + group.x = x; + } else { + restartGroups.push({ x, rows: [row] }); + } + } + const show = (datum: TooltipDatum, x: number, y: number) => + showTooltip({ tooltipData: datum, tooltipLeft: x, tooltipTop: y }); + const markProps = (rowId: string, label: string) => ({ + role: "button", + tabIndex: 0, + "aria-label": label, + "aria-pressed": rowId === selectedId, + style: { cursor: "pointer" }, + onClick: () => onSelect(rowId), + onKeyDown: (event: React.KeyboardEvent) => { + if (event.key === "Enter" || event.key === " ") { + event.preventDefault(); + onSelect(rowId); + } + }, + onPointerLeave: hideTooltip, + onBlur: hideTooltip, + }); + + return ( + + + Hydration time + + {/* The tooltip is placed in the svg's coordinates, so its container + starts where the svg does. */} + + + {ticksMs.map((tickMs) => ( + + + + {tickMs === 0 ? "0" : formatDurationExact(tickMs)} + + + ))} + + {restartGroups.map(({ x, rows: groupRows }) => { + const latest = groupRows[groupRows.length - 1]; + const name = nameById.get(latest.replicaId) ?? latest.replicaId; + const isSelected = groupRows.some(({ id }) => id === selectedId); + const { title, detail, time } = + groupRows.length === 1 + ? describeRow(latest, name) + : describeRestarts(groupRows, name); + const label = `${title}, ${detail}, ${formatDate(time, FRIENDLY_DATETIME_FORMAT_NO_SECONDS)}`; + const datum: TooltipDatum = + groupRows.length === 1 + ? { type: "row", row: latest } + : { type: "restarts", rows: groupRows }; + return ( + show(datum, x, laneY)} + onFocus={() => show(datum, x, laneY)} + > + + {isSelected && ( + + )} + + + ); + })} + {rows.flatMap((row) => { + if (row.type === "unfinished") return []; + const color = + colorById.get(row.replicaId) ?? colors.foreground.tertiary; + const name = nameById.get(row.replicaId) ?? row.replicaId; + const cx = xScale(rowTime(row)) + offsetOf(row.replicaId); + const isSelected = row.id === selectedId; + const { title, detail, time } = describeRow(row, name); + const label = `${title}, ${detail}, ${formatDate(time, FRIENDLY_DATETIME_FORMAT_NO_SECONDS)}`; + const cy = yScale(row.durationMs); + const isFull = row.kind === "full"; + return ( + show({ type: "row", row }, cx, cy)} + onFocus={() => show({ type: "row", row }, cx, cy)} + > + + {isSelected && ( + + )} + + + ); + })} + {inProgress.map((hydration) => { + const color = + colorById.get(hydration.replicaId) ?? colors.foreground.tertiary; + const cx = + xScale(hydration.startedAt) + offsetOf(hydration.replicaId); + const cy = yScale(hydration.elapsedMs); + return ( + + show({ type: "inProgress", hydration }, cx, cy) + } + onPointerLeave={hideTooltip} + > + + + + ); + })} + + + + {tooltipData && tooltipLeft !== undefined && tooltipTop !== undefined && ( + + + + )} + + ); +}; + +const HistoryTooltipContent = ({ + datum, + nameById, + colorById, +}: { + datum: TooltipDatum; + nameById: Map; + colorById: Map; +}) => { + const { colors } = useTheme(); + const replicaId = + datum.type === "row" + ? datum.row.replicaId + : datum.type === "restarts" + ? datum.rows[0].replicaId + : datum.hydration.replicaId; + const name = nameById.get(replicaId) ?? replicaId; + const { title, detail, time } = + datum.type === "row" + ? describeRow(datum.row, name) + : datum.type === "restarts" + ? describeRestarts(datum.rows, name) + : { + title: `${name} is hydrating`, + detail: `${formatDurationExact(datum.hydration.elapsedMs)} so far`, + time: datum.hydration.startedAt, + }; + return ( + + + + {title} + + {detail} + + {formatDate(time, FRIENDLY_DATETIME_FORMAT_NO_SECONDS)} + + + ); +}; + +export interface HydrationHistoryLegendProps { + replicas: Array<{ id: string; name: string }>; + colorById: Map; + hasInProgress: boolean; +} + +/** Names each replica's color and each kind of mark in the chart. */ +export const HydrationHistoryLegend = ({ + replicas, + colorById, + hasInProgress, +}: HydrationHistoryLegendProps) => { + const { colors } = useTheme(); + const glyph = (children: React.ReactNode, glyphWidth = 12) => ( + + {children} + + ); + const items: Array<{ + key: string; + swatch: React.ReactNode; + label: string; + help?: string; + }> = [ + // Replicas of one name share a color, as when a resize replaced them. + ...replicas + .filter( + ({ name }, index) => + replicas.findIndex((other) => other.name === name) === index, + ) + .map(({ id, name }) => ({ + key: id, + swatch: ( + + ), + label: name, + })), + { + key: "full", + swatch: glyph( + , + ), + label: ROW_KIND_NAMES.full.label, + help: HYDRATION_HELP.full, + }, + { + key: "new", + swatch: glyph( + , + ), + label: ROW_KIND_NAMES.additive.label, + help: HYDRATION_HELP.additive, + }, + { + key: "unfinished", + swatch: glyph( + , + ), + label: ROW_KIND_NAMES.unfinished.label, + help: HYDRATION_HELP.unfinished, + }, + ...(hasInProgress + ? [ + { + key: "in-progress", + swatch: glyph( + , + ), + label: "In progress", + help: HYDRATION_HELP.inProgress, + }, + ] + : []), + ]; + return ( + + {items.map(({ key, swatch, label, help }) => ( + + {swatch} + + {label} + + {help && } + + ))} + + ); +}; diff --git a/console/src/platform/clusters/ClusterPage/Hydration/HydrationTable.tsx b/console/src/platform/clusters/ClusterPage/Hydration/HydrationTable.tsx new file mode 100644 index 0000000000000..0c4624e3e8086 --- /dev/null +++ b/console/src/platform/clusters/ClusterPage/Hydration/HydrationTable.tsx @@ -0,0 +1,215 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { HStack, Text, useTheme, VStack } from "@chakra-ui/react"; +import { createColumnHelper } from "@tanstack/react-table"; +import React from "react"; + +import { SizeAllocation } from "~/api/materialize/cluster/replicaProcessMetrics"; +import { OOM_REASON } from "~/api/materialize/cluster/replicaStatusTimeline"; +import { TooltipColorSwatch } from "~/components/graphComponents"; +import PercentBar from "~/components/PercentBar"; +import { sortingFunctions } from "~/components/Table/tableColumnBuilders"; +import { TablePagination } from "~/components/Table/TablePagination"; +import { UniversalTable } from "~/components/Table/UniversalTable"; +import { useUniversalTable } from "~/components/Table/useUniversalTable"; +import TextLink from "~/components/TextLink"; +import { MaterializeTheme } from "~/theme"; +import { + formatDate, + FRIENDLY_DATETIME_FORMAT_NO_SECONDS, +} from "~/utils/dateFormat"; +import { formatDurationExact, formatPercentage } from "~/utils/format"; + +import { percentOf } from "../ResourceUsage/processReadingsModel"; +import { + formatPeakBytes, + HYDRATION_HELP, + HydrationRow, + ROW_KIND_NAMES, + rowKind, + rowTime, +} from "./hydrationView"; + +interface HydrationTableMeta { + colorById: Map; + nameById: Map; + heapLimitById: Map; + /** Each replica's per-process CPU, memory and disk. */ + allocationById: Map; + selectedId: string | undefined; + onSelect: (rowId: string) => void; +} + +const columnHelper = createColumnHelper(); + +const SelectCell = ({ + row, + meta, +}: { + row: HydrationRow; + meta: HydrationTableMeta; +}) => ( + meta.onSelect(row.id)} + > + {formatDate(rowTime(row), FRIENDLY_DATETIME_FORMAT_NO_SECONDS)} + +); + +const ReplicaCell = ({ + replicaId, + meta, +}: { + replicaId: string; + meta: HydrationTableMeta; +}) => { + const { colors } = useTheme(); + return ( + + + {meta.nameById.get(replicaId) ?? replicaId} + + ); +}; + +const COLUMNS = [ + columnHelper.accessor((row) => rowTime(row).getTime(), { + id: "started", + header: "Time", + meta: { + tooltip: "When the hydration began, or when the replica restarted.", + }, + cell: (info) => ( + + ), + }), + columnHelper.accessor("replicaId", { + header: "Replica", + cell: (info) => ( + + ), + }), + columnHelper.accessor(rowKind, { + id: "kind", + header: "Hydration", + meta: { tooltip: HYDRATION_HELP.kind }, + cell: (info) => ROW_KIND_NAMES[info.getValue()].short, + }), + columnHelper.accessor( + (row) => (row.type === "hydration" ? row.durationMs : null), + { + id: "duration", + header: "Duration", + sortingFn: sortingFunctions.numericNullsLast, + meta: { isNumeric: true, tooltip: HYDRATION_HELP.duration }, + cell: (info) => { + const value = info.getValue(); + return value === null ? "-" : formatDurationExact(value); + }, + }, + ), + columnHelper.accessor( + (row) => (row.type === "hydration" ? row.objectCount : null), + { + id: "objects", + header: "Objects", + sortingFn: sortingFunctions.numericNullsLast, + meta: { isNumeric: true, tooltip: HYDRATION_HELP.objects }, + cell: (info) => info.getValue() ?? "-", + }, + ), + columnHelper.display({ + id: "peakMemory", + header: "Peak memory", + meta: { tooltip: HYDRATION_HELP.peakMemory }, + cell: (info) => { + const row = info.row.original; + if (row.type === "unfinished") return "-"; + const meta = info.table.options.meta as HydrationTableMeta; + const percent = percentOf( + row.peakMemoryBytes, + meta.allocationById.get(row.replicaId)?.memoryBytes ?? null, + ); + return percent === null ? ( + (formatPeakBytes(row.peakMemoryBytes) ?? "-") + ) : ( + + ); + }, + }), + columnHelper.display({ + id: "peakDisk", + header: "Peak disk", + meta: { isNumeric: true, tooltip: HYDRATION_HELP.peakDisk }, + cell: (info) => { + const row = info.row.original; + return row.type === "hydration" + ? (formatPeakBytes(row.peakDiskBytes) ?? "-") + : "-"; + }, + }), + columnHelper.display({ + id: "peakHeap", + header: "Peak heap", + meta: { isNumeric: true, tooltip: HYDRATION_HELP.peakHeap }, + cell: (info) => { + const row = info.row.original; + if (row.type === "unfinished") { + return row.reason === OOM_REASON ? "Out of memory" : "-"; + } + const meta = info.table.options.meta as HydrationTableMeta; + const percent = percentOf( + row.peakHeapBytes, + meta.heapLimitById.get(row.replicaId) ?? null, + ); + const peakBytes = formatPeakBytes(row.peakHeapBytes) ?? "-"; + return percent === null + ? peakBytes + : `${formatPercentage(percent / 100, 0)} · ${peakBytes}`; + }, + }), +]; + +export interface HydrationTableProps extends HydrationTableMeta { + rows: HydrationRow[]; +} + +/** Every hydration in the window. */ +export const HydrationTable = ({ rows, ...meta }: HydrationTableProps) => { + const { colors } = useTheme(); + const table = useUniversalTable({ + data: rows, + columns: COLUMNS, + meta, + getRowId: ({ id }) => id, + }); + return rows.length === 0 ? ( + + No hydrations in this period. + + ) : ( + + + + + ); +}; diff --git a/console/src/platform/clusters/ClusterPage/Hydration/hydrationComponents.tsx b/console/src/platform/clusters/ClusterPage/Hydration/hydrationComponents.tsx new file mode 100644 index 0000000000000..2105fd2c05129 --- /dev/null +++ b/console/src/platform/clusters/ClusterPage/Hydration/hydrationComponents.tsx @@ -0,0 +1,41 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { Box, Tooltip, useTheme } from "@chakra-ui/react"; +import React from "react"; + +import { InfoIcon } from "~/icons"; +import { MaterializeTheme } from "~/theme"; + +export interface InfoTipProps { + /** What the tip explains, which names it for screen readers. */ + subject: string; + label: string; +} + +/** An info icon that shows `label` on hover or keyboard focus. */ +export const InfoTip = ({ subject, label }: InfoTipProps) => { + const { colors } = useTheme(); + return ( + + + + + + ); +}; diff --git a/console/src/platform/clusters/ClusterPage/Hydration/hydrationModel.ts b/console/src/platform/clusters/ClusterPage/Hydration/hydrationModel.ts index 8881165d70e45..28eec42bb0a86 100644 --- a/console/src/platform/clusters/ClusterPage/Hydration/hydrationModel.ts +++ b/console/src/platform/clusters/ClusterPage/Hydration/hydrationModel.ts @@ -191,8 +191,9 @@ export function transformHydrations( * created during the attempt starts a hydration of its own, so the attempt * finished only if a hydration finished after the last such object. * - * NOTE: the history can miss an episode that finished, so a restart after - * one can read as unfinished. + * NOTE: the collector visits one replica per interval, so a hydration that + * finished before a restart and before the replica's next visit is never + * recorded, and that restart reads as unfinished. */ export function findUnfinishedHydrations({ transitions, diff --git a/console/src/platform/clusters/ClusterPage/Hydration/hydrationView.test.ts b/console/src/platform/clusters/ClusterPage/Hydration/hydrationView.test.ts new file mode 100644 index 0000000000000..4124e1e1480e5 --- /dev/null +++ b/console/src/platform/clusters/ClusterPage/Hydration/hydrationView.test.ts @@ -0,0 +1,349 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { + formatDate, + FRIENDLY_DATETIME_FORMAT_NO_SECONDS, +} from "~/utils/dateFormat"; + +import { ReplicaStatus } from "../Configuration/clusterConfigurationModel"; +import { + Hydration, + HydrationReplica, + UnfinishedHydration, +} from "./hydrationModel"; +import { + hydrationVerdict, + summarizeReplicasNow, + toHydrationRows, +} from "./hydrationView"; + +const at = (time: string) => new Date(`2026-10-20T${time}Z`); + +const replica = (id: string, name: string): HydrationReplica => ({ + id, + name, + size: "100cc", + isCurrent: true, + createdAt: undefined, + droppedAt: undefined, +}); + +const hydration = ( + replicaId: string, + startedAt: string, + durationMs: number, +): Hydration => ({ + id: `${replicaId}|${startedAt}`, + replicaId, + startedAt: at(startedAt), + finishedAt: new Date(at(startedAt).getTime() + durationMs), + durationMs, + objectCount: 8, + peakHeapBytes: null, + peakMemoryBytes: null, + peakDiskBytes: null, + kind: "full", +}); + +const restart = ( + replicaId: string, + endedAt: string, + reason: string | null = "oom-killed", +): UnfinishedHydration => ({ + id: `${replicaId}|${endedAt}`, + replicaId, + startedAt: undefined, + endedAt: at(endedAt), + reason, +}); + +const summarize = ({ + progress, + hydrations = [], + unfinished = [], + bootTimes = new Map(), + createdObjects = [], + isCreationTimesUnknown = false, + statuses = {}, + clusterObjectCount = 9, + nowMs = at("11:22:00").getTime(), +}: { + /** Hydrated and total objects, and the ids not hydrated if they matter. */ + progress: Record; + hydrations?: Hydration[]; + unfinished?: UnfinishedHydration[]; + bootTimes?: Map; + createdObjects?: Array<{ id: string; occurredAt: Date }>; + isCreationTimesUnknown?: boolean; + /** By replica id. A replica not listed is online. */ + statuses?: Record; + clusterObjectCount?: number; + nowMs?: number; +}) => + summarizeReplicasNow({ + replicas: [replica("u1", "r1"), replica("u2", "r2")], + statusById: new Map( + ["u1", "u2"].map((id) => [id, statuses[id] ?? { kind: "online" }]), + ), + clusterObjectCount, + progressByReplicaId: new Map( + Object.entries(progress).map( + ([id, [hydratedCount, objectCount, unhydratedObjectIds]]) => [ + id, + { + hydratedCount, + objectCount, + unhydratedObjectIds: + unhydratedObjectIds ?? + Array.from( + { length: objectCount - hydratedCount }, + (_, index) => `u${100 + index}`, + ), + }, + ], + ), + ), + bootTimes, + hydrations, + unfinished, + createdObjects: isCreationTimesUnknown ? undefined : createdObjects, + nowMs, + }); + +describe("toHydrationRows", () => { + it("orders finished hydrations by start and unfinished ones by end, newest first", () => { + const rows = toHydrationRows( + [ + hydration("u1", "09:00:00", 60_000), + hydration("u1", "11:00:00", 60_000), + ], + [restart("u1", "10:00:00")], + ); + + expect(rows.map(({ type, id }) => [type, id])).toEqual([ + ["hydration", "u1|11:00:00"], + ["unfinished", "u1|10:00:00"], + ["hydration", "u1|09:00:00"], + ]); + }); +}); + +describe("summarizeReplicasNow", () => { + it("counts only restarts since a replica last finished hydrating", () => { + const replicasNow = summarize({ + progress: { u1: [3, 9], u2: [9, 9] }, + hydrations: [hydration("u1", "09:00:00", 60_000)], + unfinished: [restart("u1", "08:00:00"), restart("u1", "10:00:00")], + }); + + expect( + replicasNow.map(({ replica: { id }, state, restarts }) => [ + id, + state, + restarts.map(({ id: restartId }) => restartId), + ]), + ).toEqual([ + ["u1", "restarting", ["u1|10:00:00"]], + ["u2", "hydrated", []], + ]); + }); + + it("dates a hydration from the replica's boot", () => { + const [, r2] = summarize({ + progress: { u1: [9, 9], u2: [5, 9] }, + bootTimes: new Map([["u2", [at("11:19:50").getTime()]]]), + }); + + expect(r2).toMatchObject({ + state: "hydrating", + isWorking: true, + startedAt: at("11:19:50"), + }); + }); + + it("dates a hydration of new objects from the first one's creation, not the replica's boot", () => { + const [r1] = summarize({ + progress: { u1: [8, 9, ["u10"]], u2: [9, 9] }, + hydrations: [hydration("u1", "08:00:00", 280_000)], + // A boot after the last hydration finished, as a zero-downtime + // upgrade records. + bootTimes: new Map([["u1", [at("08:20:00").getTime()]]]), + createdObjects: [{ id: "u10", occurredAt: at("11:20:30") }], + }); + + expect(r1.startedAt).toEqual(at("11:20:30")); + }); + + it("dates a hydration from the boot when an object from before it isn't hydrated", () => { + const [r1] = summarize({ + progress: { u1: [7, 9, ["u10", "u11"]] }, + bootTimes: new Map([["u1", [at("11:19:50").getTime()]]]), + createdObjects: [{ id: "u10", occurredAt: at("11:21:00") }], + }); + + expect(r1.startedAt).toEqual(at("11:19:50")); + }); + + it("leaves out when a replica began hydrating until creation times load", () => { + const [, r2] = summarize({ + progress: { u1: [9, 9], u2: [5, 9] }, + bootTimes: new Map([["u2", [at("11:19:50").getTime()]]]), + isCreationTimesUnknown: true, + }); + + expect(r2).toMatchObject({ isWorking: true, startedAt: undefined }); + }); + + it("keeps an offline replica restarting in a loop off the clock", () => { + const [r1] = summarize({ + progress: { u1: [7, 10], u2: [10, 10] }, + unfinished: [restart("u1", "11:00:00")], + bootTimes: new Map([["u1", [at("10:59:00").getTime()]]]), + statuses: { u1: { kind: "offline", reason: "oom-killed" } }, + }); + + expect(r1).toMatchObject({ + state: "restarting", + isWorking: false, + startedAt: undefined, + }); + }); +}); + +describe("hydrationVerdict", () => { + const NOW_MS = at("11:22:00").getTime(); + + it("calls out replicas that keep restarting before they finish", () => { + const verdict = hydrationVerdict( + summarize({ + progress: { u1: [3, 10], u2: [2, 10] }, + unfinished: [ + restart("u1", "10:31:00"), + restart("u2", "10:33:00"), + restart("u1", "10:47:00"), + ], + }), + NOW_MS, + ); + + expect(verdict).toEqual({ + variant: "error", + label: `r1 and r2 have no finished hydration recorded since ${formatDate(at("10:31:00"), FRIENDLY_DATETIME_FORMAT_NO_SECONDS)}.`, + message: + "Between them, they restarted 3 times, each after running out of memory.", + isFailing: true, + }); + }); + + it("leaves a hydrating replica to the progress card", () => { + expect( + hydrationVerdict( + summarize({ + progress: { u1: [9, 9], u2: [5, 9] }, + bootTimes: new Map([["u2", [at("11:19:50").getTime()]]]), + }), + NOW_MS, + ), + ).toBeUndefined(); + }); + + it("calls out an offline replica while another hydrates", () => { + expect( + hydrationVerdict( + summarize({ + progress: { u1: [3, 9], u2: [9, 9] }, + statuses: { u2: { kind: "offline", reason: null } }, + }), + NOW_MS, + ), + ).toMatchObject({ variant: "warning", label: "r2 is offline." }); + }); + + it("has nothing to say once every replica is hydrated", () => { + expect( + hydrationVerdict( + summarize({ + progress: { u1: [9, 9], u2: [9, 9] }, + hydrations: [hydration("u2", "10:00:00", 271_000)], + }), + NOW_MS, + ), + ).toBeUndefined(); + }); + + it("has nothing to say for a cluster with nothing to hydrate", () => { + expect( + hydrationVerdict( + summarize({ + progress: { u1: [0, 0], u2: [0, 0] }, + clusterObjectCount: 0, + }), + NOW_MS, + ), + ).toBeUndefined(); + }); + + it("calls out an offline replica rather than its last progress", () => { + const replicasNow = summarize({ + progress: { u1: [9, 9], u2: [9, 9] }, + statuses: { u2: { kind: "offline", reason: null } }, + }); + + expect(replicasNow.map(({ state }) => state)).toEqual([ + "hydrated", + "offline", + ]); + expect(hydrationVerdict(replicasNow, NOW_MS)).toMatchObject({ + variant: "warning", + label: "r2 is offline.", + }); + }); + + it("stops calling a replica starting once it has been up a while without reporting", () => { + // As for a cluster with introspection turned off, whose replicas never + // report. + const replicasNow = summarize({ + progress: { u1: [0, 0], u2: [0, 0] }, + bootTimes: new Map([ + ["u1", [at("10:00:00").getTime()]], + ["u2", [at("10:00:00").getTime()]], + ]), + }); + + expect(replicasNow.map(({ state }) => state)).toEqual([ + "hydrated", + "hydrated", + ]); + expect(hydrationVerdict(replicasNow, NOW_MS)).toBeUndefined(); + }); + + it("calls a replica that hasn't reported starting, on a cluster with objects", () => { + const replicasNow = summarize({ + progress: { u1: [9, 9], u2: [0, 0] }, + bootTimes: new Map([["u2", [at("11:21:00").getTime()]]]), + }); + + expect(replicasNow.map(({ state }) => state)).toEqual([ + "hydrated", + "starting", + ]); + expect(hydrationVerdict(replicasNow, NOW_MS)).toMatchObject({ + variant: "warning", + label: "r2 is starting.", + }); + }); + + it("explains a cluster with no replicas", () => { + expect(hydrationVerdict([], NOW_MS)).toMatchObject({ + variant: "info", + label: "This cluster has no replicas, so nothing on it is hydrated.", + }); + }); +}); diff --git a/console/src/platform/clusters/ClusterPage/Hydration/hydrationView.ts b/console/src/platform/clusters/ClusterPage/Hydration/hydrationView.ts new file mode 100644 index 0000000000000..ad926c32dde97 --- /dev/null +++ b/console/src/platform/clusters/ClusterPage/Hydration/hydrationView.ts @@ -0,0 +1,329 @@ +// Copyright Materialize, Inc. and contributors. All rights reserved. +// +// Use of this software is governed by the Business Source License +// included in the LICENSE file. +// +// As of the Change Date specified in that file, in accordance with +// the Business Source License, use of this software will be governed +// by the Apache License, Version 2.0. + +import { greatest, min } from "d3"; +import { millisecondsInMinute } from "date-fns/constants"; + +import { OOM_REASON } from "~/api/materialize/cluster/replicaStatusTimeline"; +import { AlertVariant } from "~/components/Alert"; +import { + formatDate, + FRIENDLY_DATETIME_FORMAT_NO_SECONDS, +} from "~/utils/dateFormat"; +import { formatBytesShort } from "~/utils/format"; + +import { ReplicaStatus } from "../Configuration/clusterConfigurationModel"; +import { + Hydration, + HydrationReplica, + latestBoot, + latestFullHydration, + UnfinishedHydration, +} from "./hydrationModel"; +import { ReplicaHydrationProgress } from "./useLiveHydration"; + +/** A finished hydration or one a replica restarted before finishing. */ +export type HydrationRow = + | ({ type: "hydration" } & Hydration) + | ({ type: "unfinished" } & UnfinishedHydration); + +export type HydrationRowKind = "full" | "additive" | "unfinished"; + +export const rowKind = (row: HydrationRow): HydrationRowKind => + row.type === "unfinished" ? "unfinished" : row.kind; + +/** How the tab names each kind of row: in text, in a heading, and in a column. */ +export const ROW_KIND_NAMES: Record< + HydrationRowKind, + { label: string; title: string; short: string } +> = { + full: { label: "Full hydration", title: "Full Hydration", short: "Full" }, + additive: { + label: "Additive hydration", + title: "Additive Hydration", + short: "Additive", + }, + unfinished: { + label: "Restarted before finishing", + title: "Restart", + short: "Restarted before finishing", + }, +}; + +/** Bytes for display, undefined when not recorded. */ +export const formatPeakBytes = (bytes: number | null) => + bytes === null ? undefined : formatBytesShort(BigInt(Math.round(bytes))); + +/** When a row happened: a hydration's start, or an unfinished one's end. */ +export const rowTime = (row: HydrationRow) => + row.type === "hydration" ? row.startedAt : row.endedAt; + +/** Hydrations and unfinished ones, newest first. */ +export const toHydrationRows = ( + hydrations: Hydration[], + unfinished: UnfinishedHydration[], +): HydrationRow[] => + [ + ...hydrations.map((row) => ({ ...row, type: "hydration" as const })), + ...unfinished.map((row) => ({ ...row, type: "unfinished" as const })), + ].sort((a, b) => rowTime(b).getTime() - rowTime(a).getTime()); + +/** + * `restarting` is hydrating again after restarting since it last finished. + * `starting` is up but hasn't reported its objects yet. + */ +export type ReplicaHydrationState = + "hydrated" | "hydrating" | "restarting" | "offline" | "starting"; + +export interface ReplicaNow { + replica: HydrationReplica; + state: ReplicaHydrationState; + progress: ReplicaHydrationProgress; + /** + * Hydrating now: online and short of its objects. A replica restarting in a + * loop can be offline between attempts. + */ + isWorking: boolean; + /** When its current hydration began, if it's working and that's known. */ + startedAt: Date | undefined; + lastFull: Hydration | undefined; + /** Times it restarted without finishing, since it last finished. */ + restarts: UnfinishedHydration[]; +} + +/** + * When a replica's current hydration began. If every object it hasn't hydrated + * was created after its latest boot, it's hydrating new objects and began with + * the first of them. Otherwise it's rehydrating since that boot. Asking whether + * a hydration finished since the boot fails after a zero-downtime upgrade, + * which records the boot after the replica already hydrated. + */ +function currentHydrationStart( + bootedAt: Date | undefined, + unhydratedObjectIds: string[], + createdAtMsById: Map | undefined, +) { + if (!createdAtMsById) return undefined; + const bootMs = bootedAt?.getTime() ?? -Infinity; + const createdMs = unhydratedObjectIds.map((id) => createdAtMsById.get(id)); + const isNewObjects = + createdMs.length > 0 && + createdMs.every((ms) => ms !== undefined && ms > bootMs); + if (!isNewObjects) return bootedAt; + const firstMs = min(createdMs, (ms) => ms); + return firstMs === undefined ? undefined : new Date(firstMs); +} + +/** + * How long after booting a replica that hasn't reported its objects reads as + * starting. A replica can also never report, as with the cluster's + * introspection turned off, so after this it's no longer called starting. + */ +const STARTING_GRACE_MS = 2 * millisecondsInMinute; + +/** + * Where each current replica's hydration stands. A replica's live progress + * outlives its process, so an offline replica's last progress is ignored. + */ +export function summarizeReplicasNow({ + replicas, + statusById, + clusterObjectCount, + progressByReplicaId, + bootTimes, + hydrations, + unfinished, + createdObjects, + nowMs, +}: { + replicas: HydrationReplica[]; + statusById: Map; + /** The cluster's indexes and materialized views, undefined until known. */ + clusterObjectCount: number | undefined; + progressByReplicaId: Map; + bootTimes: Map; + hydrations: Hydration[]; + unfinished: UnfinishedHydration[]; + /** + * When the cluster's objects were created, undefined until known. An object + * missing from it is taken to be older than the replica's boot. + */ + createdObjects: Array<{ id: string | null; occurredAt: Date }> | undefined; + nowMs: number; +}): ReplicaNow[] { + const createdAtMsById = + createdObjects && + new Map( + createdObjects.flatMap(({ id, occurredAt }) => + id ? [[id, occurredAt.getTime()]] : [], + ), + ); + return replicas + .filter(({ isCurrent }) => isCurrent) + .map((replica) => { + const progress = progressByReplicaId.get(replica.id) ?? { + objectCount: 0, + hydratedCount: 0, + unhydratedObjectIds: [], + }; + const lastFinishedMs = + greatest( + hydrations.filter(({ replicaId }) => replicaId === replica.id), + ({ finishedAt }) => finishedAt.getTime(), + )?.finishedAt.getTime() ?? -Infinity; + const restarts = unfinished.filter( + ({ replicaId, endedAt }) => + replicaId === replica.id && endedAt.getTime() > lastFinishedMs, + ); + const isHydrating = progress.hydratedCount < progress.objectCount; + const isOffline = statusById.get(replica.id)?.kind === "offline"; + const isWorking = isHydrating && !isOffline; + // A replica reports nothing until its first process is up, which reads as + // nothing to hydrate on a cluster that has objects. + const bootedAt = latestBoot(bootTimes, replica.id); + const hasNotReported = + progress.objectCount === 0 && + (clusterObjectCount ?? 0) > 0 && + bootedAt !== undefined && + nowMs - bootedAt.getTime() < STARTING_GRACE_MS; + const state: ReplicaHydrationState = + restarts.length > 0 && (isOffline || isHydrating || hasNotReported) + ? "restarting" + : isOffline + ? "offline" + : hasNotReported + ? "starting" + : isHydrating + ? "hydrating" + : "hydrated"; + return { + replica, + state, + progress, + isWorking, + startedAt: isWorking + ? currentHydrationStart( + bootedAt, + progress.unhydratedObjectIds, + createdAtMsById, + ) + : undefined, + lastFull: latestFullHydration(hydrations, replica.id), + restarts, + }; + }); +} + +export interface HydrationVerdict { + variant: AlertVariant; + label: string; + message: string | undefined; + /** Whether to point at the hydration troubleshooting guides. */ + isFailing: boolean; +} + +const LIST_FORMAT = new Intl.ListFormat("en", { type: "conjunction" }); + +const restartsSentence = (restarts: UnfinishedHydration[], subject: string) => { + const times = restarts.length === 1 ? "once" : `${restarts.length} times`; + const isOom = restarts.every(({ reason }) => reason === OOM_REASON); + if (!isOom) return `${subject} restarted ${times}.`; + return restarts.length === 1 + ? `${subject} restarted once, after running out of memory.` + : `${subject} restarted ${times}, each after running out of memory.`; +}; + +/** + * One sentence on what needs attention in the cluster's hydration, with the + * numbers behind it. Undefined when every replica is hydrated or hydrating, + * which the progress card already shows. + */ +export function hydrationVerdict( + replicasNow: ReplicaNow[], + nowMs: number, +): HydrationVerdict | undefined { + if (replicasNow.length === 0) { + return { + variant: "info", + label: "This cluster has no replicas, so nothing on it is hydrated.", + message: undefined, + isFailing: false, + }; + } + + const restarting = replicasNow.filter(({ state }) => state === "restarting"); + if (restarting.length > 0) { + const restarts = restarting.flatMap(({ restarts: each }) => each); + // Since the first attempt that never finished began, when that's recorded. + const sinceMs = + min(restarts, ({ startedAt, endedAt }) => + (startedAt ?? endedAt).getTime(), + ) ?? nowMs; + const names = LIST_FORMAT.format( + restarting.map(({ replica }) => replica.name), + ); + return { + variant: "error", + label: `${names} ${restarting.length === 1 ? "has" : "have"} no finished hydration recorded since ${formatDate(sinceMs, FRIENDLY_DATETIME_FORMAT_NO_SECONDS)}.`, + message: restartsSentence( + restarts, + restarting.length === 1 ? "It" : "Between them, they", + ), + isFailing: true, + }; + } + + const offline = replicasNow.filter(({ state }) => state === "offline"); + if (offline.length > 0) { + return { + variant: "warning", + label: `${LIST_FORMAT.format(offline.map(({ replica }) => replica.name))} ${offline.length === 1 ? "is" : "are"} offline.`, + message: + offline.length === 1 + ? "It hydrates once it's back online." + : "They hydrate once they're back online.", + isFailing: false, + }; + } + const starting = replicasNow.filter(({ state }) => state === "starting"); + if (starting.length > 0) { + return { + variant: "warning", + label: `${LIST_FORMAT.format(starting.map(({ replica }) => replica.name))} ${starting.length === 1 ? "is" : "are"} starting.`, + message: "Hydration progress shows once the replica reports it.", + isFailing: false, + }; + } + return undefined; +} + +/** What the tab's terms mean, shared by the chart, detail and table. */ +export const HYDRATION_HELP = { + full: "Hydrating every object on a replica, as after it starts or restarts. Inferred: Materialize doesn't record why a hydration began.", + additive: + "Hydrating new objects, such as a new index, on a replica that already has the others. Inferred: Materialize doesn't record why a hydration began.", + kind: "Full or additive. Inferred: Materialize doesn't record why a hydration began.", + unfinished: + "Inferred from replica status: it went offline or ran out of memory with no finished hydration recorded. Materialize records hydrations by visiting replicas in turn, so one that finished shortly before the restart can show here.", + inProgress: "Still hydrating, placed at how long it has taken so far.", + hydrationSlot: "A replica hydrates a few objects at a time, 4 by default.", + duration: "From the first dataflow installed to the last object hydrated.", + objects: "Indexes and materialized views in the hydration.", + peakMemory: + "The most memory one process used, as a share of its memory. Materialize records the peak since the process started, so it can predate this hydration.", + peakDisk: + "The most disk one process used, as a share of its disk. Recorded since the process started.", + peakHeap: + "Memory plus disk, as a share of the process's heap limit. A process that reaches the limit runs out of memory. Recorded since the process started.", + cpu: "The highest CPU sample during the hydration. Materialize samples CPU once a minute.", + restarting: + "Inferred from replica status: it restarted with no finished hydration recorded.", + offline: "The replica is offline, so it isn't hydrating.", + starting: "The replica hasn't reported its objects yet.", +}; diff --git a/console/src/platform/clusters/ClusterPage/Hydration/useClusterHydrationData.ts b/console/src/platform/clusters/ClusterPage/Hydration/useClusterHydrationData.ts index 5d32e1a670c32..e1a496ebe2de7 100644 --- a/console/src/platform/clusters/ClusterPage/Hydration/useClusterHydrationData.ts +++ b/console/src/platform/clusters/ClusterPage/Hydration/useClusterHydrationData.ts @@ -39,7 +39,7 @@ import { useLiveHydrationProgress } from "./useLiveHydration"; const COMPUTE_OBJECT_TYPES = ["index", "materialized-view"]; -/** An object created on the cluster in the window, named from the catalog. */ +/** When an object on the cluster was created, named from the catalog. */ export interface CreatedObject { id: string | null; name: string | undefined; @@ -79,9 +79,19 @@ export function useClusterHydrationData({ ), [cluster.replicas, replicaHistory.data, startMs], ); - const statusHistory = useReplicaStatusHistory( - replicas.map(({ id }) => id), + // The current replicas' status loads at once. Dropped replicas are only + // known after the replica history, and their status no longer changes, so + // a query of their own keeps them from refetching the current replicas'. + const currentStatus = useReplicaStatusHistory( + currentReplicaIds, replicaStatusesUpdatedAt(cluster.replicas), + // A resize or burst changes the current replicas, and the history's + // restarts would otherwise drop out until the new set loads. + { keepOverlapping: true }, + ); + const droppedStatus = useReplicaStatusHistory( + replicas.filter(({ isCurrent }) => !isCurrent).map(({ id }) => id), + undefined, ); const episodes = useClusterHydrationEpisodes({ clusterId: cluster.id, @@ -94,9 +104,10 @@ export function useClusterHydrationData({ snapshotComplete: hasObjects, isError: isObjectsError, } = useClusterObjectsLive(cluster.id, COMPUTE_OBJECT_TYPES); + // Not bound to the window: a replica hydrating an object created before it + // is dated from that creation. const creationTimes = useObjectCreationTimes({ objectIds: objects.map(({ id }) => id).sort(), - timePeriodMinutes, }); // Undefined until the catalog and the creation times load. A cluster with no // objects has nothing to look up. @@ -122,10 +133,13 @@ export function useClusterHydrationData({ ]); const derived = React.useMemo(() => { - const transitions = statusHistory.data?.transitions ?? []; + const transitions = [ + ...(currentStatus.data?.transitions ?? []), + ...(droppedStatus.data?.transitions ?? []), + ]; const bootTimes = replicaBootTimes(transitions, replicas); - // Objects created after a time weren't on the cluster then. Ones created - // before the window have no row, so they count throughout. + // Objects created after a time weren't on the cluster then. Ones with no + // creation row count throughout. // NOTE: dropped objects aren't counted, so after a drop a hydration of // only some objects can read as full. const objectCountAt = createdObjects @@ -143,17 +157,25 @@ export function useClusterHydrationData({ const hydrations = allHydrations.filter( ({ objectCount }) => objectCount > 0, ); - const unfinished = findUnfinishedHydrations({ - transitions, - oomKills: statusHistory.data?.oomKills ?? [], - hydrations: allHydrations, - replicas, - bootTimes, - objectCreatedAtMs: (createdObjects ?? []).map(({ occurredAt }) => - occurredAt.getTime(), - ), - startMs, - }); + // Without the history nothing finished is recorded, so every restart + // would read as unfinished. + const unfinished = + hasHistory === true + ? findUnfinishedHydrations({ + transitions, + oomKills: [ + ...(currentStatus.data?.oomKills ?? []), + ...(droppedStatus.data?.oomKills ?? []), + ], + hydrations: allHydrations, + replicas, + bootTimes, + objectCreatedAtMs: (createdObjects ?? []).map(({ occurredAt }) => + occurredAt.getTime(), + ), + startMs, + }) + : []; const heapLimitBytesById = new Map( replicas.flatMap((replica) => { const limit = heapLimitBytes( @@ -164,9 +186,24 @@ export function useClusterHydrationData({ return limit === undefined ? [] : [[replica.id, limit]]; }), ); - return { bootTimes, hydrations, unfinished, heapLimitBytesById, startMs }; + const allocationById = new Map( + replicas.flatMap(({ id, size }) => { + const allocation = size ? allocations.data?.get(size) : undefined; + return allocation ? [[id, allocation]] : []; + }), + ); + return { + bootTimes, + hydrations, + unfinished, + heapLimitBytesById, + allocationById, + startMs, + }; }, [ - statusHistory.data, + hasHistory, + currentStatus.data, + droppedStatus.data, objects, createdObjects, episodes.data, @@ -182,20 +219,34 @@ export function useClusterHydrationData({ replicas, ...derived, progressByReplicaId: progress.progressByReplicaId, - isProgressLoading: !progress.snapshotComplete, - isLoading: + /** Live progress and the current replicas' status, which say where each stands. */ + isProgressLoading: !progress.snapshotComplete || currentStatus.isLoading, + /** + * The hydration history, whose restarts come from the current replicas' + * status. Dropped replicas' restarts fill in later. + */ + isHistoryLoading: + hasHistory === null || replicaHistory.isLoading || - statusHistory.isLoading || + currentStatus.isLoading || (hasHistory === true && (episodes.isLoading || (!createdObjects && !creationTimes.isError && !isObjectsError))), - createdObjects, - isError: + /** + * When the cluster's objects were created, undefined until known for the + * objects it has now. + */ + createdObjects: creationTimes.isPlaceholderData + ? undefined + : createdObjects, + /** The cluster's indexes and materialized views, undefined until known. */ + objectCount: hasObjects ? objects.length : undefined, + isProgressError: progress.isError || currentStatus.isError, + isHistoryError: replicaHistory.isError || - statusHistory.isError || - episodes.isError || - progress.isError, - isCreatedObjectsError: creationTimes.isError || isObjectsError, + currentStatus.isError || + droppedStatus.isError || + episodes.isError, }; } diff --git a/console/src/platform/clusters/ClusterPage/ReplicaSelect.tsx b/console/src/platform/clusters/ClusterPage/ReplicaSelect.tsx index a85fab88b9776..63a13d15a52b0 100644 --- a/console/src/platform/clusters/ClusterPage/ReplicaSelect.tsx +++ b/console/src/platform/clusters/ClusterPage/ReplicaSelect.tsx @@ -14,7 +14,13 @@ import React from "react"; import SimpleSelect from "~/components/SimpleSelect"; export interface ReplicaSelectProps { - replicas: Array<{ id: string; name: string; size: string | null }>; + replicas: Array<{ + id: string; + name: string; + size: string | null; + /** A dropped replica, offered for its history. */ + isRetired?: boolean; + }>; /** The selected replica's name, or undefined for all replicas. */ value: string | undefined; onChange: (replicaName: string | undefined) => void; @@ -45,13 +51,17 @@ export const ReplicaSelect = ({ onChange={(event) => onChange(event.target.value || undefined)} > {includeAll && } - {sort(replicas, ({ name }) => name).map( - ({ id: replicaId, name, size }) => ( - - ), - )} + {sort( + replicas, + ({ isRetired }) => Boolean(isRetired), + ({ name }) => name, + ).map(({ id: replicaId, name, size, isRetired }) => ( + + ))} ); diff --git a/console/src/platform/clusters/ClusterPage/ResourceUsage/processReadingsModel.ts b/console/src/platform/clusters/ClusterPage/ResourceUsage/processReadingsModel.ts index b79888a14c1fb..20452bc9d5363 100644 --- a/console/src/platform/clusters/ClusterPage/ResourceUsage/processReadingsModel.ts +++ b/console/src/platform/clusters/ClusterPage/ResourceUsage/processReadingsModel.ts @@ -25,7 +25,7 @@ export interface ProcessBucket { readings: Map; } -const percentOf = (value: number | null, total: number | null) => +export const percentOf = (value: number | null, total: number | null) => value === null || !total ? null : (value / total) * 100; const greatestOf = (a: number | null | undefined, b: number | null) => diff --git a/console/src/platform/clusters/ClusterPage/tabLinks.ts b/console/src/platform/clusters/ClusterPage/tabLinks.ts index b438944da3f3b..1b091095a64d0 100644 --- a/console/src/platform/clusters/ClusterPage/tabLinks.ts +++ b/console/src/platform/clusters/ClusterPage/tabLinks.ts @@ -12,6 +12,7 @@ import { To } from "react-router-dom"; import { REPLICA_SEARCH_PARAM } from "./useSelectedReplica"; export const WORKERS_TAB_PATH = "workers"; +export const HYDRATION_TAB_PATH = "hydration"; /** The search string that selects `replicaName`, or none for all replicas. */ export const replicaSearch = (replicaName: string | undefined) => diff --git a/console/src/platform/clusters/ClusterPage/useSelectedReplica.ts b/console/src/platform/clusters/ClusterPage/useSelectedReplica.ts index 1989393498eca..8d1c0fcbdcdce 100644 --- a/console/src/platform/clusters/ClusterPage/useSelectedReplica.ts +++ b/console/src/platform/clusters/ClusterPage/useSelectedReplica.ts @@ -14,11 +14,11 @@ import { Cluster } from "~/api/materialize/cluster/clusterList"; export const REPLICA_SEARCH_PARAM = "replica"; /** - * The replica the Metrics and Workers tabs are narrowed to, by name in the - * `replica` search param, so a narrowed view is a link. `selectedReplica` is - * undefined when the URL names none or a name no current replica has. The - * stale name stays in the URL, so the selection comes back if a replica of - * that name does. + * The replica the Metrics, Workers and Hydration tabs are narrowed to, by name + * in the `replica` search param, so a narrowed view is a link. + * `selectedReplica` is undefined when the URL names none or a name no current + * replica has. The stale name stays in the URL, so the selection comes back if + * a replica of that name does. */ export function useSelectedReplica(cluster: Cluster) { const [searchParams, setSearchParams] = useSearchParams(); diff --git a/console/src/platform/clusters/queries.ts b/console/src/platform/clusters/queries.ts index ed50c67055086..26c7e44a35163 100644 --- a/console/src/platform/clusters/queries.ts +++ b/console/src/platform/clusters/queries.ts @@ -340,7 +340,7 @@ export const clusterQueryKeys = { ] as const, objectCreationTimes: (params: { objectIds: string[]; - timePeriodMinutes: number; + timePeriodMinutes?: number; }) => [ ...clusterQueryKeys.all(), @@ -1261,10 +1261,14 @@ export function replicaStatusesUpdatedAt(replicas: Cluster["replicas"]) { * Status transitions and OOM kills of the given replicas, fetched again only * when `statusesUpdatedAt` changes. Pass `replicaStatusesUpdatedAt` of the * cluster's replicas. Idle while `replicaIds` is empty. + * + * `keepOverlapping` keeps the previous answer up while a new set of replicas + * shares one with it, for a view that derives history from it. */ export function useReplicaStatusHistory( replicaIds: string[], statusesUpdatedAt: number | undefined, + { keepOverlapping = false }: { keepOverlapping?: boolean } = {}, ) { return useQuery({ queryKey: clusterQueryKeys.replicaStatusHistory({ @@ -1281,13 +1285,15 @@ export function useReplicaStatusHistory( requestOptions: { signal }, }); }, - // A new transition changes the key. The same replicas' history stays up - // meanwhile, but never other replicas'. After `queryFn`, so the data type - // is inferred from it. - placeholderData: (previous, previousQuery) => - previousQuery?.queryKey[1].replicaIds.join(",") === replicaIds.join(",") - ? previous - : undefined, + // A new transition changes the key, and the same replicas' history stays + // up meanwhile. After `queryFn`, so the data type is inferred from it. + placeholderData: (previous, previousQuery) => { + const previousIds = previousQuery?.queryKey[1].replicaIds ?? []; + const isKept = keepOverlapping + ? previousIds.some((id) => replicaIds.includes(id)) + : previousIds.join(",") === replicaIds.join(","); + return isKept ? previous : undefined; + }, }); } @@ -1721,10 +1727,13 @@ export function useReplicaHeapBytes(replicaIds: string[]) { }); } -/** Creation times of the given objects over the window. Idle while `objectIds` is empty. */ +/** + * Creation times of the given objects over the window, or whenever they were + * created without one. Idle while `objectIds` is empty. + */ export function useObjectCreationTimes(params: { objectIds: string[]; - timePeriodMinutes: number; + timePeriodMinutes?: number; }) { return useQuery({ queryKey: clusterQueryKeys.objectCreationTimes(params), @@ -1737,10 +1746,13 @@ export function useObjectCreationTimes(params: { params: { objectIds: queryKeyParams.objectIds, objectTypes: ["index", "materialized-view"], - startDate: subMinutes( - new Date(), - queryKeyParams.timePeriodMinutes, - ).toISOString(), + startDate: + queryKeyParams.timePeriodMinutes === undefined + ? undefined + : subMinutes( + new Date(), + queryKeyParams.timePeriodMinutes, + ).toISOString(), }, queryKey, requestOptions: { signal },