Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions data-track/embeds/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
# Data Track embeds

Static HTML (and assets) hosted for **Notion iframes** in the HYF Data Track curriculum.

GitHub Pages: after Pages is enabled on this repo, decks are served from:

`https://hackyourfuture.github.io/Learning-Resources/data-track/embeds/...`
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added data-track/embeds/week-13-slides/dbx_compute.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added data-track/embeds/week-13-slides/dbx_job_runs.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Binary file added data-track/embeds/week-13-slides/dbx_sidebar.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
@@ -0,0 +1,357 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Code Explainer: Incremental fct_trips — compile fork</title>
<style>
* { margin: 0; padding: 0; box-sizing: border-box; }
body {
background: #0d1117;
display: flex;
justify-content: center;
align-items: flex-start;
min-height: 100vh;
font-family: 'SF Pro Text', -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
padding: 10px;
color: #c9d1d9;
}

.explainer-container {
width: 100%;
max-width: 980px;
background: #161b22;
border-radius: 10px;
overflow: hidden;
box-shadow: 0 16px 36px rgba(0,0,0,0.6);
border: 1px solid #30363d;
display: flex;
flex-direction: column;
}

.titlebar {
background: #0d1117;
padding: 10px 14px;
display: flex;
align-items: center;
gap: 8px;
border-bottom: 1px solid #21262d;
user-select: none;
flex-wrap: wrap;
}
.dot { width: 11px; height: 11px; border-radius: 50%; flex-shrink: 0; }
.dot.r { background: #ff5f56; }
.dot.y { background: #ffbd2e; }
.dot.g { background: #27c93f; }
.titlebar .title {
color: #8b949e;
font-size: 12px;
font-family: 'SF Mono', 'Fira Code', monospace;
font-weight: 500;
margin-right: auto;
}

.toggle {
display: inline-flex;
background: #21262d;
border: 1px solid #30363d;
border-radius: 8px;
overflow: hidden;
}
.toggle button {
appearance: none;
border: 0;
background: transparent;
color: #8b949e;
font: inherit;
font-size: 12px;
font-weight: 600;
padding: 7px 12px;
cursor: pointer;
transition: background 0.15s ease, color 0.15s ease;
}
.toggle button:hover { color: #f0f6fc; }
.toggle button.active {
background: #1f6feb;
color: #fff;
}
.toggle button.active.inc {
background: #238636;
}

.chips {
display: flex;
flex-wrap: wrap;
gap: 8px;
padding: 10px 14px;
border-bottom: 1px solid #21262d;
background: #0d1117;
}
.chip {
font-size: 12px;
font-weight: 600;
padding: 4px 10px;
border-radius: 999px;
border: 1px solid #30363d;
color: #c9d1d9;
background: #161b22;
font-family: 'SF Mono', 'Fira Code', monospace;
}
.chip.false { color: #ffa198; border-color: rgba(248, 81, 73, 0.45); background: rgba(248, 81, 73, 0.1); }
.chip.true { color: #3fb950; border-color: rgba(63, 185, 80, 0.45); background: rgba(63, 185, 80, 0.1); }
.chip.warn { color: #ffa657; border-color: rgba(255, 166, 87, 0.45); background: rgba(255, 166, 87, 0.1); }
.chip.ok { color: #58a6ff; border-color: rgba(88, 166, 255, 0.45); background: rgba(88, 166, 255, 0.1); }

.panes {
display: grid;
grid-template-columns: 1fr 1fr;
min-height: 0;
}
@media (max-width: 720px) {
.panes { grid-template-columns: 1fr; }
}

.pane {
display: flex;
flex-direction: column;
min-width: 0;
border-right: 1px solid #21262d;
}
.pane:last-child { border-right: 0; }
.pane-label {
padding: 8px 14px;
font-size: 11px;
font-weight: 700;
letter-spacing: 0.04em;
text-transform: uppercase;
color: #8b949e;
border-bottom: 1px solid #21262d;
background: #161b22;
}

.code-body {
padding: 10px 0;
font-family: 'SF Mono', 'Fira Code', 'Menlo', monospace;
font-size: 12px;
line-height: 1.55;
overflow: auto;
background: #0d1117;
max-height: 360px;
}

.code-line {
display: flex;
padding: 1px 12px;
user-select: text;
}
.line-num {
width: 28px;
color: #484f58;
text-align: right;
padding-right: 12px;
user-select: none;
flex-shrink: 0;
}
.line-content { white-space: pre; flex: 1; }

.kwd { color: #ff7b72; font-weight: 600; }
.fn { color: #d2a8ff; }
.str { color: #a5d6ff; }
.cmnt { color: #8b949e; font-style: italic; }
.var { color: #ffa657; }
.jja { color: #79c0ff; font-weight: 600; }

/* Compile-fork highlighting on the Jinja guard */
.jinja-guard {
border-left: 3px solid transparent;
transition: background 0.2s ease, border-color 0.2s ease, opacity 0.2s ease;
}
body[data-mode="first"] .jinja-guard {
opacity: 0.42;
border-left-color: #484f58;
background: rgba(72, 79, 88, 0.12);
}
body[data-mode="incremental"] .jinja-guard {
opacity: 1;
border-left-color: #3fb950;
background: rgba(63, 185, 80, 0.12);
}

body[data-mode="incremental"] .compiled-filter {
border-left: 3px solid #58a6ff;
background: rgba(88, 166, 255, 0.12);
}

.ghost-note {
display: none;
margin: 6px 12px 2px;
padding: 6px 10px;
border-radius: 6px;
font-size: 11px;
line-height: 1.4;
color: #8b949e;
border: 1px dashed #30363d;
background: #161b22;
}
body[data-mode="first"] .ghost-note.first { display: block; }
body[data-mode="incremental"] .ghost-note.inc { display: block; color: #3fb950; border-color: rgba(63, 185, 80, 0.35); }

.dock {
background: #161b22;
border-top: 1px solid #30363d;
padding: 12px 16px;
min-height: 72px;
}
.dock-kicker {
font-size: 11px;
font-weight: 700;
letter-spacing: 0.04em;
text-transform: uppercase;
color: #8b949e;
margin-bottom: 4px;
}
.dock-body {
font-size: 13px;
line-height: 1.45;
color: #f0f6fc;
}
.dock-body strong { color: #ffa657; font-weight: 700; }
</style>
</head>
<body data-mode="first">
<div class="explainer-container">
<div class="titlebar">
<span class="dot r"></span>
<span class="dot y"></span>
<span class="dot g"></span>
<span class="title">Compile fork · 1st build ↔ 2nd+</span>
<div class="toggle" role="group" aria-label="dbt run mode">
<button type="button" id="btn-first" class="active" aria-pressed="true">1st build / full-refresh</button>
<button type="button" id="btn-inc" class="inc" aria-pressed="false">2nd+ build (incremental)</button>
</div>
</div>

<div class="chips" aria-live="polite">
<span class="chip" id="chip-macro">is_incremental() = false</span>
<span class="chip" id="chip-scan">scans ~128M rows</span>
<span class="chip" id="chip-write">creates fct_trips</span>
</div>

<div class="panes">
<section class="pane" aria-label="Model source">
<div class="pane-label">Model (Jinja — same file both runs)</div>
<div class="code-body">
<div class="code-line"><span class="line-num">1</span><span class="line-content"><span class="jja">{{</span></span></div>
<div class="code-line"><span class="line-num">2</span><span class="line-content"> <span class="fn">config</span>(</span></div>
<div class="code-line"><span class="line-num">3</span><span class="line-content"> materialized=<span class="str">'incremental'</span>,</span></div>
<div class="code-line"><span class="line-num">4</span><span class="line-content"> incremental_strategy=<span class="str">'merge'</span>,</span></div>
<div class="code-line"><span class="line-num">5</span><span class="line-content"> unique_key=<span class="str">'trip_id'</span></span></div>
<div class="code-line"><span class="line-num">6</span><span class="line-content"> )</span></div>
<div class="code-line"><span class="line-num">7</span><span class="line-content"><span class="jja">}}</span></span></div>
<div class="code-line"><span class="line-num">8</span><span class="line-content"></span></div>
<div class="code-line"><span class="line-num">9</span><span class="line-content"><span class="kwd">select</span> t.trip_id, t.pickup_datetime, …</span></div>
<div class="code-line"><span class="line-num">10</span><span class="line-content"><span class="kwd">from</span> <span class="jja">{{</span> <span class="fn">ref</span>(<span class="str">'stg_trips'</span>) <span class="jja">}}</span> t</span></div>
<div class="code-line"><span class="line-num">11</span><span class="line-content"><span class="kwd">left join</span> <span class="jja">{{</span> <span class="fn">ref</span>(<span class="str">'stg_zones'</span>) <span class="jja">}}</span> pz …</span></div>
<div class="code-line"><span class="line-num">12</span><span class="line-content"></span></div>
<div class="ghost-note first">On this run the guard is <strong style="color:#ffa198">false</strong> — Jinja deletes the block before SQL reaches the warehouse.</div>
<div class="ghost-note inc">On this run the guard is <strong style="color:#3fb950">true</strong> — the <code>where</code> survives into compiled SQL.</div>
<div class="code-line jinja-guard"><span class="line-num">13</span><span class="line-content"><span class="jja">{%</span> <span class="kwd">if</span> <span class="fn">is_incremental</span>() <span class="jja">%}</span></span></div>
<div class="code-line jinja-guard"><span class="line-num">14</span><span class="line-content"> <span class="cmnt">-- only trips newer than what we already have</span></span></div>
<div class="code-line jinja-guard"><span class="line-num">15</span><span class="line-content"> <span class="kwd">where</span> t.pickup_datetime &gt; (</span></div>
<div class="code-line jinja-guard"><span class="line-num">16</span><span class="line-content"> <span class="kwd">select</span> <span class="fn">max</span>(pickup_datetime) <span class="kwd">from</span> <span class="jja">{{</span> <span class="var">this</span> <span class="jja">}}</span></span></div>
<div class="code-line jinja-guard"><span class="line-num">17</span><span class="line-content"> )</span></div>
<div class="code-line jinja-guard"><span class="line-num">18</span><span class="line-content"><span class="jja">{%</span> <span class="kwd">endif</span> <span class="jja">%}</span></span></div>
</div>
</section>

<section class="pane" aria-label="Compiled SQL">
<div class="pane-label">Compiled SQL (what the warehouse runs)</div>
<div class="code-body" id="compiled-first">
<div class="code-line"><span class="line-num">1</span><span class="line-content"><span class="cmnt">-- is_incremental() was false → no WHERE</span></span></div>
<div class="code-line"><span class="line-num">2</span><span class="line-content"><span class="kwd">select</span> t.trip_id, t.pickup_datetime, …</span></div>
<div class="code-line"><span class="line-num">3</span><span class="line-content"><span class="kwd">from</span> hyf.dev_yourname.stg_trips t</span></div>
<div class="code-line"><span class="line-num">4</span><span class="line-content"><span class="kwd">left join</span> hyf.dev_yourname.stg_zones pz …</span></div>
<div class="code-line"><span class="line-num">5</span><span class="line-content"></span></div>
<div class="code-line"><span class="line-num">6</span><span class="line-content"><span class="cmnt">-- (filter block compiled away)</span></span></div>
</div>
<div class="code-body" id="compiled-inc" hidden>
<div class="code-line"><span class="line-num">1</span><span class="line-content"><span class="cmnt">-- is_incremental() was true → watermark filter stays</span></span></div>
<div class="code-line"><span class="line-num">2</span><span class="line-content"><span class="kwd">select</span> t.trip_id, t.pickup_datetime, …</span></div>
<div class="code-line"><span class="line-num">3</span><span class="line-content"><span class="kwd">from</span> hyf.dev_yourname.stg_trips t</span></div>
<div class="code-line"><span class="line-num">4</span><span class="line-content"><span class="kwd">left join</span> hyf.dev_yourname.stg_zones pz …</span></div>
<div class="code-line"><span class="line-num">5</span><span class="line-content"></span></div>
<div class="code-line compiled-filter"><span class="line-num">6</span><span class="line-content"><span class="kwd">where</span> t.pickup_datetime &gt; (</span></div>
<div class="code-line compiled-filter"><span class="line-num">7</span><span class="line-content"> <span class="kwd">select</span> <span class="fn">max</span>(pickup_datetime)</span></div>
<div class="code-line compiled-filter"><span class="line-num">8</span><span class="line-content"> <span class="kwd">from</span> hyf.dev_yourname.fct_trips <span class="cmnt">-- {{ this }}</span></span></div>
<div class="code-line compiled-filter"><span class="line-num">9</span><span class="line-content">)</span></div>
<div class="code-line"><span class="line-num">10</span><span class="line-content"><span class="cmnt">-- then Delta MERGE on trip_id</span></span></div>
</div>
</section>
</div>

<div class="dock">
<div class="dock-kicker">Say this out loud</div>
<div class="dock-body" id="say-aloud">
Same model file. First build: table missing → <strong>is_incremental() = false</strong> → filter deleted → full scan.
</div>
</div>
</div>

<script>
const btnFirst = document.getElementById('btn-first');
const btnInc = document.getElementById('btn-inc');
const chipMacro = document.getElementById('chip-macro');
const chipScan = document.getElementById('chip-scan');
const chipWrite = document.getElementById('chip-write');
const compiledFirst = document.getElementById('compiled-first');
const compiledInc = document.getElementById('compiled-inc');
const sayAloud = document.getElementById('say-aloud');

const modes = {
first: {
macroClass: 'false',
scanClass: 'warn',
writeClass: 'warn',
macro: 'is_incremental() = false',
scan: 'scans ~128M rows',
write: 'creates fct_trips',
say: 'Same model file. First build: table missing → <strong>is_incremental() = false</strong> → filter deleted → full scan.'
},
incremental: {
macroClass: 'true',
scanClass: 'ok',
writeClass: 'true',
macro: 'is_incremental() = true',
scan: 'only rows after watermark',
write: 'MERGE into fct_trips',
say: 'Same model file. Second build: table exists → <strong>is_incremental() = true</strong> → <strong>{{ this }}</strong> watermark stays → that is why it is faster, not a warm warehouse.'
}
};

function setMode(mode) {
document.body.dataset.mode = mode;
const m = modes[mode];

btnFirst.classList.toggle('active', mode === 'first');
btnInc.classList.toggle('active', mode === 'incremental');
btnFirst.setAttribute('aria-pressed', mode === 'first' ? 'true' : 'false');
btnInc.setAttribute('aria-pressed', mode === 'incremental' ? 'true' : 'false');

chipMacro.textContent = m.macro;
chipScan.textContent = m.scan;
chipWrite.textContent = m.write;
chipMacro.className = 'chip ' + m.macroClass;
chipScan.className = 'chip ' + m.scanClass;
chipWrite.className = 'chip ' + m.writeClass;

compiledFirst.hidden = mode !== 'first';
compiledInc.hidden = mode !== 'incremental';
sayAloud.innerHTML = m.say;
}

btnFirst.addEventListener('click', () => setMode('first'));
btnInc.addEventListener('click', () => setMode('incremental'));
setMode('first');
</script>
</body>
</html>
Loading
Loading