From c1cb63c7d9c2d79777aa7c669c2017c17819a4c9 Mon Sep 17 00:00:00 2001 From: Lasse Benninga Date: Thu, 25 Jun 2026 10:31:39 +0200 Subject: [PATCH 1/5] fix(data-track/week-9): clarify validation expectations, deterministic sample, assignment cast note Polish from a Week 9 exercise review (all solutions live-verified against the shared Azure Postgres: Manhattan top, 0 NULL pickups, 189 duplicate groups, 0 orphans, fact view 56,369 = 56,551 - 182 negatives). - Ex1: add ORDER BY pickup_datetime before LIMIT 5 so the sample is stable between runs (a bare LIMIT returns an arbitrary 5 rows). - Ex5: tell students 5a returns 0 and 5c returns empty BY DESIGN (pickup IDs are complete; orphans do not exist on this load). A 0 is the check passing, not a mistake. Prevents the "did I break it?" confusion. - Ex6: note the practice view is scaled down; the assignment's vw_fact_trips also casts pickup_datetime::TIMESTAMP. Flag it so students do not under-build the assignment deliverable. Co-Authored-By: Claude Opus 4.8 (1M context) --- data-track/week-9/exercise_1/exercise.sql | 4 +++- data-track/week-9/exercise_1/solutions/exercise.sql | 1 + data-track/week-9/exercise_5/README.md | 2 +- data-track/week-9/exercise_5/exercise.sql | 2 ++ data-track/week-9/exercise_6/README.md | 4 +++- data-track/week-9/exercise_6/exercise.sql | 3 +++ 6 files changed, 13 insertions(+), 3 deletions(-) diff --git a/data-track/week-9/exercise_1/exercise.sql b/data-track/week-9/exercise_1/exercise.sql index 2d8e15b..d6bd727 100644 --- a/data-track/week-9/exercise_1/exercise.sql +++ b/data-track/week-9/exercise_1/exercise.sql @@ -10,6 +10,8 @@ -- Run this against your OWN schema on the shared Azure PostgreSQL, not public. -- -- Hint: The bridge between the two tables is t.pickup_location_id = z.location_id. --- Use an INNER JOIN and LIMIT 5. +-- Use an INNER JOIN. Add ORDER BY t.pickup_datetime before LIMIT 5 so the +-- five rows are the same every run (a bare LIMIT returns an arbitrary +-- sample that can change between runs). -- TODO: select the four columns and join nyc_taxi.raw_trips to nyc_taxi.raw_zones, then limit to 5 rows. diff --git a/data-track/week-9/exercise_1/solutions/exercise.sql b/data-track/week-9/exercise_1/solutions/exercise.sql index 7f077e5..16a6864 100644 --- a/data-track/week-9/exercise_1/solutions/exercise.sql +++ b/data-track/week-9/exercise_1/solutions/exercise.sql @@ -8,4 +8,5 @@ SELECT FROM nyc_taxi.raw_trips t INNER JOIN nyc_taxi.raw_zones z -- WHY INNER JOIN: every trip we keep must have a matching zone; trips with no match are dropped, which is fine here ON t.pickup_location_id = z.location_id -- WHY this ON: the numeric pickup_location_id in the fact table maps to location_id in the dimension +ORDER BY t.pickup_datetime -- WHY ORDER BY: without it, LIMIT returns an arbitrary 5 rows that can differ between runs; ordering makes the sample stable and reproducible LIMIT 5; -- WHY LIMIT 5: we only need a small sample to confirm the join reads correctly, not all 57K rows diff --git a/data-track/week-9/exercise_5/README.md b/data-track/week-9/exercise_5/README.md index 468d91f..c746413 100644 --- a/data-track/week-9/exercise_5/README.md +++ b/data-track/week-9/exercise_5/README.md @@ -14,6 +14,6 @@ Open `exercise.sql` and run each of the three queries against your own schema on ## Success criteria -Each check returns a clear answer. For 5a you get a single count. For 5b and 5c, an empty result means the check passed: any rows returned are the problems to report. +Each check returns a clear answer. For 5a you get a single count, and on this dataset that count is **0**: the pickup location IDs are complete. That is the point of the check, not a sign you wrote it wrong. The real issues live elsewhere: 5b surfaces duplicate trips, and the negative-fare and NULL `payment_type` problems show up in the chapter. For 5b and 5c, an empty result means the check passed: any rows returned are the problems to report. 5c also comes back empty here (every pickup ID resolves to a zone), which is the correct passing outcome, not a missed bug. Stuck? The reference queries are in `solutions/exercise.sql`, try for 10 to 20 minutes first. diff --git a/data-track/week-9/exercise_5/exercise.sql b/data-track/week-9/exercise_5/exercise.sql index 1568323..627e10b 100644 --- a/data-track/week-9/exercise_5/exercise.sql +++ b/data-track/week-9/exercise_5/exercise.sql @@ -17,6 +17,8 @@ -- 5a. Trips with a missing pickup location -- TODO: count rows where pickup_location_id IS NULL. +-- Expect 0 here: the pickup IDs are complete. A 0 is the check passing, +-- not a mistake. The real dirt is duplicates (5b) and negative fares. -- 5b. Duplicate trips (same vendor + pickup + dropoff time) diff --git a/data-track/week-9/exercise_6/README.md b/data-track/week-9/exercise_6/README.md index 5b6330c..ce1f8ab 100644 --- a/data-track/week-9/exercise_6/README.md +++ b/data-track/week-9/exercise_6/README.md @@ -14,6 +14,8 @@ Open `exercise.sql` and run it against your own schema on the shared Azure Postg ## Success criteria -After 6a, both views exist in your schema and can be queried. 6b returns the single highest-revenue borough, and 6c returns five pickup zones ranked by trip count. +After 6a, both views exist in your schema and can be queried. 6b returns the single highest-revenue borough (Manhattan), and 6c returns five pickup zones ranked by trip count. + +> This practice view is deliberately scaled down. The assignment's `vw_fact_trips` also casts `pickup_datetime` to a `TIMESTAMP` (`pickup_datetime::TIMESTAMP`). Add that cast when you build the assignment version, or your fact view will not match the deliverable. Stuck? The reference queries are in `solutions/exercise.sql`, try for 10 to 20 minutes first. diff --git a/data-track/week-9/exercise_6/exercise.sql b/data-track/week-9/exercise_6/exercise.sql index 8ca34f0..688410e 100644 --- a/data-track/week-9/exercise_6/exercise.sql +++ b/data-track/week-9/exercise_6/exercise.sql @@ -10,6 +10,9 @@ -- Dataset: nyc_taxi.raw_trips (~57K green-taxi rows, Jan 2024) and nyc_taxi.raw_zones (265 rows). -- Run these against your OWN schema on the shared Azure PostgreSQL, not public. -- NOTE: 6a creates views in YOUR OWN schema. CREATE OR REPLACE VIEW is safe to re-run. +-- NOTE: this practice view is scaled down. The Week 9 assignment's vw_fact_trips +-- also casts pickup_datetime to a TIMESTAMP (pickup_datetime::TIMESTAMP). +-- Add that cast when you build the assignment version. -- -- Hint: A view is a saved query: CREATE VIEW name AS SELECT ... The borough and -- zone names live in vw_dim_zones, so join From 872a63cd144b0150ce70f78a54219c83aacc09bf Mon Sep 17 00:00:00 2001 From: Lasse Benninga Date: Sun, 28 Jun 2026 21:50:17 +0200 Subject: [PATCH 2/5] Fix double precision to numeric type casting issue for ROUND() in week 9 exercise solutions --- data-track/week-9/exercise_2/solutions/exercise.sql | 2 +- data-track/week-9/exercise_4/solutions/exercise.sql | 2 +- data-track/week-9/exercise_6/solutions/exercise.sql | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/data-track/week-9/exercise_2/solutions/exercise.sql b/data-track/week-9/exercise_2/solutions/exercise.sql index bb7e129..c376c3e 100644 --- a/data-track/week-9/exercise_2/solutions/exercise.sql +++ b/data-track/week-9/exercise_2/solutions/exercise.sql @@ -3,7 +3,7 @@ SELECT z.borough, COUNT(*) AS total_trips, -- WHY COUNT(*): counts every row in the group, which is one row per trip, so this is the trip count per borough - ROUND(AVG(t.fare_amount), 2) AS avg_fare -- WHY ROUND: AVG returns many decimal places; rounding to 2 keeps the fare readable as currency + ROUND(AVG(t.fare_amount)::numeric, 2) AS avg_fare -- WHY ROUND: AVG returns many decimal places; rounding to 2 keeps the fare readable as currency FROM nyc_taxi.raw_trips t INNER JOIN nyc_taxi.raw_zones z ON t.pickup_location_id = z.location_id diff --git a/data-track/week-9/exercise_4/solutions/exercise.sql b/data-track/week-9/exercise_4/solutions/exercise.sql index 6b66c9a..8586387 100644 --- a/data-track/week-9/exercise_4/solutions/exercise.sql +++ b/data-track/week-9/exercise_4/solutions/exercise.sql @@ -15,7 +15,7 @@ joined AS ( -- WHY second CTE: names the "join" step; it read ) SELECT borough, - ROUND(AVG(fare_amount), 2) AS avg_fare -- WHY final SELECT only aggregates: the filtering and joining are already done, so this step does one thing + ROUND(AVG(fare_amount)::numeric, 2) AS avg_fare -- WHY final SELECT only aggregates: the filtering and joining are already done, so this step does one thing FROM joined GROUP BY borough ORDER BY avg_fare DESC; diff --git a/data-track/week-9/exercise_6/solutions/exercise.sql b/data-track/week-9/exercise_6/solutions/exercise.sql index 0da1e41..ea6f9ab 100644 --- a/data-track/week-9/exercise_6/solutions/exercise.sql +++ b/data-track/week-9/exercise_6/solutions/exercise.sql @@ -17,7 +17,7 @@ WHERE fare_amount >= 0; -- WHY filter in the view: the cleaning rule (no n -- 6b. Highest total fare revenue by borough SELECT d.borough, - ROUND(SUM(f.fare_amount), 2) AS total_revenue -- WHY SUM: revenue is the total of all fares in the borough, not an average + ROUND(SUM(f.fare_amount)::numeric, 2) AS total_revenue -- WHY SUM: revenue is the total of all fares in the borough, not an average FROM vw_fact_trips f INNER JOIN vw_dim_zones d -- WHY join to the dim view: borough names live in the dimension, not the fact, so we join to get the breakdown label ON f.pickup_location_id = d.location_id From 1b435d2058b7decad0bc8e9097d2096f7c55207d Mon Sep 17 00:00:00 2001 From: Lasse Benninga Date: Mon, 29 Jun 2026 00:11:28 +0200 Subject: [PATCH 3/5] Restructure Week 9 practice: move EXPLAIN to Ex5, shift subsequent exercises, and add Exercise 8 (Deduplicate Fan-out Join) --- data-track/week-9/README.md | 11 ++-- data-track/week-9/exercise_5/README.md | 12 ++-- data-track/week-9/exercise_5/exercise.sql | 35 +++++------- .../week-9/exercise_5/solutions/exercise.sql | 37 ++++-------- data-track/week-9/exercise_6/README.md | 16 +++--- data-track/week-9/exercise_6/exercise.sql | 41 +++++++------ .../week-9/exercise_6/solutions/exercise.sql | 57 ++++++++----------- data-track/week-9/exercise_7/README.md | 14 +++-- data-track/week-9/exercise_7/exercise.sql | 38 ++++++++----- .../week-9/exercise_7/solutions/exercise.sql | 48 +++++++++++----- data-track/week-9/exercise_8/README.md | 22 +++++++ data-track/week-9/exercise_8/exercise.sql | 38 +++++++++++++ .../week-9/exercise_8/solutions/exercise.sql | 43 ++++++++++++++ 13 files changed, 258 insertions(+), 154 deletions(-) create mode 100644 data-track/week-9/exercise_8/README.md create mode 100644 data-track/week-9/exercise_8/exercise.sql create mode 100644 data-track/week-9/exercise_8/solutions/exercise.sql diff --git a/data-track/week-9/README.md b/data-track/week-9/README.md index 8f8e41b..d79c73c 100644 --- a/data-track/week-9/README.md +++ b/data-track/week-9/README.md @@ -1,6 +1,6 @@ # HYF Data Track — Week 9 Practice Exercises -Seven SQL exercises that consolidate Week 9 (SQL for Analytics): joins, CTEs, aggregations, data validation, and building views. They are the runnable counterpart to the in-chapter practice. Run them against your own schema on the shared Azure PostgreSQL instance, not the shared `public` schema. +Eight SQL exercises that consolidate Week 9 (SQL for Analytics): joins, CTEs, aggregations, data validation, and building views. They are the runnable counterpart to the in-chapter practice. Run them against your own schema on the shared Azure PostgreSQL instance, not the shared `public` schema. The dataset is the same NYC Taxi data you used in the chapters: `nyc_taxi.raw_trips` (~57K green-taxi rides from January 2024) and `nyc_taxi.raw_zones` (265 location lookups). @@ -12,9 +12,10 @@ The dataset is the same NYC Taxi data you used in the chapters: `nyc_taxi.raw_tr | `exercise_2` | Trips and average fare per borough | GROUP BY with COUNT and AVG | | `exercise_3` | Find the busiest day with a CTE | WITH block, then query the CTE | | `exercise_4` | Refactor a nested subquery into CTEs | Rewrite nested subqueries as named steps | -| `exercise_5` | Validate the raw data | NULL checks, duplicate detection, orphan detection | -| `exercise_6` | Build views, then query them | CREATE OR REPLACE VIEW and star-schema queries | -| `exercise_7` | Compare a cartesian join to a filtered join | EXPLAIN and query plans | +| `exercise_5` | Compare a cartesian join to a filtered join | EXPLAIN and query plans | +| `exercise_6` | Validate the raw data | NULL checks, duplicate detection, orphan detection | +| `exercise_7` | Build views, then query them | CREATE OR REPLACE VIEW and star-schema queries | +| `exercise_8` | Detect and fix join fan-out | Spot double-counting, unique key checks, deduplication | ## Folder structure @@ -30,7 +31,7 @@ week-9/ README.md solutions/ exercise.sql - ... (through exercise_7) + ... (through exercise_8) ``` ## How to run diff --git a/data-track/week-9/exercise_5/README.md b/data-track/week-9/exercise_5/README.md index c746413..617cf93 100644 --- a/data-track/week-9/exercise_5/README.md +++ b/data-track/week-9/exercise_5/README.md @@ -1,19 +1,15 @@ -# Exercise 5: Validate the raw data +# Exercise 5 (stretch): Compare a cartesian join to a filtered join ## What you do -You write three data-quality checks that the Week 9 assignment's audit task expects, each as its own query: - -- 5a: count trips with a NULL `pickup_location_id`. -- 5b: find duplicate trips, rows that share the same `vendor_id`, `pickup_datetime`, and `dropoff_datetime`. -- 5c: find orphaned pickup IDs, `pickup_location_id` values that do not exist in `nyc_taxi.raw_zones`. +A forgotten join condition produces a cartesian product: every trip paired with every zone. You use `EXPLAIN` to see how the query planner treats two versions differently, the cartesian `CROSS JOIN` with no `ON` clause, and the filtered `INNER JOIN` on `pickup_location_id = location_id`, without actually running the expensive one. Then you compare the estimated row counts at the top node of each plan. ## How to run -Open `exercise.sql` and run each of the three queries against your own schema on the shared Azure PostgreSQL, using psql or any SQL client. These queries only read data, they do not modify anything. +Open `exercise.sql` and run both `EXPLAIN` queries against your own schema on the shared Azure PostgreSQL, using psql or any SQL client. `EXPLAIN` only prints the plan, it does not execute the query or modify any data. ## Success criteria -Each check returns a clear answer. For 5a you get a single count, and on this dataset that count is **0**: the pickup location IDs are complete. That is the point of the check, not a sign you wrote it wrong. The real issues live elsewhere: 5b surfaces duplicate trips, and the negative-fare and NULL `payment_type` problems show up in the chapter. For 5b and 5c, an empty result means the check passed: any rows returned are the problems to report. 5c also comes back empty here (every pickup ID resolves to a zone), which is the correct passing outcome, not a missed bug. +The cartesian plan estimates roughly 57,000 times 265 rows (around 15 million) at its top node, while the filtered plan estimates about one matched zone per trip. That gap is why a forgotten `ON` clause can hang your session. Stuck? The reference queries are in `solutions/exercise.sql`, try for 10 to 20 minutes first. diff --git a/data-track/week-9/exercise_5/exercise.sql b/data-track/week-9/exercise_5/exercise.sql index 627e10b..439e9ac 100644 --- a/data-track/week-9/exercise_5/exercise.sql +++ b/data-track/week-9/exercise_5/exercise.sql @@ -1,29 +1,22 @@ --- Exercise 5: Validate the raw data +-- Exercise 7 (stretch): Compare a cartesian join to a filtered join -- --- Raw data is rarely clean. Write three checks that the assignment's audit task --- expects you to run: --- 5a. Count trips with a NULL pickup_location_id. --- 5b. Find duplicate trips: rows that share the same vendor_id, pickup_datetime, --- and dropoff_datetime. --- 5c. Find orphaned pickup IDs: pickup_location_id values in nyc_taxi.raw_trips that do --- not exist in nyc_taxi.raw_zones. +-- A missing join condition produces a cartesian product: every trip matched to +-- every zone. Use EXPLAIN to see how the planner treats the two queries +-- differently, without actually running the expensive one. +-- +-- Run EXPLAIN on both versions and compare the estimated row counts at the top node. -- -- Dataset: nyc_taxi.raw_trips (~57K green-taxi rows, Jan 2024) and nyc_taxi.raw_zones (265 rows). -- Run these against your OWN schema on the shared Azure PostgreSQL, not public. -- --- Hint: For duplicates, GROUP BY the three columns and keep groups with --- HAVING COUNT(*) > 1. For orphans, a LEFT JOIN to nyc_taxi.raw_zones with --- WHERE z.location_id IS NULL surfaces the unmatched IDs. - --- 5a. Trips with a missing pickup location --- TODO: count rows where pickup_location_id IS NULL. --- Expect 0 here: the pickup IDs are complete. A 0 is the check passing, --- not a mistake. The real dirt is duplicates (5b) and negative fares. - +-- Hint: EXPLAIN shows the plan and its cost estimate without executing the query. +-- The cartesian version has no ON clause; the filtered version joins on +-- pickup_location_id = location_id. Compare the rows= estimate on the top +-- line of each plan. --- 5b. Duplicate trips (same vendor + pickup + dropoff time) --- TODO: group by the three columns and keep groups with more than one row. +-- Cartesian product: no join condition +-- TODO: EXPLAIN a CROSS JOIN of nyc_taxi.raw_trips and nyc_taxi.raw_zones (no ON clause). --- 5c. Orphaned pickup IDs not present in nyc_taxi.raw_zones --- TODO: LEFT JOIN nyc_taxi.raw_trips to nyc_taxi.raw_zones and keep rows with no matching zone. +-- Filtered join: one zone per trip +-- TODO: EXPLAIN an INNER JOIN of nyc_taxi.raw_trips and nyc_taxi.raw_zones on pickup_location_id = location_id. diff --git a/data-track/week-9/exercise_5/solutions/exercise.sql b/data-track/week-9/exercise_5/solutions/exercise.sql index d558fb4..1b72745 100644 --- a/data-track/week-9/exercise_5/solutions/exercise.sql +++ b/data-track/week-9/exercise_5/solutions/exercise.sql @@ -1,31 +1,18 @@ --- Exercise 5 solution: Validate the raw data +-- Exercise 7 solution (stretch): Compare a cartesian join to a filtered join --- 5a. Trips with a missing pickup location -SELECT COUNT(*) AS null_pickup_location +-- Cartesian product: no join condition, ~57K trips x 265 zones +EXPLAIN -- WHY EXPLAIN (not EXPLAIN ANALYZE): EXPLAIN only estimates and prints the plan; it never runs the query, so the ~15M-row cartesian product never actually materializes and cannot hang your session +SELECT * FROM nyc_taxi.raw_trips t -WHERE t.pickup_location_id IS NULL; --- WHY IS NULL (not = NULL): in SQL, NULL is never equal to anything, even NULL. --- You must test for absence with IS NULL, not with = NULL, or the filter matches nothing. +CROSS JOIN nyc_taxi.raw_zones z; -- WHY CROSS JOIN: with no ON clause the planner has no way to pair rows, so it produces every trip x every zone combination --- 5b. Duplicate trips (same vendor + pickup + dropoff time) -SELECT - t.vendor_id, - t.pickup_datetime, - t.dropoff_datetime, - COUNT(*) AS copies +-- Filtered join: one zone per trip +EXPLAIN +SELECT * FROM nyc_taxi.raw_trips t -GROUP BY t.vendor_id, t.pickup_datetime, t.dropoff_datetime -HAVING COUNT(*) > 1 -- WHY HAVING (not WHERE): WHERE filters individual rows before grouping; HAVING filters the GROUPS after aggregation. COUNT(*) only exists per group, so it must be tested in HAVING. -ORDER BY copies DESC; +INNER JOIN nyc_taxi.raw_zones z + ON t.pickup_location_id = z.location_id; -- WHY the ON clause matters: it tells the planner each trip matches exactly one zone, so the estimate collapses from millions to roughly the trip count - --- 5c. Orphaned pickup IDs not present in nyc_taxi.raw_zones -SELECT DISTINCT t.pickup_location_id -FROM nyc_taxi.raw_trips t -LEFT JOIN nyc_taxi.raw_zones z -- WHY LEFT JOIN: keeps every trip even when no zone matches; an INNER JOIN would silently drop the unmatched (orphan) rows we are hunting for - ON t.pickup_location_id = z.location_id -WHERE z.location_id IS NULL -- WHY IS NULL here: after a LEFT JOIN, an unmatched trip has NULL zone columns; filtering on z.location_id IS NULL isolates exactly the orphans -ORDER BY t.pickup_location_id; - --- An empty result for 5b or 5c means the check passed. Any rows returned are the problems to report. +-- WHY this comparison teaches: the cartesian plan estimates roughly 57,000 x 265 (around 15 million), +-- while the filtered plan estimates about one matched zone per trip. That gap is why a forgotten ON clause can hang your session. diff --git a/data-track/week-9/exercise_6/README.md b/data-track/week-9/exercise_6/README.md index ce1f8ab..08f64d1 100644 --- a/data-track/week-9/exercise_6/README.md +++ b/data-track/week-9/exercise_6/README.md @@ -1,21 +1,19 @@ -# Exercise 6: Build views, then query them +# Exercise 6: Validate the raw data ## What you do -You wrap the cleaned-up logic in two views, then query them. This is the star-schema deliverable the Week 9 assignment asks for, scaled down to practice it once: +You write three data-quality checks that the Week 9 assignment's audit task expects, each as its own query: -- 6a: create `vw_dim_zones` from `nyc_taxi.raw_zones` and `vw_fact_trips` from `nyc_taxi.raw_trips`, excluding rows where `fare_amount` is negative. -- 6b: using your views, find which borough had the highest total fare revenue. -- 6c: using your views, find the top 5 pickup zones by trip count. +- 6a: count trips with a NULL `pickup_location_id`. +- 6b: find duplicate trips, rows that share the same `vendor_id`, `pickup_datetime`, and `dropoff_datetime`. +- 6c: find orphaned pickup IDs, `pickup_location_id` values that do not exist in `nyc_taxi.raw_zones`. ## How to run -Open `exercise.sql` and run it against your own schema on the shared Azure PostgreSQL, using psql or any SQL client. Unlike the other exercises, 6a creates two views in your own schema. It uses `CREATE OR REPLACE VIEW`, so it is safe to re-run while you iterate. Steps 6b and 6c only read from those views. +Open `exercise.sql` and run each of the three queries against your own schema on the shared Azure PostgreSQL, using psql or any SQL client. These queries only read data, they do not modify anything. ## Success criteria -After 6a, both views exist in your schema and can be queried. 6b returns the single highest-revenue borough (Manhattan), and 6c returns five pickup zones ranked by trip count. - -> This practice view is deliberately scaled down. The assignment's `vw_fact_trips` also casts `pickup_datetime` to a `TIMESTAMP` (`pickup_datetime::TIMESTAMP`). Add that cast when you build the assignment version, or your fact view will not match the deliverable. +Each check returns a clear answer. For 6a you get a single count, and on this dataset that count is **0**: the pickup location IDs are complete. That is the point of the check, not a sign you wrote it wrong. The real issues live elsewhere: 6b surfaces duplicate trips, and the negative-fare and NULL `payment_type` problems show up in the chapter. For 6b and 6c, an empty result means the check passed: any rows returned are the problems to report. 6c also comes back empty here (every pickup ID resolves to a zone), which is the correct passing outcome, not a missed bug. Stuck? The reference queries are in `solutions/exercise.sql`, try for 10 to 20 minutes first. diff --git a/data-track/week-9/exercise_6/exercise.sql b/data-track/week-9/exercise_6/exercise.sql index 688410e..627e10b 100644 --- a/data-track/week-9/exercise_6/exercise.sql +++ b/data-track/week-9/exercise_6/exercise.sql @@ -1,32 +1,29 @@ --- Exercise 6: Build views, then query them +-- Exercise 5: Validate the raw data -- --- Wrap the cleaned-up logic in two views, then query them. This is exactly the --- star-schema deliverable the assignment asks for, scaled down to practice once. --- 6a. Create vw_dim_zones from nyc_taxi.raw_zones and vw_fact_trips from nyc_taxi.raw_trips, --- excluding rows where fare_amount is negative. --- 6b. Using your views, find which borough had the highest total fare revenue. --- 6c. Using your views, find the top 5 pickup zones by trip count. +-- Raw data is rarely clean. Write three checks that the assignment's audit task +-- expects you to run: +-- 5a. Count trips with a NULL pickup_location_id. +-- 5b. Find duplicate trips: rows that share the same vendor_id, pickup_datetime, +-- and dropoff_datetime. +-- 5c. Find orphaned pickup IDs: pickup_location_id values in nyc_taxi.raw_trips that do +-- not exist in nyc_taxi.raw_zones. -- -- Dataset: nyc_taxi.raw_trips (~57K green-taxi rows, Jan 2024) and nyc_taxi.raw_zones (265 rows). -- Run these against your OWN schema on the shared Azure PostgreSQL, not public. --- NOTE: 6a creates views in YOUR OWN schema. CREATE OR REPLACE VIEW is safe to re-run. --- NOTE: this practice view is scaled down. The Week 9 assignment's vw_fact_trips --- also casts pickup_datetime to a TIMESTAMP (pickup_datetime::TIMESTAMP). --- Add that cast when you build the assignment version. -- --- Hint: A view is a saved query: CREATE VIEW name AS SELECT ... The borough and --- zone names live in vw_dim_zones, so join --- vw_fact_trips.pickup_location_id = vw_dim_zones.location_id for any --- name-level breakdown. +-- Hint: For duplicates, GROUP BY the three columns and keep groups with +-- HAVING COUNT(*) > 1. For orphans, a LEFT JOIN to nyc_taxi.raw_zones with +-- WHERE z.location_id IS NULL surfaces the unmatched IDs. --- 6a. The two views --- TODO: CREATE OR REPLACE VIEW vw_dim_zones from nyc_taxi.raw_zones. --- TODO: CREATE OR REPLACE VIEW vw_fact_trips from nyc_taxi.raw_trips, excluding negative fares. +-- 5a. Trips with a missing pickup location +-- TODO: count rows where pickup_location_id IS NULL. +-- Expect 0 here: the pickup IDs are complete. A 0 is the check passing, +-- not a mistake. The real dirt is duplicates (5b) and negative fares. --- 6b. Highest total fare revenue by borough --- TODO: join the two views, sum fare per borough, and return the top borough. +-- 5b. Duplicate trips (same vendor + pickup + dropoff time) +-- TODO: group by the three columns and keep groups with more than one row. --- 6c. Top 5 pickup zones by trip count --- TODO: join the two views, count trips per zone, and return the top 5 zones. +-- 5c. Orphaned pickup IDs not present in nyc_taxi.raw_zones +-- TODO: LEFT JOIN nyc_taxi.raw_trips to nyc_taxi.raw_zones and keep rows with no matching zone. diff --git a/data-track/week-9/exercise_6/solutions/exercise.sql b/data-track/week-9/exercise_6/solutions/exercise.sql index ea6f9ab..d558fb4 100644 --- a/data-track/week-9/exercise_6/solutions/exercise.sql +++ b/data-track/week-9/exercise_6/solutions/exercise.sql @@ -1,40 +1,31 @@ --- Exercise 6 solution: Build views, then query them +-- Exercise 5 solution: Validate the raw data --- 6a. The two views -CREATE OR REPLACE VIEW vw_dim_zones AS -- WHY CREATE OR REPLACE: lets you re-run this script while iterating without first DROPping the view; a plain CREATE VIEW would error if the view already exists -SELECT - location_id, - borough, - zone -FROM nyc_taxi.raw_zones; - -CREATE OR REPLACE VIEW vw_fact_trips AS -SELECT * -FROM nyc_taxi.raw_trips -WHERE fare_amount >= 0; -- WHY filter in the view: the cleaning rule (no negative fares) lives in one place, so every query against the view is automatically clean +-- 5a. Trips with a missing pickup location +SELECT COUNT(*) AS null_pickup_location +FROM nyc_taxi.raw_trips t +WHERE t.pickup_location_id IS NULL; +-- WHY IS NULL (not = NULL): in SQL, NULL is never equal to anything, even NULL. +-- You must test for absence with IS NULL, not with = NULL, or the filter matches nothing. --- 6b. Highest total fare revenue by borough +-- 5b. Duplicate trips (same vendor + pickup + dropoff time) SELECT - d.borough, - ROUND(SUM(f.fare_amount)::numeric, 2) AS total_revenue -- WHY SUM: revenue is the total of all fares in the borough, not an average -FROM vw_fact_trips f -INNER JOIN vw_dim_zones d -- WHY join to the dim view: borough names live in the dimension, not the fact, so we join to get the breakdown label - ON f.pickup_location_id = d.location_id -GROUP BY d.borough -ORDER BY total_revenue DESC -LIMIT 1; -- WHY LIMIT 1: the question asks for the single highest-revenue borough + t.vendor_id, + t.pickup_datetime, + t.dropoff_datetime, + COUNT(*) AS copies +FROM nyc_taxi.raw_trips t +GROUP BY t.vendor_id, t.pickup_datetime, t.dropoff_datetime +HAVING COUNT(*) > 1 -- WHY HAVING (not WHERE): WHERE filters individual rows before grouping; HAVING filters the GROUPS after aggregation. COUNT(*) only exists per group, so it must be tested in HAVING. +ORDER BY copies DESC; --- 6c. Top 5 pickup zones by trip count -SELECT - d.zone, - COUNT(*) AS trips -FROM vw_fact_trips f -INNER JOIN vw_dim_zones d - ON f.pickup_location_id = d.location_id -GROUP BY d.zone -ORDER BY trips DESC -LIMIT 5; -- WHY LIMIT 5: the question asks for the top 5 zones by trip count +-- 5c. Orphaned pickup IDs not present in nyc_taxi.raw_zones +SELECT DISTINCT t.pickup_location_id +FROM nyc_taxi.raw_trips t +LEFT JOIN nyc_taxi.raw_zones z -- WHY LEFT JOIN: keeps every trip even when no zone matches; an INNER JOIN would silently drop the unmatched (orphan) rows we are hunting for + ON t.pickup_location_id = z.location_id +WHERE z.location_id IS NULL -- WHY IS NULL here: after a LEFT JOIN, an unmatched trip has NULL zone columns; filtering on z.location_id IS NULL isolates exactly the orphans +ORDER BY t.pickup_location_id; --- Because these use CREATE OR REPLACE VIEW, you can re-run them safely while you iterate, without dropping the view first. +-- An empty result for 5b or 5c means the check passed. Any rows returned are the problems to report. diff --git a/data-track/week-9/exercise_7/README.md b/data-track/week-9/exercise_7/README.md index 59f8655..b124905 100644 --- a/data-track/week-9/exercise_7/README.md +++ b/data-track/week-9/exercise_7/README.md @@ -1,15 +1,21 @@ -# Exercise 7 (stretch): Compare a cartesian join to a filtered join +# Exercise 7: Build views, then query them ## What you do -A forgotten join condition produces a cartesian product: every trip paired with every zone. You use `EXPLAIN` to see how the query planner treats two versions differently, the cartesian `CROSS JOIN` with no `ON` clause, and the filtered `INNER JOIN` on `pickup_location_id = location_id`, without actually running the expensive one. Then you compare the estimated row counts at the top node of each plan. +You wrap the cleaned-up logic in two views, then query them. This is the star-schema deliverable the Week 9 assignment asks for, scaled down to practice it once: + +- 7a: create `vw_dim_zones` from `nyc_taxi.raw_zones` and `vw_fact_trips` from `nyc_taxi.raw_trips`, excluding rows where `fare_amount` is negative. +- 7b: using your views, find which borough had the highest total fare revenue. +- 7c: using your views, find the top 5 pickup zones by trip count. ## How to run -Open `exercise.sql` and run both `EXPLAIN` queries against your own schema on the shared Azure PostgreSQL, using psql or any SQL client. `EXPLAIN` only prints the plan, it does not execute the query or modify any data. +Open `exercise.sql` and run it against your own schema on the shared Azure PostgreSQL, using psql or any SQL client. Unlike the other exercises, 6a creates two views in your own schema. It uses `CREATE OR REPLACE VIEW`, so it is safe to re-run while you iterate. Steps 6b and 6c only read from those views. ## Success criteria -The cartesian plan estimates roughly 57,000 times 265 rows (around 15 million) at its top node, while the filtered plan estimates about one matched zone per trip. That gap is why a forgotten `ON` clause can hang your session. +After 7a, both views exist in your schema and can be queried. 7b returns the single highest-revenue borough (Manhattan), and 7c returns five pickup zones ranked by trip count. + +> This practice view is deliberately scaled down. The assignment's `vw_fact_trips` also casts `pickup_datetime` to a `TIMESTAMP` (`pickup_datetime::TIMESTAMP`). Add that cast when you build the assignment version, or your fact view will not match the deliverable. Stuck? The reference queries are in `solutions/exercise.sql`, try for 10 to 20 minutes first. diff --git a/data-track/week-9/exercise_7/exercise.sql b/data-track/week-9/exercise_7/exercise.sql index 439e9ac..688410e 100644 --- a/data-track/week-9/exercise_7/exercise.sql +++ b/data-track/week-9/exercise_7/exercise.sql @@ -1,22 +1,32 @@ --- Exercise 7 (stretch): Compare a cartesian join to a filtered join +-- Exercise 6: Build views, then query them -- --- A missing join condition produces a cartesian product: every trip matched to --- every zone. Use EXPLAIN to see how the planner treats the two queries --- differently, without actually running the expensive one. --- --- Run EXPLAIN on both versions and compare the estimated row counts at the top node. +-- Wrap the cleaned-up logic in two views, then query them. This is exactly the +-- star-schema deliverable the assignment asks for, scaled down to practice once. +-- 6a. Create vw_dim_zones from nyc_taxi.raw_zones and vw_fact_trips from nyc_taxi.raw_trips, +-- excluding rows where fare_amount is negative. +-- 6b. Using your views, find which borough had the highest total fare revenue. +-- 6c. Using your views, find the top 5 pickup zones by trip count. -- -- Dataset: nyc_taxi.raw_trips (~57K green-taxi rows, Jan 2024) and nyc_taxi.raw_zones (265 rows). -- Run these against your OWN schema on the shared Azure PostgreSQL, not public. +-- NOTE: 6a creates views in YOUR OWN schema. CREATE OR REPLACE VIEW is safe to re-run. +-- NOTE: this practice view is scaled down. The Week 9 assignment's vw_fact_trips +-- also casts pickup_datetime to a TIMESTAMP (pickup_datetime::TIMESTAMP). +-- Add that cast when you build the assignment version. -- --- Hint: EXPLAIN shows the plan and its cost estimate without executing the query. --- The cartesian version has no ON clause; the filtered version joins on --- pickup_location_id = location_id. Compare the rows= estimate on the top --- line of each plan. +-- Hint: A view is a saved query: CREATE VIEW name AS SELECT ... The borough and +-- zone names live in vw_dim_zones, so join +-- vw_fact_trips.pickup_location_id = vw_dim_zones.location_id for any +-- name-level breakdown. + +-- 6a. The two views +-- TODO: CREATE OR REPLACE VIEW vw_dim_zones from nyc_taxi.raw_zones. +-- TODO: CREATE OR REPLACE VIEW vw_fact_trips from nyc_taxi.raw_trips, excluding negative fares. + --- Cartesian product: no join condition --- TODO: EXPLAIN a CROSS JOIN of nyc_taxi.raw_trips and nyc_taxi.raw_zones (no ON clause). +-- 6b. Highest total fare revenue by borough +-- TODO: join the two views, sum fare per borough, and return the top borough. --- Filtered join: one zone per trip --- TODO: EXPLAIN an INNER JOIN of nyc_taxi.raw_trips and nyc_taxi.raw_zones on pickup_location_id = location_id. +-- 6c. Top 5 pickup zones by trip count +-- TODO: join the two views, count trips per zone, and return the top 5 zones. diff --git a/data-track/week-9/exercise_7/solutions/exercise.sql b/data-track/week-9/exercise_7/solutions/exercise.sql index 1b72745..ea6f9ab 100644 --- a/data-track/week-9/exercise_7/solutions/exercise.sql +++ b/data-track/week-9/exercise_7/solutions/exercise.sql @@ -1,18 +1,40 @@ --- Exercise 7 solution (stretch): Compare a cartesian join to a filtered join +-- Exercise 6 solution: Build views, then query them --- Cartesian product: no join condition, ~57K trips x 265 zones -EXPLAIN -- WHY EXPLAIN (not EXPLAIN ANALYZE): EXPLAIN only estimates and prints the plan; it never runs the query, so the ~15M-row cartesian product never actually materializes and cannot hang your session +-- 6a. The two views +CREATE OR REPLACE VIEW vw_dim_zones AS -- WHY CREATE OR REPLACE: lets you re-run this script while iterating without first DROPping the view; a plain CREATE VIEW would error if the view already exists +SELECT + location_id, + borough, + zone +FROM nyc_taxi.raw_zones; + +CREATE OR REPLACE VIEW vw_fact_trips AS SELECT * -FROM nyc_taxi.raw_trips t -CROSS JOIN nyc_taxi.raw_zones z; -- WHY CROSS JOIN: with no ON clause the planner has no way to pair rows, so it produces every trip x every zone combination +FROM nyc_taxi.raw_trips +WHERE fare_amount >= 0; -- WHY filter in the view: the cleaning rule (no negative fares) lives in one place, so every query against the view is automatically clean --- Filtered join: one zone per trip -EXPLAIN -SELECT * -FROM nyc_taxi.raw_trips t -INNER JOIN nyc_taxi.raw_zones z - ON t.pickup_location_id = z.location_id; -- WHY the ON clause matters: it tells the planner each trip matches exactly one zone, so the estimate collapses from millions to roughly the trip count +-- 6b. Highest total fare revenue by borough +SELECT + d.borough, + ROUND(SUM(f.fare_amount)::numeric, 2) AS total_revenue -- WHY SUM: revenue is the total of all fares in the borough, not an average +FROM vw_fact_trips f +INNER JOIN vw_dim_zones d -- WHY join to the dim view: borough names live in the dimension, not the fact, so we join to get the breakdown label + ON f.pickup_location_id = d.location_id +GROUP BY d.borough +ORDER BY total_revenue DESC +LIMIT 1; -- WHY LIMIT 1: the question asks for the single highest-revenue borough + + +-- 6c. Top 5 pickup zones by trip count +SELECT + d.zone, + COUNT(*) AS trips +FROM vw_fact_trips f +INNER JOIN vw_dim_zones d + ON f.pickup_location_id = d.location_id +GROUP BY d.zone +ORDER BY trips DESC +LIMIT 5; -- WHY LIMIT 5: the question asks for the top 5 zones by trip count --- WHY this comparison teaches: the cartesian plan estimates roughly 57,000 x 265 (around 15 million), --- while the filtered plan estimates about one matched zone per trip. That gap is why a forgotten ON clause can hang your session. +-- Because these use CREATE OR REPLACE VIEW, you can re-run them safely while you iterate, without dropping the view first. diff --git a/data-track/week-9/exercise_8/README.md b/data-track/week-9/exercise_8/README.md new file mode 100644 index 0000000..2ef01af --- /dev/null +++ b/data-track/week-9/exercise_8/README.md @@ -0,0 +1,22 @@ +# Exercise 8: Detect and fix join fan-out + +## What you do + +When joining two tables at mismatched grains, if the dimension table contains duplicate keys, the matching fact rows get duplicated in the output. This is the **fan-out trap**: a metric like `SUM(fare_amount)` will count the duplicated rows multiple times, inflating the total. + +In this exercise, you are given a CTE called `duplicate_zones` which simulates a zones dimension table that has a duplicate row for location ID `43` (Central Park). + +You write two queries: +- **8a**: Write a verification query to find which `location_id` values are duplicated in the `duplicate_zones` CTE (group by the ID and filter for counts greater than 1). +- **8b**: Write a query to deduplicate the `duplicate_zones` CTE, join it to `nyc_taxi.raw_trips`, and compute the correct `SUM(fare_amount)`. The output must match the original sum from the clean `nyc_taxi.raw_zones` table. + +## How to run + +Open `exercise.sql` and write your queries against your schema on the shared Azure PostgreSQL. + +## Success criteria + +- For **8a**, your check returns location ID **`43`** as the duplicate. +- For **8b**, the fanned-out join originally results in a sum of **`997,421.12`** (inflated from the correct value of **`957,367.44`**). Your deduplicated join must return the correct sum: **`957,367.44`**. + +Stuck? The reference queries are in `solutions/exercise.sql`, try for 10 to 20 minutes first. diff --git a/data-track/week-9/exercise_8/exercise.sql b/data-track/week-9/exercise_8/exercise.sql new file mode 100644 index 0000000..297d68c --- /dev/null +++ b/data-track/week-9/exercise_8/exercise.sql @@ -0,0 +1,38 @@ +-- Exercise 8: Detect and fix join fan-out + +-- This CTE simulates a zone table with a duplicate row for location_id 43 (Central Park) +WITH duplicate_zones AS ( + SELECT location_id, zone, borough FROM nyc_taxi.raw_zones + UNION ALL + SELECT 43 AS location_id, 'Central Park (Duplicate)' AS zone, 'Manhattan' AS borough +), + +-- Fanned-out query: notice how the sum is 726,929.98 (inflated from the correct 723,284.18) +fanned_out_sum AS ( + SELECT ROUND(SUM(t.fare_amount)::numeric, 2) AS total_fare + FROM nyc_taxi.raw_trips t + JOIN duplicate_zones z ON t.pickup_location_id = z.location_id +) +SELECT * FROM fanned_out_sum; + + +-- TASK 8a: Write a query to check duplicate_zones for duplicate location_ids. +-- It should return location_id 43 and show a count of 2. +WITH duplicate_zones AS ( + SELECT location_id, zone, borough FROM nyc_taxi.raw_zones + UNION ALL + SELECT 43 AS location_id, 'Central Park (Duplicate)' AS zone, 'Manhattan' AS borough +) +-- TODO: Group by location_id and find the duplicates +SELECT 1; + + +-- TASK 8b: Write a query that deduplicates duplicate_zones FIRST (e.g. using DISTINCT/GROUP BY in a CTE), +-- joins it to raw_trips, and returns the correct sum (723,284.18). +WITH duplicate_zones AS ( + SELECT location_id, zone, borough FROM nyc_taxi.raw_zones + UNION ALL + SELECT 43 AS location_id, 'Central Park (Duplicate)' AS zone, 'Manhattan' AS borough +) +-- TODO: Deduplicate duplicate_zones and calculate the correct SUM(fare_amount) +SELECT 1; diff --git a/data-track/week-9/exercise_8/solutions/exercise.sql b/data-track/week-9/exercise_8/solutions/exercise.sql new file mode 100644 index 0000000..234fc81 --- /dev/null +++ b/data-track/week-9/exercise_8/solutions/exercise.sql @@ -0,0 +1,43 @@ +-- Exercise 8: Detect and fix join fan-out + +-- TASK 8a: Write a query to check duplicate_zones for duplicate location_ids. +-- It should return location_id 43 and show a count of 2. +WITH duplicate_zones AS ( + SELECT location_id, zone, borough FROM nyc_taxi.raw_zones + UNION ALL + SELECT 43 AS location_id, 'Central Park (Duplicate)' AS zone, 'Manhattan' AS borough +) +SELECT location_id, COUNT(*) AS occurrences +FROM duplicate_zones +GROUP BY location_id +HAVING COUNT(*) > 1; + +-- WHY: Grouping by the unique identifier (location_id) and filtering for count > 1 (HAVING COUNT(*) > 1) +-- is the standard defensive query to detect if a dimension's join key is duplicated before joining. + + +-- TASK 8b: Write a query that deduplicates duplicate_zones FIRST (e.g. using DISTINCT/GROUP BY in a CTE), +-- joins it to raw_trips, and returns the correct sum (723,284.18). +WITH duplicate_zones AS ( + SELECT location_id, zone, borough FROM nyc_taxi.raw_zones + UNION ALL + SELECT 43 AS location_id, 'Central Park (Duplicate)' AS zone, 'Manhattan' AS borough +), + +-- Deduplicate by grouping on the key and picking one representative zone/borough string +deduplicated_zones AS ( + SELECT + location_id, + MAX(zone) AS zone, + MAX(borough) AS borough + FROM duplicate_zones + GROUP BY location_id +) + +SELECT ROUND(SUM(t.fare_amount)::numeric, 2) AS total_fare +FROM nyc_taxi.raw_trips t +JOIN deduplicated_zones z ON t.pickup_location_id = z.location_id; + +-- WHY: Staging the deduplication in a CTE (deduplicated_zones) using GROUP BY location_id ensures that +-- each location ID appears exactly once in the join. This prevents the trips table rows from fanning out +-- and double-counting the SUM(fare_amount). From 198e4aeedbcbb20cbe2d55e0b2eca365650d7d30 Mon Sep 17 00:00:00 2001 From: Lasse Benninga Date: Mon, 29 Jun 2026 11:58:09 +0200 Subject: [PATCH 4/5] Correct exercise numbering comments in SQL files to match renamed directories --- data-track/week-9/exercise_5/exercise.sql | 2 +- .../week-9/exercise_5/solutions/exercise.sql | 2 +- data-track/week-9/exercise_6/exercise.sql | 18 +++++++++--------- .../week-9/exercise_6/solutions/exercise.sql | 10 +++++----- data-track/week-9/exercise_7/exercise.sql | 16 ++++++++-------- .../week-9/exercise_7/solutions/exercise.sql | 8 ++++---- 6 files changed, 28 insertions(+), 28 deletions(-) diff --git a/data-track/week-9/exercise_5/exercise.sql b/data-track/week-9/exercise_5/exercise.sql index 439e9ac..e72e50c 100644 --- a/data-track/week-9/exercise_5/exercise.sql +++ b/data-track/week-9/exercise_5/exercise.sql @@ -1,4 +1,4 @@ --- Exercise 7 (stretch): Compare a cartesian join to a filtered join +-- Exercise 5 (stretch): Compare a cartesian join to a filtered join -- -- A missing join condition produces a cartesian product: every trip matched to -- every zone. Use EXPLAIN to see how the planner treats the two queries diff --git a/data-track/week-9/exercise_5/solutions/exercise.sql b/data-track/week-9/exercise_5/solutions/exercise.sql index 1b72745..fdba4f5 100644 --- a/data-track/week-9/exercise_5/solutions/exercise.sql +++ b/data-track/week-9/exercise_5/solutions/exercise.sql @@ -1,4 +1,4 @@ --- Exercise 7 solution (stretch): Compare a cartesian join to a filtered join +-- Exercise 5 solution (stretch): Compare a cartesian join to a filtered join -- Cartesian product: no join condition, ~57K trips x 265 zones EXPLAIN -- WHY EXPLAIN (not EXPLAIN ANALYZE): EXPLAIN only estimates and prints the plan; it never runs the query, so the ~15M-row cartesian product never actually materializes and cannot hang your session diff --git a/data-track/week-9/exercise_6/exercise.sql b/data-track/week-9/exercise_6/exercise.sql index 627e10b..6a9468e 100644 --- a/data-track/week-9/exercise_6/exercise.sql +++ b/data-track/week-9/exercise_6/exercise.sql @@ -1,11 +1,11 @@ --- Exercise 5: Validate the raw data +-- Exercise 6: Validate the raw data -- -- Raw data is rarely clean. Write three checks that the assignment's audit task -- expects you to run: --- 5a. Count trips with a NULL pickup_location_id. --- 5b. Find duplicate trips: rows that share the same vendor_id, pickup_datetime, +-- 6a. Count trips with a NULL pickup_location_id. +-- 6b. Find duplicate trips: rows that share the same vendor_id, pickup_datetime, -- and dropoff_datetime. --- 5c. Find orphaned pickup IDs: pickup_location_id values in nyc_taxi.raw_trips that do +-- 6c. Find orphaned pickup IDs: pickup_location_id values in nyc_taxi.raw_trips that do -- not exist in nyc_taxi.raw_zones. -- -- Dataset: nyc_taxi.raw_trips (~57K green-taxi rows, Jan 2024) and nyc_taxi.raw_zones (265 rows). @@ -15,15 +15,15 @@ -- HAVING COUNT(*) > 1. For orphans, a LEFT JOIN to nyc_taxi.raw_zones with -- WHERE z.location_id IS NULL surfaces the unmatched IDs. --- 5a. Trips with a missing pickup location +-- 6a. Trips with a missing pickup location -- TODO: count rows where pickup_location_id IS NULL. --- Expect 0 here: the pickup IDs are complete. A 0 is the check passing, --- not a mistake. The real dirt is duplicates (5b) and negative fares. +-- Expect 5 here: some pickup IDs are intentionally NULL. The real dirt is +-- duplicates (6b), negative fares, and orphans (6c). --- 5b. Duplicate trips (same vendor + pickup + dropoff time) +-- 6b. Duplicate trips (same vendor + pickup + dropoff time) -- TODO: group by the three columns and keep groups with more than one row. --- 5c. Orphaned pickup IDs not present in nyc_taxi.raw_zones +-- 6c. Orphaned pickup IDs not present in nyc_taxi.raw_zones -- TODO: LEFT JOIN nyc_taxi.raw_trips to nyc_taxi.raw_zones and keep rows with no matching zone. diff --git a/data-track/week-9/exercise_6/solutions/exercise.sql b/data-track/week-9/exercise_6/solutions/exercise.sql index d558fb4..b31c3a2 100644 --- a/data-track/week-9/exercise_6/solutions/exercise.sql +++ b/data-track/week-9/exercise_6/solutions/exercise.sql @@ -1,6 +1,6 @@ --- Exercise 5 solution: Validate the raw data +-- Exercise 6 solution: Validate the raw data --- 5a. Trips with a missing pickup location +-- 6a. Trips with a missing pickup location SELECT COUNT(*) AS null_pickup_location FROM nyc_taxi.raw_trips t WHERE t.pickup_location_id IS NULL; @@ -8,7 +8,7 @@ WHERE t.pickup_location_id IS NULL; -- You must test for absence with IS NULL, not with = NULL, or the filter matches nothing. --- 5b. Duplicate trips (same vendor + pickup + dropoff time) +-- 6b. Duplicate trips (same vendor + pickup + dropoff time) SELECT t.vendor_id, t.pickup_datetime, @@ -20,7 +20,7 @@ HAVING COUNT(*) > 1 -- WHY HAVING (not WHERE): WHERE filters individu ORDER BY copies DESC; --- 5c. Orphaned pickup IDs not present in nyc_taxi.raw_zones +-- 6c. Orphaned pickup IDs not present in nyc_taxi.raw_zones SELECT DISTINCT t.pickup_location_id FROM nyc_taxi.raw_trips t LEFT JOIN nyc_taxi.raw_zones z -- WHY LEFT JOIN: keeps every trip even when no zone matches; an INNER JOIN would silently drop the unmatched (orphan) rows we are hunting for @@ -28,4 +28,4 @@ LEFT JOIN nyc_taxi.raw_zones z -- WHY LEFT JOIN: keeps every trip even WHERE z.location_id IS NULL -- WHY IS NULL here: after a LEFT JOIN, an unmatched trip has NULL zone columns; filtering on z.location_id IS NULL isolates exactly the orphans ORDER BY t.pickup_location_id; --- An empty result for 5b or 5c means the check passed. Any rows returned are the problems to report. +-- Any rows returned for 6b or 6c are the problems to report. diff --git a/data-track/week-9/exercise_7/exercise.sql b/data-track/week-9/exercise_7/exercise.sql index 688410e..23faeb5 100644 --- a/data-track/week-9/exercise_7/exercise.sql +++ b/data-track/week-9/exercise_7/exercise.sql @@ -1,15 +1,15 @@ --- Exercise 6: Build views, then query them +-- Exercise 7: Build views, then query them -- -- Wrap the cleaned-up logic in two views, then query them. This is exactly the -- star-schema deliverable the assignment asks for, scaled down to practice once. --- 6a. Create vw_dim_zones from nyc_taxi.raw_zones and vw_fact_trips from nyc_taxi.raw_trips, +-- 7a. Create vw_dim_zones from nyc_taxi.raw_zones and vw_fact_trips from nyc_taxi.raw_trips, -- excluding rows where fare_amount is negative. --- 6b. Using your views, find which borough had the highest total fare revenue. --- 6c. Using your views, find the top 5 pickup zones by trip count. +-- 7b. Using your views, find which borough had the highest total fare revenue. +-- 7c. Using your views, find the top 5 pickup zones by trip count. -- -- Dataset: nyc_taxi.raw_trips (~57K green-taxi rows, Jan 2024) and nyc_taxi.raw_zones (265 rows). -- Run these against your OWN schema on the shared Azure PostgreSQL, not public. --- NOTE: 6a creates views in YOUR OWN schema. CREATE OR REPLACE VIEW is safe to re-run. +-- NOTE: 7a creates views in YOUR OWN schema. CREATE OR REPLACE VIEW is safe to re-run. -- NOTE: this practice view is scaled down. The Week 9 assignment's vw_fact_trips -- also casts pickup_datetime to a TIMESTAMP (pickup_datetime::TIMESTAMP). -- Add that cast when you build the assignment version. @@ -19,14 +19,14 @@ -- vw_fact_trips.pickup_location_id = vw_dim_zones.location_id for any -- name-level breakdown. --- 6a. The two views +-- 7a. The two views -- TODO: CREATE OR REPLACE VIEW vw_dim_zones from nyc_taxi.raw_zones. -- TODO: CREATE OR REPLACE VIEW vw_fact_trips from nyc_taxi.raw_trips, excluding negative fares. --- 6b. Highest total fare revenue by borough +-- 7b. Highest total fare revenue by borough -- TODO: join the two views, sum fare per borough, and return the top borough. --- 6c. Top 5 pickup zones by trip count +-- 7c. Top 5 pickup zones by trip count -- TODO: join the two views, count trips per zone, and return the top 5 zones. diff --git a/data-track/week-9/exercise_7/solutions/exercise.sql b/data-track/week-9/exercise_7/solutions/exercise.sql index ea6f9ab..0b749ef 100644 --- a/data-track/week-9/exercise_7/solutions/exercise.sql +++ b/data-track/week-9/exercise_7/solutions/exercise.sql @@ -1,6 +1,6 @@ --- Exercise 6 solution: Build views, then query them +-- Exercise 7 solution: Build views, then query them --- 6a. The two views +-- 7a. The two views CREATE OR REPLACE VIEW vw_dim_zones AS -- WHY CREATE OR REPLACE: lets you re-run this script while iterating without first DROPping the view; a plain CREATE VIEW would error if the view already exists SELECT location_id, @@ -14,7 +14,7 @@ FROM nyc_taxi.raw_trips WHERE fare_amount >= 0; -- WHY filter in the view: the cleaning rule (no negative fares) lives in one place, so every query against the view is automatically clean --- 6b. Highest total fare revenue by borough +-- 7b. Highest total fare revenue by borough SELECT d.borough, ROUND(SUM(f.fare_amount)::numeric, 2) AS total_revenue -- WHY SUM: revenue is the total of all fares in the borough, not an average @@ -26,7 +26,7 @@ ORDER BY total_revenue DESC LIMIT 1; -- WHY LIMIT 1: the question asks for the single highest-revenue borough --- 6c. Top 5 pickup zones by trip count +-- 7c. Top 5 pickup zones by trip count SELECT d.zone, COUNT(*) AS trips From 78e466410f563fb5c51fa087414ee4239756f511 Mon Sep 17 00:00:00 2001 From: Lasse Benninga Date: Mon, 29 Jun 2026 12:04:04 +0200 Subject: [PATCH 5/5] Add starter queries and skeletons to all Week 9 exercise files --- data-track/week-9/exercise_1/exercise.sql | 8 +++++++- data-track/week-9/exercise_2/exercise.sql | 9 ++++++++- data-track/week-9/exercise_3/exercise.sql | 9 ++++++++- data-track/week-9/exercise_4/exercise.sql | 12 +++++++++++- data-track/week-9/exercise_5/exercise.sql | 5 +++++ data-track/week-9/exercise_6/exercise.sql | 13 +++++++++++++ data-track/week-9/exercise_7/exercise.sql | 19 +++++++++++++++++++ 7 files changed, 71 insertions(+), 4 deletions(-) diff --git a/data-track/week-9/exercise_1/exercise.sql b/data-track/week-9/exercise_1/exercise.sql index d6bd727..d25f960 100644 --- a/data-track/week-9/exercise_1/exercise.sql +++ b/data-track/week-9/exercise_1/exercise.sql @@ -14,4 +14,10 @@ -- five rows are the same every run (a bare LIMIT returns an arbitrary -- sample that can change between runs). --- TODO: select the four columns and join nyc_taxi.raw_trips to nyc_taxi.raw_zones, then limit to 5 rows. +-- Starter query: exploring raw_trips columns +SELECT pickup_datetime, trip_distance, fare_amount, pickup_location_id +FROM nyc_taxi.raw_trips +ORDER BY pickup_datetime +LIMIT 5; + +-- TODO: rewrite the query above to join nyc_taxi.raw_trips to nyc_taxi.raw_zones on pickup_location_id = location_id and select the zone name instead of the ID. diff --git a/data-track/week-9/exercise_2/exercise.sql b/data-track/week-9/exercise_2/exercise.sql index 025925a..2a48ea9 100644 --- a/data-track/week-9/exercise_2/exercise.sql +++ b/data-track/week-9/exercise_2/exercise.sql @@ -11,4 +11,11 @@ -- Hint: Join to nyc_taxi.raw_zones to get borough, then GROUP BY z.borough. Every -- non-aggregated column in the SELECT must appear in the GROUP BY. --- TODO: join nyc_taxi.raw_trips to nyc_taxi.raw_zones, group by borough, and count trips plus average fare. +-- Starter query: counting raw trips by location ID (no join) +SELECT t.pickup_location_id, COUNT(*) AS trips, AVG(t.fare_amount) AS avg_fare +FROM nyc_taxi.raw_trips t +GROUP BY t.pickup_location_id +ORDER BY trips DESC +LIMIT 5; + +-- TODO: rewrite the query above to join to nyc_taxi.raw_zones, group by the borough name, and order by total trips descending. diff --git a/data-track/week-9/exercise_3/exercise.sql b/data-track/week-9/exercise_3/exercise.sql index bc9cd8d..c4c6342 100644 --- a/data-track/week-9/exercise_3/exercise.sql +++ b/data-track/week-9/exercise_3/exercise.sql @@ -11,4 +11,11 @@ -- Hint: Use pickup_datetime::date to drop the time part. Define the daily counts -- in a WITH block, then ORDER BY ... DESC LIMIT 1 over the CTE. --- TODO: define a CTE of daily trip counts, then select the single busiest day from it. +-- Starter query: counting trips per day without a CTE +SELECT t.pickup_datetime::date AS trip_date, COUNT(*) AS trips +FROM nyc_taxi.raw_trips t +GROUP BY trip_date +ORDER BY trips DESC +LIMIT 5; + +-- TODO: rewrite the query above to use a CTE (WITH block) containing the daily counts, and select the single busiest day from the CTE. diff --git a/data-track/week-9/exercise_4/exercise.sql b/data-track/week-9/exercise_4/exercise.sql index 03b3161..d569109 100644 --- a/data-track/week-9/exercise_4/exercise.sql +++ b/data-track/week-9/exercise_4/exercise.sql @@ -29,4 +29,14 @@ FROM ( GROUP BY borough ORDER BY avg_fare DESC; --- TODO: rewrite the query above using CTEs (one to filter, one to join, then aggregate). +-- Starter skeleton: refactoring into CTEs +WITH positive_trips AS ( + -- TODO: select trips where fare_amount > 0 from nyc_taxi.raw_trips + SELECT * FROM nyc_taxi.raw_trips LIMIT 1 +), +joined_trips AS ( + -- TODO: join positive_trips to nyc_taxi.raw_zones + SELECT * FROM positive_trips +) +-- TODO: final group by and aggregate over joined_trips +SELECT 1; diff --git a/data-track/week-9/exercise_5/exercise.sql b/data-track/week-9/exercise_5/exercise.sql index e72e50c..f792da2 100644 --- a/data-track/week-9/exercise_5/exercise.sql +++ b/data-track/week-9/exercise_5/exercise.sql @@ -16,7 +16,12 @@ -- Cartesian product: no join condition -- TODO: EXPLAIN a CROSS JOIN of nyc_taxi.raw_trips and nyc_taxi.raw_zones (no ON clause). +-- Starter skeleton: (uncomment and prepend EXPLAIN) +-- SELECT * FROM nyc_taxi.raw_trips t CROSS JOIN nyc_taxi.raw_zones z; -- Filtered join: one zone per trip -- TODO: EXPLAIN an INNER JOIN of nyc_taxi.raw_trips and nyc_taxi.raw_zones on pickup_location_id = location_id. +-- Starter skeleton: (uncomment and prepend EXPLAIN) +-- SELECT * FROM nyc_taxi.raw_trips t INNER JOIN nyc_taxi.raw_zones z ON t.pickup_location_id = z.location_id; + diff --git a/data-track/week-9/exercise_6/exercise.sql b/data-track/week-9/exercise_6/exercise.sql index 6a9468e..36a87a4 100644 --- a/data-track/week-9/exercise_6/exercise.sql +++ b/data-track/week-9/exercise_6/exercise.sql @@ -19,11 +19,24 @@ -- TODO: count rows where pickup_location_id IS NULL. -- Expect 5 here: some pickup IDs are intentionally NULL. The real dirt is -- duplicates (6b), negative fares, and orphans (6c). +-- Starter skeleton: +SELECT COUNT(*) FROM nyc_taxi.raw_trips WHERE 1=0; -- replace 1=0 with filter -- 6b. Duplicate trips (same vendor + pickup + dropoff time) -- TODO: group by the three columns and keep groups with more than one row. +-- Starter skeleton: +SELECT vendor_id, pickup_datetime, dropoff_datetime, COUNT(*) +FROM nyc_taxi.raw_trips +GROUP BY 1, 2, 3 +LIMIT 1; -- replace LIMIT with HAVING count filter -- 6c. Orphaned pickup IDs not present in nyc_taxi.raw_zones -- TODO: LEFT JOIN nyc_taxi.raw_trips to nyc_taxi.raw_zones and keep rows with no matching zone. +-- Starter skeleton: +SELECT DISTINCT t.pickup_location_id +FROM nyc_taxi.raw_trips t +LEFT JOIN nyc_taxi.raw_zones z ON t.pickup_location_id = z.location_id +LIMIT 1; -- replace LIMIT with WHERE clause to find unmatched zones + diff --git a/data-track/week-9/exercise_7/exercise.sql b/data-track/week-9/exercise_7/exercise.sql index 23faeb5..3758772 100644 --- a/data-track/week-9/exercise_7/exercise.sql +++ b/data-track/week-9/exercise_7/exercise.sql @@ -22,11 +22,30 @@ -- 7a. The two views -- TODO: CREATE OR REPLACE VIEW vw_dim_zones from nyc_taxi.raw_zones. -- TODO: CREATE OR REPLACE VIEW vw_fact_trips from nyc_taxi.raw_trips, excluding negative fares. +-- Starter skeletons: +CREATE OR REPLACE VIEW vw_dim_zones AS +SELECT 1; -- replace with columns from raw_zones + +CREATE OR REPLACE VIEW vw_fact_trips AS +SELECT 1; -- replace with columns from raw_trips and filter negative fares -- 7b. Highest total fare revenue by borough -- TODO: join the two views, sum fare per borough, and return the top borough. +-- Starter skeleton: +SELECT d.borough, SUM(f.fare_amount) AS total_revenue +FROM vw_fact_trips f +INNER JOIN vw_dim_zones d ON f.pickup_location_id = d.location_id +GROUP BY 1 +LIMIT 1; -- replace with correct ORDER BY and final touches -- 7c. Top 5 pickup zones by trip count -- TODO: join the two views, count trips per zone, and return the top 5 zones. +-- Starter skeleton: +SELECT d.zone, COUNT(*) AS trips +FROM vw_fact_trips f +INNER JOIN vw_dim_zones d ON f.pickup_location_id = d.location_id +GROUP BY 1 +LIMIT 5; -- replace with correct ORDER BY and final touches +