From 33666b9f976a815c331086e52bc754ab7ee4c466 Mon Sep 17 00:00:00 2001 From: forhappy Date: Fri, 4 Sep 2026 00:00:11 -0700 Subject: [PATCH] refactor(remote-git): remove obsolete HTTP browser example --- Cargo.lock | 3 - crates/crab-remote-git/Cargo.toml | 5 +- crates/crab-remote-git/README.md | 71 +-- .../crab-remote-git/examples/browse_http.html | 127 ----- .../crab-remote-git/examples/browse_http.rs | 470 ------------------ 5 files changed, 2 insertions(+), 674 deletions(-) delete mode 100644 crates/crab-remote-git/examples/browse_http.html delete mode 100644 crates/crab-remote-git/examples/browse_http.rs diff --git a/Cargo.lock b/Cargo.lock index ea61a4562..0138e0243 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2360,7 +2360,6 @@ name = "crab-remote-git" version = "0.1.0" dependencies = [ "async-trait", - "axum 0.8.9", "blake3", "bytes", "crab-git", @@ -2376,8 +2375,6 @@ dependencies = [ "gix-pack", "object_store", "proptest", - "rand 0.9.4", - "serde", "serde_json", "sha1 0.10.6", "tempfile", diff --git a/crates/crab-remote-git/Cargo.toml b/crates/crab-remote-git/Cargo.toml index f9cd926b0..6aaa95220 100644 --- a/crates/crab-remote-git/Cargo.toml +++ b/crates/crab-remote-git/Cargo.toml @@ -35,12 +35,9 @@ tracing = { workspace = true } [dev-dependencies] async-trait = { workspace = true } -axum = { version = "0.8", default-features = false, features = ["http1", "json", "query", "tokio"] } crab-metadata = { workspace = true, features = ["remote-index", "storage"] } object_store = { workspace = true } proptest = "1" -rand = "0.9" -serde = { workspace = true } tempfile = { workspace = true } -tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "signal", "time"] } +tokio = { workspace = true, features = ["macros", "rt-multi-thread", "time"] } tracing-subscriber = "0.3" diff --git a/crates/crab-remote-git/README.md b/crates/crab-remote-git/README.md index e756f9866..660ea48d3 100644 --- a/crates/crab-remote-git/README.md +++ b/crates/crab-remote-git/README.md @@ -159,76 +159,7 @@ the reader walks verified raw commits breadth-first from the pinned refs, checki nearby merge parents before older ancestry on either branch. That fallback remains bounded by the operation's history and object budgets. -## Live qualification example - -### Local HTTP browser and latency measurements - -`browse_http` is a small Rust/Axum example with a bundled browser UI. It uses -the crate directly for refs, commit metadata, first-parent history, directories, -and exact Git blob bytes. It needs an already-published repository and current -object catalog, as described below. No Git executable, checkout, or local object -database is used by the server. HTTP dependencies are dev-dependencies only. - -Configure S3 credentials in the process environment; for local RustFS, also set -`AWS_ENDPOINT_URL=http://127.0.0.1:9000`, `AWS_ALLOW_HTTP=true`, -`AWS_REGION=us-east-1`, and `AWS_VIRTUAL_HOSTED_STYLE_REQUEST=false`. -See the [local RustFS guide](../../crab/docs/guides/local-dev-rustfs.md). -Build using a separate target directory for this checkout on the workspace volume: - -```sh -CARGO_TARGET_DIR="$HOME/Workspace/crabbuild-target/crab-http-example" \ - cargo build --locked --release -p crab-remote-git --example browse_http - -# Run from an empty directory; a source repository is not an input. -"$HOME/Workspace/crabbuild-target/crab-http-example/release/examples/browse_http" \ - 8787 -``` - -Open `http://127.0.0.1:8787`. Select a ref or full commit SHA, navigate the tree, -read files, or browse commits. **Benchmark this request** performs one cold -read, one shared-runtime priming read, and five measured warm reads. It reports -the warm median and retains the individual request measurements. Ctrl-C drains -requests and shuts down the reader runtime. - -The JSON API and binary blob endpoint also work with `curl -i`: - -| GET endpoint | Response | -| --- | --- | -| `/api/refs` | Pinned generation, pack count, HEAD, and refs | -| `/api/commit?rev=main` | Commit OID, tree, parents, author, and message | -| `/api/commits?rev=main&limit=20` | First-parent commit page, including the selected commit | -| `/api/tree?rev=main&path=pkg&limit=50` | Immediate entries with OID, mode, kind, and byte-preserving `path_hex` | -| `/api/blob?rev=main&path=README.md` | Exact Git bytes; blob OID in `X-Crab-Blob-Oid` | - -`rev` defaults to the pinned HEAD. `path_hex` can replace UTF-8 `path` to preserve -arbitrary Git path bytes. Display strings are lossy UTF-8; commit `message_hex` -preserves message bytes. Pages return a signed opaque `next` value; pass it as -`cursor` with the same revision, path, and limit. Page limits are 1–200, default -50. Cursors expire when the server restarts. Submodules are metadata-only; -symlinks return their stored target bytes. Crab/LFS pointers remain pointers. - -Every handled read returns `Server-Timing` durations in milliseconds: - -- `open`: repository handshake for `mode=cold`; zero for `mode=warm`. -- `read`: snapshot resolution, semantic read, response encoding, and explicit - locator close. `/api/refs` reads the already-open handle's in-memory refs. -- `shutdown`: draining a cold request's runtime. -- `total`: handler time through response construction, excluding HTTP body - transmission. The UI separately measures round trip through full body receipt. - -`mode=warm` (default) shares the startup repository handle and bounded runtime -caches; it does not guarantee cache hits. `mode=cold` creates a fresh runtime and -reopens the repository without disturbing the shared caches. It shares the S3 -transport and does not flush OS or RustFS caches. This compares Crab cache -behavior on local RustFS, not production cloud latency. Responses disable HTTP -caching so repeated browser requests reach the server. - -The server pins one generation at startup. Restart after publishing changes; -a cold read of a different generation returns 409 instead of comparing different -data. The example binds only to loopback and allows four concurrent reads -(additional requests get 429), with 30-second semantic operation budgets and -an 8 MiB response budget. This is a local inspection tool, not an authenticated -multi-user service. +## Live qualification ### Command-line qualification diff --git a/crates/crab-remote-git/examples/browse_http.html b/crates/crab-remote-git/examples/browse_http.html deleted file mode 100644 index 212212ead..000000000 --- a/crates/crab-remote-git/examples/browse_http.html +++ /dev/null @@ -1,127 +0,0 @@ - - - - - -Crab · Remote Git inspector - - - -

Crab / Remote Git inspector

Browse repository data directly from object storage. No local clone.

-
-
-
- - - - -
-

Opening repository…

-
- -
-
-

/

-

Benchmark: one cold request, one shared-runtime warm-up, then five warm samples.

-

-
-
-

Request latency

-

Milliseconds. Open = fresh repository handshake. Read = crate operation, JSON encoding, and locator close. Total also includes cold runtime shutdown. Round trip includes receiving the full response body.

-
RequestModeOpenReadTotalRound tripBytesStatus
-

Cold resets Crab runtime caches only; RustFS, OS, and transport caches may remain warm. Shared-runtime requests can still miss caches. Repository generation is pinned at startup; restart after a push. Blob responses contain Git bytes, including pointer files.

-
-
- - - diff --git a/crates/crab-remote-git/examples/browse_http.rs b/crates/crab-remote-git/examples/browse_http.rs deleted file mode 100644 index 454c58d5b..000000000 --- a/crates/crab-remote-git/examples/browse_http.rs +++ /dev/null @@ -1,470 +0,0 @@ -use std::io; -use std::sync::Arc; -use std::time::{Duration, Instant}; - -use axum::extract::{Path, Query, Request, State}; -use axum::http::StatusCode; -use axum::middleware::{self, Next}; -use axum::response::{Html, IntoResponse, Response}; -use axum::{Json, Router, routing::get}; -use crab_remote_git::{ - Commit, Error, GitPath, HistoryTraversal, OperationKind, OperationLimits, PageCursor, - PageRequest, RemoteGitRepository, RemoteGitRuntime, RepositoryIdentity, RepositoryOptions, - Revision, RevisionError, -}; -use crab_storage::{StorageProviderKind, Store, StoreLayout, build_static_env_store}; -use serde::Deserialize; -use serde_json::{Value, json}; -use tokio::sync::Semaphore; -use tokio_util::sync::CancellationToken; - -struct Server { - store: Store, - layout: StoreLayout, - identity: RepositoryIdentity, - options: RepositoryOptions, - repository: RemoteGitRepository, - cursor_key: [u8; 32], - admission: Semaphore, - cancellation: CancellationToken, - port: u16, -} - -#[derive(Clone, Copy, Deserialize)] -#[serde(rename_all = "lowercase")] -enum Action { - Refs, - Commit, - Commits, - Tree, - Blob, -} - -#[derive(Clone, Copy, Default, Deserialize)] -#[serde(rename_all = "lowercase")] -enum CacheMode { - Cold, - #[default] - Warm, -} - -#[derive(Default, Deserialize)] -#[serde(default, deny_unknown_fields)] -struct Parameters { - mode: CacheMode, - rev: Option, - path: Option, - path_hex: Option, - limit: Option, - cursor: Option, -} - -#[derive(Debug, thiserror::Error)] -enum ApiError { - #[error("{0}")] - Input(&'static str), - #[error("remote Git operation failed")] - Remote(#[from] Error), - #[error("repository generation changed; restart the server before comparing timings")] - Changed, -} - -impl IntoResponse for ApiError { - fn into_response(self) -> Response { - let (status, message) = match &self { - Self::Input(message) => (StatusCode::BAD_REQUEST, *message), - Self::Changed => ( - StatusCode::CONFLICT, - "Repository changed; restart the server", - ), - Self::Remote(error) => match error { - Error::PathNotFound - | Error::EmptyRepository - | Error::Revision { - reason: RevisionError::NotFound | RevisionError::NotReachable, - } => (StatusCode::NOT_FOUND, "Path or revision not found"), - Error::InvalidPath { .. } - | Error::InvalidCursor { .. } - | Error::InvalidLimit { .. } - | Error::Revision { .. } - | Error::EntryNotBlob { .. } - | Error::PathComponentNotTree { .. } => ( - StatusCode::BAD_REQUEST, - "Invalid path, revision, cursor, or entry kind", - ), - Error::LimitExceeded { .. } => ( - StatusCode::UNPROCESSABLE_ENTITY, - "Read exceeds the example's operation limits", - ), - Error::Timeout { .. } => (StatusCode::GATEWAY_TIMEOUT, "Remote read timed out"), - Error::Cancelled => (StatusCode::SERVICE_UNAVAILABLE, "Remote read cancelled"), - Error::RepositoryIndexing { .. } => ( - StatusCode::SERVICE_UNAVAILABLE, - "Repository is indexing; run the metadata owner from the uploader", - ), - _ => ( - StatusCode::BAD_GATEWAY, - "Remote Git read failed; check storage and repository health", - ), - }, - }; - (status, Json(json!({"error": message}))).into_response() - } -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - let mut args = std::env::args().skip(1); - let usage = "usage: browse_http [port=8787]"; - let bucket = args.next().ok_or_else(|| io::Error::other(usage))?; - let prefix = args.next().ok_or_else(|| io::Error::other(usage))?; - let port: u16 = args - .next() - .map(|value| value.parse()) - .transpose()? - .unwrap_or(8787); - if args.next().is_some() { - return Err(io::Error::other(usage).into()); - } - let listener = tokio::net::TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, port)).await?; - let port = listener.local_addr()?.port(); - let store = build_static_env_store(&bucket, StorageProviderKind::S3)?; - let layout = StoreLayout::new(store.clone(), prefix.clone()); - let identity = RepositoryIdentity::new(format!("s3:{bucket}"), prefix, 1)?; - let options = RepositoryOptions::new( - Default::default(), - OperationLimits { - max_duration: Duration::from_secs(30), - max_response_bytes: 8 * 1024 * 1024, - ..Default::default() - }, - )?; - let runtime = Arc::new(RemoteGitRuntime::default()); - let cancellation = CancellationToken::new(); - let started = Instant::now(); - let result = async { - let repository = RemoteGitRepository::open( - store.clone(), - layout.clone(), - identity.clone(), - Arc::clone(&runtime), - options, - &cancellation, - ) - .await?; - println!( - "Opened generation {} in {:.3} ms", - repository.generation(), - milliseconds(started) - ); - let server = Arc::new(Server { - store, - layout, - identity, - options, - repository, - cursor_key: rand::random(), - admission: Semaphore::new(4), - cancellation: cancellation.clone(), - port, - }); - let router = Router::new() - .route( - "/", - get(|| async { Html(include_str!("browse_http.html")) }), - ) - .route("/api/{action}", get(read)) - .layer(middleware::from_fn_with_state( - Arc::clone(&server), - local_only, - )) - .with_state(server); - println!("Browse http://127.0.0.1:{port} (Ctrl-C to stop)"); - axum::serve(listener, router) - .with_graceful_shutdown(async move { - if tokio::signal::ctrl_c().await.is_err() { - eprintln!("Unable to listen for Ctrl-C; shutting down"); - } - cancellation.cancel(); - }) - .await?; - Ok::<_, Box>(()) - } - .await; - runtime.shutdown().await; - result -} - -async fn local_only(State(server): State>, request: Request, next: Next) -> Response { - let host = request - .headers() - .get("host") - .and_then(|value| value.to_str().ok()); - let allowed = [ - format!("127.0.0.1:{}", server.port), - format!("localhost:{}", server.port), - ]; - // A loopback listener alone does not prevent browser DNS rebinding. - if !allowed.iter().any(|value| Some(value.as_str()) == host) { - return StatusCode::FORBIDDEN.into_response(); - } - ( - [ - ("cache-control", "no-store"), - ("x-content-type-options", "nosniff"), - ("content-security-policy", "default-src 'self'; script-src 'self' 'unsafe-inline'; style-src 'self' 'unsafe-inline'; base-uri 'none'; frame-ancestors 'none'"), - ], - next.run(request).await, - ).into_response() -} - -async fn read( - State(server): State>, - Path(action): Path, - Query(params): Query, -) -> Response { - let started = Instant::now(); - let Ok(_permit) = server.admission.try_acquire() else { - return ( - StatusCode::TOO_MANY_REQUESTS, - Json(json!({"error": "Four reads already in flight; retry after they finish"})), - ) - .into_response(); - }; - let cancellation = server.cancellation.child_token(); - let _cancel_on_drop = cancellation.clone().drop_guard(); - let mut open_ms = 0.0; - let mut read_ms = 0.0; - let mut shutdown_ms = 0.0; - let cold_runtime = - matches!(params.mode, CacheMode::Cold).then(|| Arc::new(RemoteGitRuntime::default())); - let result = async { - let limit = params.limit.unwrap_or(50); - if !(1..=200).contains(&limit) { - return Err(ApiError::Input("limit must be between 1 and 200")); - } - let path = match (¶ms.path, ¶ms.path_hex) { - (Some(_), Some(_)) => return Err(ApiError::Input("use path or path_hex, not both")), - (_, Some(value)) => GitPath::new(decode_hex(value)?)?, - (Some(value), _) => GitPath::new(value.as_bytes().to_vec())?, - _ => GitPath::root(), - }; - let cursor = params - .cursor - .as_deref() - .map(|value| server.decode_cursor(value)) - .transpose()?; - let page = PageRequest::new(limit, cursor)?; - let repository = match &cold_runtime { - Some(runtime) => { - let timer = Instant::now(); - let opened = RemoteGitRepository::open( - server.store.clone(), - server.layout.clone(), - server.identity.clone(), - Arc::clone(runtime), - server.options, - &cancellation, - ) - .await; - open_ms = milliseconds(timer); - let repository = opened?; - if repository.generation() != server.repository.generation() { - return Err(ApiError::Changed); - } - repository - } - None => server.repository.clone(), - }; - let timer = Instant::now(); - let response = execute( - &server, - &repository, - action, - params.rev.as_deref(), - path, - page, - &cancellation, - ) - .await; - read_ms = milliseconds(timer); - response.map_err(ApiError::from) - } - .await; - // Cold runtimes own locator cleanup tasks too; drain them on success and failure. - if let Some(runtime) = cold_runtime { - let timer = Instant::now(); - runtime.shutdown().await; - shutdown_ms = milliseconds(timer); - } - let mode = match params.mode { - CacheMode::Cold => "cold", - CacheMode::Warm => "warm", - }; - let response = match result { - Ok(response) => response, - Err(error) => error.into_response(), - }; - ( - [ - ("server-timing", format!("open;dur={open_ms:.3}, read;dur={read_ms:.3}, shutdown;dur={shutdown_ms:.3}, total;dur={:.3}", milliseconds(started))), - ("x-crab-cache-mode", mode.to_owned()), - ("x-crab-generation", server.repository.generation().to_string()), - ], - response, - ).into_response() -} - -async fn execute( - server: &Server, - repository: &RemoteGitRepository, - action: Action, - revision: Option<&str>, - path: GitPath, - page: PageRequest, - cancellation: &CancellationToken, -) -> crab_remote_git::Result { - if matches!(action, Action::Refs) { - let refs = repository.refs(); - return Ok(Json(json!({ - "generation": repository.generation(), "packs": repository.pack_count(), - "head": refs.head.as_ref().map(|head| json!({"name": head.name, "oid": head.target.to_string()})), - "refs": refs.entries.iter().map(|entry| json!({ - "name": entry.name, "oid": entry.target.to_string(), "peeled": entry.peeled.map(|oid| oid.to_string()), - })).collect::>(), - })).into_response()); - } - let revision = match revision { - Some(value) => Revision::parse(value)?, - None => Revision::Reference( - repository - .refs() - .head - .as_ref() - .ok_or(Error::EmptyRepository)? - .name - .clone(), - ), - }; - let kind = match action { - Action::Refs => OperationKind::Repository, - Action::Commit => OperationKind::Commit, - Action::Commits => OperationKind::History, - Action::Tree => OperationKind::Tree, - Action::Blob => OperationKind::Content, - }; - let operation = repository.operation(kind, cancellation).await?; - let result = async { - let snapshot = repository.snapshot(&revision, &operation).await?; - let response = match action { - Action::Refs | Action::Commit => { - Json(commit_json(&snapshot.commit(&operation).await?)).into_response() - } - Action::Commits => { - let result = snapshot - .history(HistoryTraversal::FirstParent, &page, &operation) - .await?; - Json(json!({ - "items": result.items.iter().map(commit_json).collect::>(), - "next": result.next.map(|cursor| server.encode_cursor(cursor)), - })) - .into_response() - } - Action::Tree => { - let result = snapshot.list_directory(&path, &page, &operation).await?; - Json(json!({ - "items": result.items.iter().map(|entry| json!({ - "path": String::from_utf8_lossy(entry.path.as_bytes()), - "path_hex": encode_hex(entry.path.as_bytes()), - "oid": entry.oid.to_string(), "mode": format!("{:06o}", entry.mode.raw()), - "kind": format!("{:?}", entry.kind), - })).collect::>(), - "next": result.next.map(|cursor| server.encode_cursor(cursor)), - })) - .into_response() - } - Action::Blob => { - let blob = snapshot.read_blob(&path, &operation).await?; - ( - [ - ("content-type", "application/octet-stream".to_owned()), - ("content-disposition", "attachment".to_owned()), - ("x-crab-blob-oid", blob.metadata.oid.to_string()), - ( - "x-crab-content-class", - format!("{:?}", blob.metadata.classification), - ), - ], - blob.bytes, - ) - .into_response() - } - }; - Ok(( - [("x-crab-commit", snapshot.commit_oid().to_string())], - response, - ) - .into_response()) - } - .await; - operation.finish(result).await -} - -fn commit_json(commit: &Commit) -> Value { - json!({ - "oid": commit.oid.to_string(), "tree": commit.tree.to_string(), - "parents": commit.parents.iter().map(ToString::to_string).collect::>(), - "author": String::from_utf8_lossy(&commit.author.name), - "author_seconds": commit.author.seconds, - "message": String::from_utf8_lossy(&commit.message), - "message_hex": encode_hex(&commit.message), - }) -} - -impl Server { - fn encode_cursor(&self, cursor: PageCursor) -> String { - let bytes = cursor.as_bytes(); - format!( - "{}.{}", - encode_hex(bytes), - blake3::keyed_hash(&self.cursor_key, bytes).to_hex() - ) - } - - fn decode_cursor(&self, value: &str) -> Result { - let (payload, signature) = value - .split_once('.') - .ok_or(ApiError::Input("Invalid cursor"))?; - let bytes = decode_hex(payload)?; - let signature = decode_hex(signature)?; - // Compare Hash values with Blake3's constant-time equality implementation. - if blake3::keyed_hash(&self.cursor_key, &bytes) != *signature.as_slice() { - return Err(ApiError::Input("Invalid cursor signature")); - } - Ok(PageCursor::from_bytes(bytes)?) - } -} - -fn encode_hex(bytes: &[u8]) -> String { - bytes.iter().map(|byte| format!("{byte:02x}")).collect() -} - -fn decode_hex(value: &str) -> Result, ApiError> { - if !value.len().is_multiple_of(2) - || !value.bytes().all(|byte| byte.is_ascii_hexdigit()) - || value.len() > 128 * 1024 - { - return Err(ApiError::Input("Invalid hex bytes")); - } - (0..value.len()) - .step_by(2) - .map(|index| { - u8::from_str_radix(&value[index..index + 2], 16) - .map_err(|_| ApiError::Input("Invalid hex bytes")) - }) - .collect() -} - -fn milliseconds(started: Instant) -> f64 { - started.elapsed().as_secs_f64() * 1000.0 -}