Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions .buckconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
# jaylene-slide Buck2 configuration
#
# Uses sensenet's Buck2 infrastructure via flake module.
# Run `nix develop .#buck2-default` to enter the build environment.

[cells]
root = .
prelude = nix/build/prelude
toolchains = toolchains
none = none

[cell_aliases]
config = prelude
ovr_config = prelude
fbcode = none
fbsource = none
fbcode_macros = none
buck = none

[parser]
target_platform_detector_spec = target:root//...->prelude//platforms:default

[buck2]
materializations = deferred
digest_algorithms = BLAKE3

[build]
execution_platforms = toolchains//:default
1 change: 1 addition & 0 deletions .clang-format
23 changes: 23 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1 +1,24 @@
dist-newstyle/
result
result-*
.direnv/
buck-out/
.buckconfig.local
nix/build/

# Generated BUCK file (from BUILD.dhall)
BUCK

# GHC build artifacts
*.hi
*.o
*.dyn_hi
*.dyn_o

# LSP generated files
/compile_commands.json
/.clangd
/.clang-tidy
/.rustfmt.toml
/.stylua.toml
/hie.yaml
115 changes: 115 additions & 0 deletions BUILD.dhall
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
--| slide - Console cowboy for the sprawl
--|
--| Ingress adapter that jacks into OpenAI-compatible inference endpoints,
--| parses their SSE/JSON garbage, and emits clean SIGIL binary frames over ZMQ.
--|
--| Build: ./dhall-to-buck BUILD.dhall > BUCK

let A = ./dhall/prelude/package.dhall
let S = ./dhall/prelude/to-starlark.dhall

-- Library sources (all modules under src/Slide/)
let librarySrcs =
[ "src/Slide/Chunk.hs"
, "src/Slide/HotTable.hs"
, "src/Slide/Jack.hs"
, "src/Slide/Model.hs"
, "src/Slide/Parse.hs"
, "src/Slide/Provider.hs"
, "src/Slide/Provider/Baseten.hs"
, "src/Slide/Tokenizer.hs"
, "src/Slide/Tokenizer/FFI.hs"
, "src/Slide/Wire/Decode.hs"
, "src/Slide/Wire/Encode.hs"
, "src/Slide/Wire/Frame.hs"
, "src/Slide/Wire/Types.hs"
, "src/Slide/Wire/Varint.hs"
]

-- Executable sources
let appSrcs = [ "app/Main.hs" ]

-- C++ FFI sources for tokenizers binding
let cxxSrcs = [ "cbits/tokenizers_c.cpp" ]

-- All Haskell packages needed
let packages =
[ "base"
, "aeson"
, "async"
, "blake3"
, "bytestring"
, "case-insensitive"
, "containers"
, "crypton"
, "data-default-class"
, "dhall"
, "http2"
, "http-semantics"
, "http-types"
, "katip"
, "megaparsec"
, "memory"
, "network"
, "optparse-applicative"
, "prometheus-client"
, "prometheus-metrics-ghc"
, "random"
, "text"
, "time-manager"
, "tls"
, "vector"
, "wai"
, "warp"
, "zeromq4-haskell"
]

-- Language extensions
let extensions =
[ "BangPatterns"
, "CApiFFI"
, "DerivingStrategies"
, "ForeignFunctionInterface"
, "LambdaCase"
, "NumericUnderscores"
, "OverloadedStrings"
, "PatternSynonyms"
, "StrictData"
]

-- GHC options
let ghcOptions =
[ "-O2"
, "-threaded"
, "-rtsopts"
, "-with-rtsopts=-N -A64m -I0"
, "-isrc"
]

-- External libraries (from Nix via .buckconfig.local)
let extraLibs =
[ "tokenizers_cpp"
, "tokenizers_c"
, "sentencepiece"
]

-- slide executable with FFI
let slide =
(A.haskellFFIBinary "slide" (librarySrcs # appSrcs) cxxSrcs)
with packages = packages
with language_extensions = extensions
with ghc_options = ghcOptions
with extra_libs = extraLibs
with include_dirs = [ "cbits" ]

in { rules = [ S.haskellFFIBinary slide ]
, header = ''
# Generated from BUILD.dhall
#
# Library paths for tokenizers-cpp come from .buckconfig.local [slide] section
# which is auto-generated by `nix develop`
# The haskell_ffi_binary rule reads these paths via read_root_config()

load("@toolchains//:haskell.bzl", "haskell_ffi_binary")
''
}
58 changes: 58 additions & 0 deletions dhall-to-buck
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
#!/usr/bin/env bash
# dhall-to-buck - Transpile BUILD.dhall to BUCK
set -euo pipefail

[[ $# -lt 1 ]] && { echo "Usage: $0 <BUILD.dhall>" >&2; exit 1; }

INPUT="$1"

# Check if this is the new format (exports { rules, header }) or legacy (List Binary)
STRUCTURE=$(dhall type <<< "./$INPUT" 2>/dev/null || echo "unknown")

if [[ "$STRUCTURE" == *"rules"* && "$STRUCTURE" == *"header"* ]]; then
# New format: { rules : List Text, header : Text }
HEADER=$(dhall text <<< "(./$INPUT).header")
echo "# Generated from $INPUT"
echo "$HEADER"
dhall text <<< "let P = ./dhall/prelude/Prelude.dhall in P.Text.concatSep \"\\n\" (./$INPUT).rules"
exit 0
fi

# Legacy format: List C.Binary (for backward compat with existing BUILD.dhall files)
DEPS=$(dhall text <<< "(./dhall/prelude/extract-deps.dhall) ./$INPUT" 2>/dev/null || echo "")

if [[ -z "$DEPS" ]]; then
echo "# Generated from $INPUT"
echo 'load("@toolchains//:cxx.bzl", "cxx_binary")'
echo ""
dhall text <<< "
let S = ./dhall/prelude/to-starlark.dhall
let C = ./dhall/prelude/Cxx.dhall
let t = List/head C.Binary ./$INPUT
in merge { None = \"\", Some = \\(b : C.Binary) -> S.binary b { compiler = [] : List Text, linker = [] : List Text } } t"
exit 0
fi

FLAGS=$(buck2 run //src/nix-analyze:nix-analyze -- resolve $DEPS 2>/dev/null | grep -v "^\[")

CF="" LF="" next=false
while IFS= read -r f; do
[[ -z "$f" ]] && continue
if $next; then CF+="\"$f\", "; next=false
elif [[ "$f" == "-isystem" ]]; then CF+="\"$f\", "; next=true
elif [[ "$f" == -I* ]]; then CF+="\"$f\", "
elif [[ "$f" == -L* || "$f" == -Wl,* || "$f" == -l* ]]; then LF+="\"$f\", "
fi
done <<< "$FLAGS"

echo "# Generated from $INPUT"
echo "# Deps: ${DEPS//$'\n'/, }"
echo ""
echo 'load("@toolchains//:cxx.bzl", "cxx_binary")'
echo ""
dhall text <<< "
let S = ./dhall/prelude/to-starlark.dhall
let C = ./dhall/prelude/Cxx.dhall
let t = List/head C.Binary ./$INPUT
let f = { compiler = [${CF%,*}], linker = [${LF%,*}] }
in merge { None = \"\", Some = \\(b : C.Binary) -> S.binary b f } t"
125 changes: 125 additions & 0 deletions dhall/Build.dhall
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
{- Build.dhall

Build target definition with coeffects.
-}

let Resource = ./Resource.dhall
let Toolchain = ./Toolchain.dhall
let Triple = ./Triple.dhall

-- Dependency reference
let Dep
: Type
= < -- Local target (same repo)
Local : Text -- ":foo" or "//pkg:foo"
-- External (content-addressed)
| External : { hash : Text, name : Text }
-- Pkg-config
| PkgConfig : Text -- "openssl", "zlib"
-- Nix flake reference (resolved at graph construction)
| Flake : Text -- "nixpkgs#openssl", ".#libfoo", "github:owner/repo#pkg"
>

-- Source specification
let Src
: Type
= < -- Local files
Files : List Text
-- Content-addressed fetch
| Fetch : { url : Text, hash : Text }
-- Git
| Git : { url : Text, rev : Text, hash : Text }
>

-- Build target
let Target
: Type
= { name : Text
, srcs : Src
, deps : List Dep
, toolchain : Toolchain.Toolchain
, requires : Resource.Resources -- coeffects (what this build needs)
}

-- Convenience constructor
let target
: Target → Target
= λ(t : Target) → t

-- C/C++ library
let cxx-library
: { name : Text
, srcs : List Text
, deps : List Dep
, toolchain : Toolchain.Toolchain
, requires : Resource.Resources
} → Target
= λ(cfg : { name : Text
, srcs : List Text
, deps : List Dep
, toolchain : Toolchain.Toolchain
, requires : Resource.Resources
}) →
{ name = cfg.name
, srcs = Src.Files cfg.srcs
, deps = cfg.deps
, toolchain = cfg.toolchain
, requires = cfg.requires
}

-- C/C++ binary
let cxx-binary
: { name : Text
, srcs : List Text
, deps : List Dep
, toolchain : Toolchain.Toolchain
, requires : Resource.Resources
} → Target
= λ(cfg : { name : Text
, srcs : List Text
, deps : List Dep
, toolchain : Toolchain.Toolchain
, requires : Resource.Resources
}) →
{ name = cfg.name
, srcs = Src.Files cfg.srcs
, deps = cfg.deps
, toolchain = cfg.toolchain
, requires = cfg.requires
}

-- Fetch from URL (content-addressed)
let fetch
: { url : Text, hash : Text } → Src
= Src.Fetch

-- Git source
let git
: { url : Text, rev : Text, hash : Text } → Src
= Src.Git

-- Dependency constructors
let dep =
{ local = Dep.Local
, pkgconfig = Dep.PkgConfig
, external = λ(hash : Text) → λ(name : Text) →
Dep.External { hash, name }
, flake = Dep.Flake
, nixpkgs = λ(pkg : Text) → Dep.Flake "nixpkgs#${pkg}"
, nixpkgsMusl = λ(pkg : Text) → Dep.Flake "nixpkgs#pkgsMusl.${pkg}"
, nixpkgsStatic = λ(pkg : Text) → Dep.Flake "nixpkgs#pkgsStatic.${pkg}"
}

in { Dep
, Src
, Target
, target
, cxx-library
, cxx-binary
, fetch
, git
, dep
, Resource = Resource
, Toolchain = Toolchain
, Triple = Triple
}
Loading