Sprint 1: bounded Kaufman observability and DevSecOps foundation

.github/workflows/ci.yml

@@ -12,6 +12,8 @@ on:
       - "apps/api/**"
       - "apps/web/**"
       - "scripts/ci/**"
+      - "infra/observability/**"
+      - "infra/scripts/monitoring/**"
       - ".env.example"
       - "docs/adr/0015-github-actions-docker-verification.md"
       - "docs/audits/SPRINT_1_WORK_UNIT_1_REPORT.md"
@@ -26,6 +28,8 @@ on:
       - "apps/api/**"
       - "apps/web/**"
       - "scripts/ci/**"
+      - "infra/observability/**"
+      - "infra/scripts/monitoring/**"
       - ".env.example"
       - "docs/adr/0015-github-actions-docker-verification.md"
       - "docs/audits/SPRINT_1_WORK_UNIT_1_REPORT.md"
@@ -47,10 +51,10 @@ jobs:
         working-directory: apps/api
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
 
       - name: Set up Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff
         with:
           go-version-file: apps/api/go.mod
           cache-dependency-path: apps/api/go.mod
@@ -74,18 +78,21 @@ jobs:
         working-directory: apps/api/ai_worker
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
 
       - name: Set up Python
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
         with:
           python-version: "3.11"
 
       - name: Compile worker source
         run: python -m py_compile main.py zip_safety.py
 
       - name: Run worker tests
-        run: python -m unittest discover -s tests
+        # Only lightweight test dependencies; no model download in unit tests.
+        run: |
+          python -m pip install prometheus-client==0.22.1
+          python -m unittest discover -s tests
 
   web:
     name: Web Build
@@ -95,10 +102,10 @@ jobs:
         working-directory: apps/web
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
 
       - name: Set up Node
-        uses: actions/setup-node@v4
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020
         with:
           node-version: "20"
           cache: npm
@@ -110,25 +117,33 @@ jobs:
       - name: Build web app
         run: npm run build
 
+      - name: Web telemetry tests and type checking
+        run: |
+          npm test
+          npm run typecheck
+
       - name: Record remaining web dependency risk
         shell: bash
         run: |
           npm audit --json > npm-audit.json || true
           node -e 'const a=require("./npm-audit.json"); if (!a.metadata) process.exit(1); console.log(a.metadata.vulnerabilities)'
+          python ../../scripts/ci/security-policy.py npm npm-audit.json
 
       - name: Upload web audit evidence
-        uses: actions/upload-artifact@v4
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
         with:
           name: web-audit-${{ github.sha }}
           path: apps/web/npm-audit.json
+          if-no-files-found: error
           retention-days: 14
 
   compose:
     name: Docker Compose Config
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
 
       - name: Validate compose file
         run: docker compose config
@@ -143,19 +158,23 @@ jobs:
       SOURCE_SHA: ${{ github.sha }}
       APP_ENV: test
       MAX_UPLOAD_REQUEST_BYTES: "1048576"
+      COMPOSE_PROFILES: observability
     steps:
       - name: Checkout exact source
-        uses: actions/checkout@v4
+        uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f
 
       - name: Configure ephemeral test signing key
         shell: bash
         run: |
           secret="$(openssl rand -base64 32)"
           echo "::add-mask::$secret"
           echo "JWT_SECRET=$secret" >> "$GITHUB_ENV"
+          grafana_secret="$(openssl rand -hex 24)"
+          echo "::add-mask::$grafana_secret"
+          echo "GRAFANA_ADMIN_PASSWORD=$grafana_secret" >> "$GITHUB_ENV"
 
       - name: Record runner and source attribution
         shell: bash
@@ -199,9 +218,23 @@ jobs:
         shell: bash
         run: |
           set -euo pipefail
+          docker compose run --rm --no-deps --entrypoint promtool prometheus check config /etc/prometheus/prometheus.yml
+          python scripts/ci/verify-observability.py | tee "$EVIDENCE_DIR/observability.txt"
           python scripts/ci/verify-security-boundaries.py | tee "$EVIDENCE_DIR/security-boundaries.txt"
           docker compose exec -T worker python - < scripts/ci/verify-media-boundary.py | tee "$EVIDENCE_DIR/media-boundary.txt"
 
+      - name: Container and Python security inventory plus SBOM
+        run: bash scripts/ci/scan-images.sh
+
+      - name: Retain security and supply-chain evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: image-security-sbom-${{ github.sha }}
+          path: artifacts/security
+          if-no-files-found: error
+          retention-days: 14
+
       - name: Collect final diagnostics
         if: always()
         shell: bash
@@ -222,7 +255,7 @@ jobs:
 
       - name: Upload diagnostic evidence
         if: always()
-        uses: actions/upload-artifact@v4
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
         with:
           name: docker-runtime-${{ github.sha }}
           path: artifacts/docker-runtime

.github/workflows/security.yml

@@ -0,0 +1,44 @@
+name: Security source
+on:
+  pull_request:
+  push:
+    branches: [main]
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  source-security:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.sha || github.sha }}
+      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065
+        with:
+          python-version: '3.11'
+      - name: SAST (blocking rules, Go Python JavaScript TypeScript)
+        run: |
+          mkdir -p artifacts/security
+          git rev-parse HEAD > artifacts/security/source-sha.txt
+          python -m pip install semgrep==1.136.0
+          python scripts/ci/test-sast.py
+          python scripts/ci/test-security-policy.py
+          status=0
+          semgrep scan --metrics=off --config .semgrep.yml --error --json --output artifacts/security/sast.json apps || status=$?
+          python scripts/ci/sanitize-sast.py artifacts/security/sast.json
+          exit "$status"
+      - name: Secret scan (blocking, complete Git history)
+        if: always()
+        run: |
+          mkdir -p artifacts/security
+          docker run --rm -v "$PWD:/repo" zricethezav/gitleaks:v8.28.0 git /repo --redact --report-format json --report-path /repo/artifacts/security/secrets.json
+      - name: Evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02
+        with:
+          name: source-security-${{ github.event.pull_request.head.sha || github.sha }}
+          path: artifacts/security
+          if-no-files-found: error
+          retention-days: 14

.gitignore

@@ -19,6 +19,7 @@ yarn-debug.log*
 yarn-error.log*
 pnpm-debug.log*
 .next/
+*.tsbuildinfo
 out/
 dist/
 coverage/

.semgrep.yml

@@ -0,0 +1,37 @@
+rules:
+  - id: python-dynamic-execution
+    languages: [python]
+    severity: ERROR
+    message: Do not evaluate runtime strings as code.
+    pattern-either:
+      - pattern: eval(...)
+      - pattern: exec(...)
+  - id: python-shell-execution
+    languages: [python]
+    severity: ERROR
+    message: Avoid shell interpretation of runtime commands.
+    pattern: subprocess.$F(..., shell=True, ...)
+  - id: javascript-dynamic-execution
+    languages: [javascript, typescript]
+    severity: ERROR
+    message: Do not execute dynamic JavaScript.
+    pattern-either:
+      - pattern: eval(...)
+      - pattern: new Function(...)
+  - id: go-disabled-tls-validation
+    languages: [go]
+    severity: ERROR
+    message: Certificate verification must remain enabled.
+    pattern: 'tls.Config{..., InsecureSkipVerify: true, ...}'
+  - id: go-shell-command
+    languages: [go]
+    severity: ERROR
+    message: Shell command construction requires security review.
+    pattern-either:
+      - pattern: exec.Command("sh", "-c", ...)
+      - pattern: exec.Command("bash", "-c", ...)
+  - id: javascript-raw-html
+    languages: [javascript, typescript]
+    severity: ERROR
+    message: Raw HTML injection requires a reviewed sanitizer.
+    pattern: '<$TAG dangerouslySetInnerHTML={$X} ... />'

apps/api/ai_worker/Dockerfile

@@ -7,7 +7,7 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/* \
     && pip install --no-cache-dir "setuptools<81" \
     && pip install --no-cache-dir --no-build-isolation -r requirements.txt
-COPY main.py zip_safety.py ./
+COPY main.py zip_safety.py kaufmanlogger.py ./
 RUN python -m py_compile main.py zip_safety.py \
     && python -c "import main; import zip_safety" \
     && ffmpeg -version

apps/api/ai_worker/kaufmanlogger.py

@@ -0,0 +1,127 @@
+"""Allowlisted operational events over standard logging; never serialize payloads."""
+import contextlib
+import contextvars
+import functools
+import json
+import logging
+import os
+import re
+import shutil
+import time
+import uuid
+from datetime import datetime, timezone
+
+from prometheus_client import Counter, Gauge, Histogram, start_http_server
+
+CONTEXT = contextvars.ContextVar('diagnostics', default={})
+EVENTS = frozenset('service.started service.failed job.claimed job.invalid job.failed job.completed processing.started processing.completed processing.stage.started processing.stage.completed processing.stage.failed dependency.degraded dependency.recovered upload.rejected upload.cleanup.failed'.split())
+STAGES = frozenset('validation ffmpeg transcription embedding qdrant persistence classification models processing'.split())
+event_count = Counter('clipsense_events_total', 'Allowlisted operational events', ['event'])
+stage_duration = Histogram('clipsense_stage_duration_seconds', 'Major worker stages', ['stage', 'outcome'], buckets=(.1, 1, 5, 30, 60, 300, 900, 3600))
+job_duration = Histogram('clipsense_job_duration_seconds', 'Claimed job processing duration', buckets=(1, 5, 30, 60, 300, 900, 3600))
+disk_available = Gauge('clipsense_storage_available_bytes', 'Free bytes on processing filesystem')
+disk_total = Gauge('clipsense_storage_total_bytes', 'Capacity of processing filesystem')
+for _event in EVENTS:
+    event_count.labels(_event)
+
+
+def valid_id(value):
+    if not isinstance(value, str) or len(value) != 36:
+        return None
+    try:
+        return value if str(uuid.UUID(value)) == value else None
+    except ValueError:
+        return None
+
+
+def job_context(data):
+    return {key: value for key, value in {
+        'batch_id': valid_id(data.get('id')),
+        'request_id': valid_id(data.get('request_id')),
+        'correlation_id': valid_id(data.get('correlation_id')) or str(uuid.uuid4()),
+    }.items() if value}
+
+
+class KaufmanFormatter(logging.Formatter):
+    def format(self, record):
+        name = record.msg if record.msg in EVENTS else 'service.failed'
+        version = os.getenv('BUILD_SHA', '')
+        environment = os.getenv('APP_ENV', 'production')
+        output = dict(timestamp=datetime.now(timezone.utc).isoformat(), level=record.levelname,
+                      service='worker', environment=environment if environment in ('development', 'test', 'production') else 'production',
+                      version=version if re.fullmatch('[a-f0-9]{40}', version) else 'unknown', event=name)
+        for key, value in CONTEXT.get().items():
+            if key in ('batch_id', 'request_id', 'correlation_id') and valid_id(value):
+                output[key] = value
+        fields = getattr(record, 'safe_fields', {})
+        if fields.get('stage') in STAGES:
+            output['stage'] = fields['stage']
+        if fields.get('dependency') in ('redis', 'database', 'qdrant'):
+            output['dependency'] = fields['dependency']
+        for key in ('duration_ms', 'retry_count'):
+            value = fields.get(key)
+            if type(value) in (int, float) and 0 <= value <= 10**12:
+                output[key] = value
+        # No record.args, exception text/stack, arbitrary extras, or library payloads.
+        return json.dumps(output, allow_nan=False)
+
+
+logger = logging.getLogger('clipsense.kaufman')
+logger.setLevel(logging.INFO)
+logger.propagate = False
+if not logger.handlers:
+    handler = logging.StreamHandler()
+    handler.setFormatter(KaufmanFormatter())
+    logger.addHandler(handler)
+
+
+def event(name, **fields):
+    if name not in EVENTS:
+        return
+    level = logging.ERROR if name.endswith('.failed') else logging.INFO
+    logger.log(level, name, extra={'safe_fields': fields})
+    event_count.labels(name).inc()
+
+
+@contextlib.contextmanager
+def stage(name):
+    if name not in STAGES:
+        raise ValueError('unknown processing stage')
+    start = time.monotonic()
+    outcome = 'failed'
+    event('processing.stage.started', stage=name)
+    try:
+        yield
+        outcome = 'completed'
+    finally:
+        elapsed = time.monotonic() - start
+        stage_duration.labels(name, outcome).observe(elapsed)
+        event('processing.stage.' + outcome, stage=name, duration_ms=int(elapsed * 1000))
+
+
+def measured(name):
+    def decorate(function):
+        @functools.wraps(function)
+        def call(*args, **kwargs):
+            with stage(name):
+                return function(*args, **kwargs)
+        return call
+    return decorate
+
+
+def safety_event(message):
+    # Existing safety helper emits fixed strings; unknown input is never forwarded.
+    if message in ('event=archive_cleanup_failed', 'event=upload_cleanup_failed'):
+        event('upload.cleanup.failed')
+    elif message in ('event=archive_rejected', 'event=upload_validation_failed'):
+        event('upload.rejected')
+
+
+def start_metrics():
+    start_http_server(9092, addr=os.getenv('METRICS_HOST', '127.0.0.1'))
+    def available():
+        return shutil.disk_usage(os.getenv('PROCESS_DIR', '/tmp')).free
+    def total():
+        return shutil.disk_usage(os.getenv('PROCESS_DIR', '/tmp')).total
+    disk_available.set_function(available)
+    disk_total.set_function(total)