feat: DataFog API v2 MVP — detection, enforcement, demo

.github/workflows/harness-docs.yml

@@ -0,0 +1,30 @@
+name: Harness Docs
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+
+jobs:
+  docs_lint:
+    name: Docs Lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Lint harness docs
+        run: |
+          bash scripts/ci/he-docs-lint.sh
+          bash scripts/ci/he-specs-lint.sh
+          bash scripts/ci/he-plans-lint.sh
+          bash scripts/ci/he-spikes-lint.sh
+
+  docs_drift:
+    name: Docs Drift Gate
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Enforce doc updates on relevant changes
+        run: bash scripts/ci/he-docs-drift.sh

.github/workflows/main-cicd.yml

@@ -1,5 +1,4 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
-# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
+# Go-based build + test pipeline for datafog-api
 
 name: Main CICD datafog-api app
 
@@ -14,21 +13,30 @@ permissions:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.11
-      uses: actions/setup-python@v3
+    - name: Set up Go
+      uses: actions/setup-go@v5
       with:
-        python-version: "3.11"
-    - name: Install dependencies
+        go-version: "1.24.13"
+    - name: Run gofmt check
       run: |
-        python -m pip install --upgrade pip
-        pip install -r app/requirements-dev.txt
-    - name: Test with pytest with coverage minimum
+        test -z "$(gofmt -l cmd internal | tee /tmp/gofmt-diff.txt)" || {
+          echo "Run gofmt on:"
+          cat /tmp/gofmt-diff.txt
+          exit 1
+        }
+    - name: Run tests
+      run: go test ./...
+    - name: Lint (go vet)
+      run: go vet ./...
+    - name: Security checks
       run: |
-        cd app && pytest
-    - name: Build the Docker image
-      run: docker build . --file Dockerfile --tag datafog-api:$(date +%s)
+        go install github.com/securego/gosec/v2/cmd/gosec@latest
+        go install golang.org/x/vuln/cmd/govulncheck@latest
+        gosec -severity medium -confidence medium ./...
+        govulncheck ./...
+    - name: Build container image
+      run: docker build . --file Dockerfile --tag datafog-api:$(date +%s)

.gitignore

@@ -28,4 +28,5 @@ db.sqlite3
 
 # ignore local datafog-python
 datafog-python/
-*.coverage
+datafog_receipts.jsonl
+*.coverage

AGENTS.md

@@ -0,0 +1,28 @@
+# AGENTS.md
+
+## Start Here
+
+This file is a map, not an encyclopedia.
+
+The system of record is `docs/`. Keep durable knowledge (specs, plans, logs, decisions, checklists) there and link to it from here.
+
+## Golden Principles
+
+- Prove it works: never claim completion without running the most relevant validation (tests, build, or a small end-to-end check) or explicitly recording why it could not be run.
+- Keep AGENTS.md minimal and stable; detailed procedure belongs in `docs/runbooks/`.
+
+## Source Of Truth (Table Of Contents)
+
+- Workflow contract + artifact rules: `docs/PLANS.md`
+- Specs (intent): `docs/specs/`
+- Spikes (investigation findings): `docs/spikes/`
+- Plans (execution + evidence): `docs/plans/`
+- Runbooks (process checklists): `docs/runbooks/`
+- Generated context (scratchpad/reference): `docs/generated/`
+- Architecture (if present): `ARCHITECTURE.md`
+
+## Workflow (Phases)
+
+intake -> spike (optional) -> plan -> implement -> review -> verify-release -> learn
+
+If this file grows beyond a compact index, move detailed guidance into `docs/` and keep links here.

ARCHITECTURE.md

@@ -0,0 +1,47 @@
+# Architecture
+
+This file is a compact map to answer: "Where do I change code to do X?"
+
+Write only stable facts. Do not include procedures, external links, or volatile implementation details.
+
+Keep this file small:
+
+- Prefer bullets over paragraphs
+- Keep each bullet concise
+- If it grows, move detail to `docs/` and keep only pointers here
+
+## Purpose
+
+2-4 bullets, max 6 lines total:
+
+- System purpose: <what this repo/system does>
+- Primary users/actors: <who uses it>
+- Main runtime pieces: <CLI/API/worker/UI/etc>
+- Primary flows: <highest-value flows>
+
+## Codemap (Where To Change Code)
+
+4-8 bullets plus one flow line, max 14 lines total:
+
+- `path/or/module` -> owns <what>; key types: <TypeA>, <TypeB>
+- `path/or/module` -> owns <what>; key types: <TypeC>
+
+Flow: `<entry>` -> `<layer>` -> `<layer>` -> `<store/service>`
+
+## Invariants (Must Remain True)
+
+3-7 bullets, max 10 lines total:
+
+- `X` must not depend on `Y`.
+- Side effects occur only in `<boundary/module>`.
+- Business rules live in `<layer>` and not in `<layer>`.
+- Security/data boundary: `<what is sensitive and where it may flow>`.
+
+## Details Live Elsewhere
+
+3-6 pointers, max 8 lines total. Use path + short label only:
+
+- `docs/PLANS.md` - workflow and artifact contract
+- `docs/runbooks/` - procedures and checklists
+- `docs/<DOMAIN>.md` - domain-specific guardrails
+- `docs/generated/` - generated context snapshots

Dockerfile

@@ -1,20 +1,28 @@
-FROM ubuntu:22.04
-ENV PYTHONUNBUFFERED=1
-ENV DEBIAN_FRONTEND=noninteractive
+FROM golang:1.22 AS build
 
-EXPOSE 8000
+WORKDIR /workspace
 
-RUN apt-get update && apt-get install -y \
-    vim \
-    git \
-    python3-pip \
-    python3.11 \
-    wget
+COPY go.mod ./
+COPY cmd ./cmd
+COPY internal ./internal
+COPY config ./config
 
-ADD app /root/app
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/datafog-api ./cmd/datafog-api
+RUN mkdir -p /workspace/var/lib/datafog && chmod 0777 /workspace/var/lib/datafog
 
-RUN python3.11 -m pip install -r /root/app/requirements.txt
+FROM gcr.io/distroless/base-debian11
 
+WORKDIR /app
+COPY --from=build /out/datafog-api /usr/local/bin/datafog-api
+COPY --from=build /workspace/config/policy.json /app/config/policy.json
+COPY --from=build /workspace/var/lib/datafog /var/lib/datafog
 
-WORKDIR /root/app
-ENTRYPOINT ["python3.11", "-m", "uvicorn", "--host=0.0.0.0","main:app"]
+ENV DATAFOG_POLICY_PATH=/app/config/policy.json
+ENV DATAFOG_RECEIPT_PATH=/var/lib/datafog/datafog_receipts.jsonl
+ENV DATAFOG_ADDR=:8080
+
+USER 65532:65532
+
+EXPOSE 8080
+
+ENTRYPOINT ["/usr/local/bin/datafog-api"]