diff --git a/.github/workflows/harness-docs.yml b/.github/workflows/harness-docs.yml new file mode 100644 index 0000000..7eb9652 --- /dev/null +++ b/.github/workflows/harness-docs.yml @@ -0,0 +1,30 @@ +name: Harness Docs + +on: + pull_request: + types: [opened, synchronize, reopened] + +jobs: + docs_lint: + name: Docs Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Lint harness docs + run: | + bash scripts/ci/he-docs-lint.sh + bash scripts/ci/he-specs-lint.sh + bash scripts/ci/he-plans-lint.sh + bash scripts/ci/he-spikes-lint.sh + + docs_drift: + name: Docs Drift Gate + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Enforce doc updates on relevant changes + run: bash scripts/ci/he-docs-drift.sh diff --git a/.github/workflows/main-cicd.yml b/.github/workflows/main-cicd.yml index 07143d4..74fc5af 100644 --- a/.github/workflows/main-cicd.yml +++ b/.github/workflows/main-cicd.yml @@ -1,5 +1,4 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python +# Go-based build + test pipeline for datafog-api name: Main CICD datafog-api app @@ -14,21 +13,30 @@ permissions: jobs: build: - runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - name: Set up Python 3.11 - uses: actions/setup-python@v3 + - name: Set up Go + uses: actions/setup-go@v5 with: - python-version: "3.11" - - name: Install dependencies + go-version: "1.24.13" + - name: Run gofmt check run: | - python -m pip install --upgrade pip - pip install -r app/requirements-dev.txt - - name: Test with pytest with coverage minimum + test -z "$(gofmt -l cmd internal | tee /tmp/gofmt-diff.txt)" || { + echo "Run gofmt on:" + cat /tmp/gofmt-diff.txt + exit 1 + } + - name: Run tests + run: go test ./... + - name: Lint (go vet) + run: go vet ./... + - name: Security checks run: | - cd app && pytest - - name: Build the Docker image - run: docker build . --file Dockerfile --tag datafog-api:$(date +%s) \ No newline at end of file + go install github.com/securego/gosec/v2/cmd/gosec@latest + go install golang.org/x/vuln/cmd/govulncheck@latest + gosec -severity medium -confidence medium ./... + govulncheck ./... + - name: Build container image + run: docker build . --file Dockerfile --tag datafog-api:$(date +%s) diff --git a/.gitignore b/.gitignore index 3c797d4..92a8a74 100644 --- a/.gitignore +++ b/.gitignore @@ -28,4 +28,5 @@ db.sqlite3 # ignore local datafog-python datafog-python/ -*.coverage \ No newline at end of file +datafog_receipts.jsonl +*.coverage diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..b83c628 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,28 @@ +# AGENTS.md + +## Start Here + +This file is a map, not an encyclopedia. + +The system of record is `docs/`. Keep durable knowledge (specs, plans, logs, decisions, checklists) there and link to it from here. + +## Golden Principles + +- Prove it works: never claim completion without running the most relevant validation (tests, build, or a small end-to-end check) or explicitly recording why it could not be run. +- Keep AGENTS.md minimal and stable; detailed procedure belongs in `docs/runbooks/`. + +## Source Of Truth (Table Of Contents) + +- Workflow contract + artifact rules: `docs/PLANS.md` +- Specs (intent): `docs/specs/` +- Spikes (investigation findings): `docs/spikes/` +- Plans (execution + evidence): `docs/plans/` +- Runbooks (process checklists): `docs/runbooks/` +- Generated context (scratchpad/reference): `docs/generated/` +- Architecture (if present): `ARCHITECTURE.md` + +## Workflow (Phases) + +intake -> spike (optional) -> plan -> implement -> review -> verify-release -> learn + +If this file grows beyond a compact index, move detailed guidance into `docs/` and keep links here. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..09c1976 --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,47 @@ +# Architecture + +This file is a compact map to answer: "Where do I change code to do X?" + +Write only stable facts. Do not include procedures, external links, or volatile implementation details. + +Keep this file small: + +- Prefer bullets over paragraphs +- Keep each bullet concise +- If it grows, move detail to `docs/` and keep only pointers here + +## Purpose + +2-4 bullets, max 6 lines total: + +- System purpose: +- Primary users/actors: +- Main runtime pieces: +- Primary flows: + +## Codemap (Where To Change Code) + +4-8 bullets plus one flow line, max 14 lines total: + +- `path/or/module` -> owns ; key types: , +- `path/or/module` -> owns ; key types: + +Flow: `` -> `` -> `` -> `` + +## Invariants (Must Remain True) + +3-7 bullets, max 10 lines total: + +- `X` must not depend on `Y`. +- Side effects occur only in ``. +- Business rules live in `` and not in ``. +- Security/data boundary: ``. + +## Details Live Elsewhere + +3-6 pointers, max 8 lines total. Use path + short label only: + +- `docs/PLANS.md` - workflow and artifact contract +- `docs/runbooks/` - procedures and checklists +- `docs/.md` - domain-specific guardrails +- `docs/generated/` - generated context snapshots diff --git a/Dockerfile b/Dockerfile index 6671440..7efdda1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,28 @@ -FROM ubuntu:22.04 -ENV PYTHONUNBUFFERED=1 -ENV DEBIAN_FRONTEND=noninteractive +FROM golang:1.22 AS build -EXPOSE 8000 +WORKDIR /workspace -RUN apt-get update && apt-get install -y \ - vim \ - git \ - python3-pip \ - python3.11 \ - wget +COPY go.mod ./ +COPY cmd ./cmd +COPY internal ./internal +COPY config ./config -ADD app /root/app +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /out/datafog-api ./cmd/datafog-api +RUN mkdir -p /workspace/var/lib/datafog && chmod 0777 /workspace/var/lib/datafog -RUN python3.11 -m pip install -r /root/app/requirements.txt +FROM gcr.io/distroless/base-debian11 +WORKDIR /app +COPY --from=build /out/datafog-api /usr/local/bin/datafog-api +COPY --from=build /workspace/config/policy.json /app/config/policy.json +COPY --from=build /workspace/var/lib/datafog /var/lib/datafog -WORKDIR /root/app -ENTRYPOINT ["python3.11", "-m", "uvicorn", "--host=0.0.0.0","main:app"] \ No newline at end of file +ENV DATAFOG_POLICY_PATH=/app/config/policy.json +ENV DATAFOG_RECEIPT_PATH=/var/lib/datafog/datafog_receipts.jsonl +ENV DATAFOG_ADDR=:8080 + +USER 65532:65532 + +EXPOSE 8080 + +ENTRYPOINT ["/usr/local/bin/datafog-api"] diff --git a/README.md b/README.md index 6c68857..924959c 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,221 @@ +# datafog-api (Go) -# Local Development +This repository implements the MVP `datafog-api` service in Go. -> [!IMPORTANT] -> datafog-api requires Python 3.11+ +## Local development +```sh +go mod download +go test ./... +go run ./cmd/datafog-api +``` + +Default configuration: + +- `DATAFOG_POLICY_PATH`: `config/policy.json` +- `DATAFOG_RECEIPT_PATH`: `datafog_receipts.jsonl` +- `DATAFOG_ADDR`: `:8080` +- `DATAFOG_API_TOKEN`: optional API token for endpoint protection +- `DATAFOG_RATE_LIMIT_RPS`: `0` (disabled, else max requests per second) +- `DATAFOG_READ_TIMEOUT`: `5s` +- `DATAFOG_WRITE_TIMEOUT`: `10s` +- `DATAFOG_READ_HEADER_TIMEOUT`: `2s` +- `DATAFOG_IDLE_TIMEOUT`: `30s` +- `DATAFOG_SHUTDOWN_TIMEOUT`: `10s` + +Durations accept Go duration syntax (for example: `1s`, `500ms`, `2m`). + +## HTTP API + +All examples use `localhost:8080`. + +Canonical contract: + +- See `docs/contracts/datafog-api-contract.md` for endpoint schemas, error codes, and idempotency semantics. +- See `docs/generated/api-schema.md` for registered routes. + +### `GET /health` + +```sh +curl http://localhost:8080/health +``` + +### `GET /v1/policy/version` + +```sh +curl http://localhost:8080/v1/policy/version +``` + +### Idempotency + +The following endpoints accept `idempotency_key`: + +- `/v1/scan` +- `/v1/decide` +- `/v1/transform` +- `/v1/anonymize` + +On repeated requests with the same key: +- identical payload returns the exact same response body and status. +- mismatched payload returns `409` with `code: idempotency_conflict`. + +### `POST /v1/scan` + +```sh +curl -X POST http://localhost:8080/v1/scan \ + -H "Content-Type: application/json" \ + -d '{"text":"email alice@example.com and card 4111111111111111"}' +``` + +### `POST /v1/decide` + +```sh +curl -X POST http://localhost:8080/v1/decide \ + -H "Content-Type: application/json" \ + -d '{"action":{"type":"file.write","resource":"notes.txt"},"text":"email alice@example.com"}' +``` + +### `POST /v1/transform` -## Setup ```sh -python -m venv myenv -source myenv/bin/activate -cd app -pip install -r requirements.txt -uvicorn main:app +curl -X POST http://localhost:8080/v1/transform \ + -H "Content-Type: application/json" \ + -d '{"text":"email alice@example.com", "mode":"mask"}' ``` +### `POST /v1/anonymize` + +```sh +curl -X POST http://localhost:8080/v1/anonymize \ + -H "Content-Type: application/json" \ + -d '{"text":"email alice@example.com", "findings":[{"entity_type":"email","value":"alice@example.com","start":0,"end":17,"confidence":0.99}]}' +``` -# Docker +### `GET /v1/receipts/{id}` -## Build ```sh -docker build -t datafog-api . +curl http://localhost:8080/v1/receipts/ ``` -## Run +## Tests + ```sh -docker run -p 8000:8000 -it datafog-api +go test ./... ``` -> [!TIP] -> Change the first 8000 to a new port if there is a conflict. -## Test +## Deployment + +The service is deployed as a single stateless binary with optional mounted policy and receipt storage. + +### Local/container quick start + +```sh +docker build -t datafog-api:v2 . +docker run --rm -p 8080:8080 \ + -e DATAFOG_API_TOKEN=changeme \ + -e DATAFOG_RATE_LIMIT_RPS=50 \ + -e DATAFOG_RECEIPT_PATH=/var/lib/datafog/datafog_receipts.jsonl \ + -v $(pwd)/config:/app/config:ro \ + -v datafog-receipts:/var/lib/datafog \ + datafog-api:v2 +``` + +### Kubernetes-style production pattern + +Use `/health` for liveness/readiness checks and mount writable storage for receipts. + +## Enforcement shim (runtime gate) + +`datafog-api` is a policy decision service. For runtime enforcement, use the optional shim: + ```sh -curl -X POST http://127.0.0.1:8000/api/annotation/default \ - -H "Content-Type: application/json" \ - -d '{"text": "My name is Peter Parker. I live in Queens, NYC. I work at the Daily Bugle."}' +go build -o datafog-shim ./cmd/datafog-shim + +./datafog-shim shell --policy-url http://localhost:8080 rm -rf /tmp/test +./datafog-shim hooks install --target /usr/bin/git git +DATAFOG_SHIM_POLICY_URL=http://localhost:8080 git status +``` + +The shim supports explicit API mode and wrapper-based PATH interception. + +`datafog-shim` can call policy checks directly for an arbitrary adapter/action (`run`) or install command shims (`hooks install`) that wrap target binaries. + +`datafog-shim` calls `/v1/decide` before side-effect actions and only permits actions that resolve to: + +- `allow` +- `allow_with_redaction` + +Actions that resolve to `transform` or `deny` are blocked until the caller applies an explicit transformation path. + +Supported actions: + +- `shell` (command + args) +- `run [--adapter ] --target ` (adapter name inferred from binary path when omitted) +- `read-file ` +- `write-file ` +- `adapters list` (show built-in adapter families used by shim policy metadata) +- `hooks install ` (PATH interception with generated wrapper) +- `hooks list` +- `hooks uninstall ` + +Decision receipts are returned in stderr for every executed action. + +Managed wrapper scripts are generated in `~/.datafog/shims` by default. To use a wrapper in PATH, add that directory to the front of your `PATH`: + +```sh +export PATH="$HOME/.datafog/shims:$PATH" +``` -{"entities":[{"text":"Peter Parker","start":11,"end":23,"type":"PER"},{"text":"Queens","start":35,"end":41,"type":"LOC"},{"text":"NYC","start":43,"end":46,"type":"LOC"},{"text":"the Daily Bugle","start":58,"end":73,"type":"ORG"}]} +```yaml +apiVersion: apps/v1 +kind: Deployment +metadata: + name: datafog-api +spec: + replicas: 1 + selector: + matchLabels: + app: datafog-api + template: + metadata: + labels: + app: datafog-api + spec: + securityContext: + runAsNonRoot: true + runAsUser: 65532 + runAsGroup: 65532 + fsGroup: 65532 + containers: + - name: datafog-api + image: ghcr.io/datafog/datafog-api:v2 + ports: + - containerPort: 8080 + env: + - name: DATAFOG_ADDR + value: ":8080" + - name: DATAFOG_POLICY_PATH + value: "/app/config/policy.json" + - name: DATAFOG_RECEIPT_PATH + value: "/var/lib/datafog/datafog_receipts.jsonl" + - name: DATAFOG_RATE_LIMIT_RPS + value: "100" + volumeMounts: + - name: policy + mountPath: /app/config + readOnly: true + - name: receipts + mountPath: /var/lib/datafog + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: ["ALL"] + volumes: + - name: policy + configMap: + name: datafog-policy + - name: receipts + persistentVolumeClaim: + claimName: datafog-receipts ``` diff --git a/app/constants.py b/app/constants.py deleted file mode 100644 index c599705..0000000 --- a/app/constants.py +++ /dev/null @@ -1,16 +0,0 @@ -"""Constants.py - to maintain project wide constants""" -from enum import Enum - -# Define a regex pattern to encompass extended ASCII characters -VALID_CHARACTERS_PATTERN = r"^[\x00-\xFF]+$" - -# List of languages codes supported by DataFog -SUPPORTED_LANGUAGES = ["EN"] - -class ResponseKeys(Enum): - """Define API response headers as an enum""" - TITLE = "entities" - PII_TEXT = "text" - START_IDX = "start" - END_IDX = "end" - ENTITY_TYPE = "type" diff --git a/app/custom_exceptions.py b/app/custom_exceptions.py deleted file mode 100644 index f569823..0000000 --- a/app/custom_exceptions.py +++ /dev/null @@ -1,22 +0,0 @@ -"""Collection of custom exceptions""" -from enum import Enum -from fastapi.exceptions import RequestValidationError - -class CustomExceptionTypes(Enum): - """Enumeration of all custom exception types to be update with each addition""" - LANG = "value_error.str.language" - -class LanguageValidationError(RequestValidationError): - """To be raised when an invalid or non supported language is requested""" - def __init__(self, msg: str, loc: list[str] | None = None): - if loc is None: - loc = ["body", "lang"] - self.detail = build_error_detail(loc, CustomExceptionTypes.LANG.value, msg) - super().__init__(self.detail) - -def build_error_detail(loc: list[str], error_type: str, msg: str, ctx: dict | None = None): - """Helper function to build the error body""" - detail = {"loc": loc, "type": error_type, "msg": msg} - if ctx: - detail.update({"ctx": ctx}) - return [detail] diff --git a/app/exception_handler.py b/app/exception_handler.py deleted file mode 100644 index 9d3a368..0000000 --- a/app/exception_handler.py +++ /dev/null @@ -1,19 +0,0 @@ -"""Exception handling routines""" -from fastapi import Request, status -from fastapi.exceptions import RequestValidationError -from fastapi.responses import JSONResponse - - -def exception_processor(request: Request, exc: RequestValidationError): - """Provide the opportunity for custom handling of standard fastapi errors if required""" - for e in exc.errors(): - # switch on e["type"] if more standard fastapi 422 errors need to be altered - # custom exceptions should manage output formatting during creation not here - if e['type'] == 'value_error.str.regex': - e['msg'] = "string contains characters beyond Extended ASCII which are not supported" - e['ctx']['pattern'] = 'Extended ASCII' - - return JSONResponse( - status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, - content={"detail": exc.errors()} - ) diff --git a/app/input_validation.py b/app/input_validation.py deleted file mode 100644 index 97d61e3..0000000 --- a/app/input_validation.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Custom input validation routines""" -from custom_exceptions import LanguageValidationError -from constants import SUPPORTED_LANGUAGES - -def validate_annotate(lang: str): - """Validation of annotate endpoint parameters not built into fastapi""" - #currently only lang needs to be validated outside of standard fastapi checks - validate_language(lang) - -def validate_anonymize(lang: str): - """Validation of anonymize endpoint parameters not built into fastapi""" - #currently only lang needs to be validated outside of standard fastapi checks - validate_language(lang) - -def validate_language(lang: str): - """Check that the input is in the list of languages supported by DataFog""" - if lang not in SUPPORTED_LANGUAGES: - raise LanguageValidationError("Unsupported language request, please try a language listed in the DataFog documentation") diff --git a/app/main.py b/app/main.py deleted file mode 100644 index 09ee031..0000000 --- a/app/main.py +++ /dev/null @@ -1,44 +0,0 @@ -"""API REST endpoints""" -from fastapi import FastAPI, Body, Request -from fastapi.exceptions import RequestValidationError -from datafog import DataFog -from processor import format_pii_for_output, anonymize_pii_for_output -from constants import VALID_CHARACTERS_PATTERN -from exception_handler import exception_processor -from input_validation import validate_annotate, validate_anonymize - -app = FastAPI() -df = DataFog() - -@app.post("/api/annotation/default") -def annotate(text: str = Body(embed=True, - min_length=1, - max_length=1000, - pattern=VALID_CHARACTERS_PATTERN), - lang: str = Body(embed=True, - default="EN")): - """entry point for annotate functionality""" - #Use the custom validation imported above, currently only lang requires custom validation - validate_annotate(lang) - result = df.run_text_pipeline_sync([text]) - output = format_pii_for_output(result) - return output - -@app.post("/api/anonymize/non-reversible") -def anonymize(text: str = Body(embed=True, - min_length=1, - max_length=1000, - pattern=VALID_CHARACTERS_PATTERN), - lang: str = Body(embed=True, default="EN")): - """entry point for anonymize functionality""" - #Use the custom validation imported above, currently only lang requires custom validation - validate_anonymize(lang) - result = df.run_text_pipeline_sync([text]) - output = anonymize_pii_for_output(result) - return output - -@app.exception_handler(RequestValidationError) -async def validation_exception_handler(request: Request, exc: RequestValidationError): - """exception handling hook for input validation failures""" - #offload actual processing to another to keep this uncluttered - return exception_processor(request, exc) diff --git a/app/processor.py b/app/processor.py deleted file mode 100644 index 9c26637..0000000 --- a/app/processor.py +++ /dev/null @@ -1,119 +0,0 @@ -"""Collection of functional hooks that leverage specialized classes""" -from constants import ResponseKeys - -def format_pii_for_output(pii: dict[str, dict]) -> dict: - """Reformat datafog library results to meet API contract""" - sorted_entities = get_entities_from_pii(pii) - return {ResponseKeys.TITLE.value: sorted_entities} - -def get_entities_from_pii(pii: dict[str, dict]) -> list: - """Produce a sorted list of entities from the datafog library results""" - entities = [] #list of entities to output - claimed_start_indices = set() #Set of start indices of PII that have been found - original_text = list(pii.keys())[0] #original text fed to datafog library - dict_of_pii_types = pii[original_text] #dict of PII entities keyed by type - for k,v in dict_of_pii_types.items(): - #loop through each PII entity type and add the found entities to the output list - #must use extend to add each entity from the returned list, append would add the whole list - entities.extend(create_entities(original_text, k, v, claimed_start_indices)) - #sort entities by the start index of the PII in the original text and add to the output dict - return sorted(entities, key=lambda d: d[ResponseKeys.START_IDX.value]) - -def create_entities(original_text: str, pii_type: str, pii_list, seen_indices: set) -> list: - """Create an output list of PII entities from a list of PII of a particular type""" - result = [] - start_index = 0 - for pii in pii_list: - #for each pii in the input list find it in the original text and create an response - # entity to add to the output list - result.append(create_entity(original_text, start_index, pii_type, pii, seen_indices)) - #begin the search for the next PII at the next character after the end of the PII - # just added to the output by updating startIndex - start_index = result[-1][ResponseKeys.END_IDX.value] + 1 - return result - -def create_entity(original_text: str, - start_index: int, - pii_type: str, - pii: str, - seen_indices: set) -> dict: - """Create an output PII entity from a singular datafog library result""" - #TODO: fail gracefully if we cant find the pii in the original text - start, end = find_pii_in_text(original_text, start_index, pii, seen_indices) - result = {ResponseKeys.PII_TEXT.value: pii, - ResponseKeys.START_IDX.value: start, - ResponseKeys.END_IDX.value: end, - ResponseKeys.ENTITY_TYPE.value: pii_type} - return result - -def find_pii_in_text(original_text: str, - start_index: int, - pii: str, - seen_indices: set) -> tuple[int, int]: - """Find pii in the original text and return the start and end index""" - start = None - end = None - #Currently returns the 0-based start index and the end index non-inclusive - while start_index < len(original_text): - start = original_text.find(pii, start_index) - if start == -1: - #unable to find PII, return None - return (None, None) - - end = start + len(pii) - - if start > 0 and original_text[start - 1 : start].isalnum(): - #if the start is not the first char and the char before it is an alpha numeric - # we have found a substring and should continue - start_index = start + 1 - #continue search after start - start = None - end = None - continue - elif end < len(original_text) and original_text[end : end + 1].isalnum(): - #if end is not the last char in the text and the char after it is an alpha numeric - # we have found a substring and should continue - start_index = start + 1 - #continue search after start - start = None - end = None - continue - elif start in seen_indices: - # we have previously found this pii and should continue - start_index = start + 1 - #continue search after start - start = None - end = None - continue - - #Valid PII found, break out of loop - break - - #add start to the set of seen indices - seen_indices.add(start) - return (start, end) - -def anonymize_pii_for_output(pii: dict[str, dict]) -> dict: - """Given datafog library results uses helper functions to anonymize and return the text""" - original_text = list(pii.keys())[0] #original text fed to datafog library - entities = get_entities_from_pii(pii) - anonymized_text = anonymize_pii_in_text(entities, original_text) - response = {ResponseKeys.PII_TEXT.value: anonymized_text, - ResponseKeys.TITLE.value: entities} - return response - -def anonymize_pii_in_text(pii_entities: list, text: str) -> str: - """Anonymize the provided entities in the text""" - offset = 0 #track the changes in length of the text - original_text_length = len(text) - for ent in pii_entities: - place_holder = "[" + ent[ResponseKeys.ENTITY_TYPE.value] + "]" - #calculate the new start and stop indices to account for the updates so far - start = ent[ResponseKeys.START_IDX.value] - offset - stop = ent[ResponseKeys.END_IDX.value] - offset - #substitute into text subtracting offset - text = text[:start] + place_holder + text[stop:] - #update offset to account for the new string - offset = original_text_length - len(text) - - return text diff --git a/app/pytest.ini b/app/pytest.ini deleted file mode 100644 index 2c91776..0000000 --- a/app/pytest.ini +++ /dev/null @@ -1,2 +0,0 @@ -[pytest] -addopts = --cov --cov-report=term --cov-fail-under=70 -v \ No newline at end of file diff --git a/app/requirements-dev.txt b/app/requirements-dev.txt deleted file mode 100644 index f4f8dcc..0000000 --- a/app/requirements-dev.txt +++ /dev/null @@ -1,12 +0,0 @@ --r requirements.txt - -# Development and testing dependencies -just -isort -black -flake8 -tox -pytest -pytest-cov -mypy -autoflake \ No newline at end of file diff --git a/app/requirements.txt b/app/requirements.txt deleted file mode 100644 index 853eddb..0000000 --- a/app/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -fastapi -uvicorn[standard] -numpy -datafog==3.3.0 \ No newline at end of file diff --git a/app/test_custom_exceptions.py b/app/test_custom_exceptions.py deleted file mode 100644 index 83043f7..0000000 --- a/app/test_custom_exceptions.py +++ /dev/null @@ -1,47 +0,0 @@ -"""Unit tests for custom_exceptions.py""" -from custom_exceptions import build_error_detail, LanguageValidationError, CustomExceptionTypes - - -def test_build_error_detail_with_ctx(): - loc = ["body", "text"] - error_type = "test.error" - msg = "test error message" - ctx = {"limit_value": 10} - result = build_error_detail(loc, error_type, msg, ctx) - assert(loc == result[0]["loc"]), "loc not created correctly" - assert(error_type == result[0]["type"]), "error type not created correctly" - assert(msg == result[0]["msg"]), "message not created correctly" - assert(ctx == result[0]["ctx"]), "context not created correctly" - -def test_build_error_detail_no_ctx(): - loc = ["body", "text"] - error_type = "test.error" - msg = "test error message" - result = build_error_detail(loc, error_type, msg) - assert(loc == result[0]["loc"]), "loc not created correctly" - assert(error_type == result[0]["type"]), "error type not created correctly" - assert(msg == result[0]["msg"]), "message not created correctly" - assert("ctx" not in result[0]), "context field should not be assigned" - -def test_template_error_init_with_loc(): - msg = "test error" - loc = ["query"] - test_error = LanguageValidationError(msg, loc) - result = test_error.errors() - assert(LanguageValidationError == type(test_error)), "error type mismatch" - assert(1 == len(result)), "error payload incorrect length" - assert(msg == result[0]["msg"]), "error message not set correctly" - assert(loc == result[0]["loc"]), "loc not set correctly" - assert(CustomExceptionTypes.LANG.value == result[0]["type"]), "error type incorrect" - assert("ctx" not in result[0]), "context field should not be assigned" - -def test_template_error_init_no_loc(): - msg = "test error" - test_error = LanguageValidationError(msg) - result = test_error.errors() - assert(LanguageValidationError == type(test_error)), "error type mismatch" - assert(1 == len(result)), "error payload incorrect length" - assert(msg == result[0]["msg"]), "error message not set correctly" - assert(["body", "lang"] == result[0]["loc"]), "loc not set correctly" - assert(CustomExceptionTypes.LANG.value == result[0]["type"]), "error type incorrect" - assert("ctx" not in result[0]), "context field should not be assigned" diff --git a/app/test_exception_handler.py b/app/test_exception_handler.py deleted file mode 100644 index 7e9108d..0000000 --- a/app/test_exception_handler.py +++ /dev/null @@ -1,33 +0,0 @@ -"""Unit tests for exception_handler.py""" -import json -from fastapi import status -from fastapi.exceptions import RequestValidationError -from exception_handler import exception_processor -from custom_exceptions import LanguageValidationError - -REGEX_MSG = "string contains characters beyond Extended ASCII which are not supported" -REGEX_PATTERN = "Extended ASCII" - -def test_exception_processor_status_code(): - exc = RequestValidationError([{"loc": ["body", "text"], - "type": "value_error.str", - "msg": "test error"}]) - result = exception_processor(None, exc) - assert(status.HTTP_422_UNPROCESSABLE_ENTITY == result.status_code), "incorrect status code" - -def test_exception_processor_regex_override(): - exc = RequestValidationError([{"loc": ["body", "text"], - "type": "value_error.str.regex", - "msg": "test error", - "ctx": {"pattern": "^[\x00-\xFF]+$"}}]) - result = exception_processor(None, exc) - msg = json.loads(result.body)["detail"][0]["msg"] - pattern = json.loads(result.body)["detail"][0]["ctx"]["pattern"] - assert(REGEX_MSG == msg), "regex error message wasn't overriden" - assert(REGEX_PATTERN == pattern), "regex ctx-pattern wasn't overriden" - -def test_exception_processor_non_regex_type_passthrough(): - exc = LanguageValidationError("test error message") - result = exception_processor(None, exc) - msg = json.loads(result.body)["detail"][0]["msg"] - assert("test error message" == msg), "error message overriden incorrectly" diff --git a/app/test_input_validation.py b/app/test_input_validation.py deleted file mode 100644 index b2162c1..0000000 --- a/app/test_input_validation.py +++ /dev/null @@ -1,18 +0,0 @@ -"""Unit tests for input_validation.py""" -import pytest -from input_validation import validate_language -from custom_exceptions import LanguageValidationError - - -def test_validate_language_supported(): - lang = "EN" - try: - validate_language(lang) - except LanguageValidationError as e: - pytest.fail(f"validate_language raised {e} unexpectedly when provided {lang}") - -def test_validate_language_unsupported(): - lang = "FR" - with pytest.raises(LanguageValidationError) as excinfo: - validate_language(lang) - assert ("Unsupported language request, please try a language listed in the DataFog documentation" == str(excinfo.value)) diff --git a/app/test_processor.py b/app/test_processor.py deleted file mode 100644 index 5e2c837..0000000 --- a/app/test_processor.py +++ /dev/null @@ -1,65 +0,0 @@ -"""Unit tests for processor.py""" -from processor import format_pii_for_output, find_pii_in_text, anonymize_pii_for_output - - -def test_format_pii_for_output(): - data = { - "My name is Peter Parker. I live in Queens, NYC. I work at the Daily Bugle.": { - "DATE_TIME": [], - "LOC": ["Queens", "NYC"], - "NRP": [], - "ORG": ["the Daily Bugle"], - "PER": ["Peter Parker"], - } - } - res = format_pii_for_output(data) - assert ( - res["entities"][0]["text"] == "Peter Parker" - ), "The first entity's text is not 'Peter Parker'." - - -def test_find_pii_in_text_duplicate_pii_of_different_type(): - original_text = "Kaladin works for Apple on the main Apple campus" - start_index = 0 - pii = "Apple" - seen = set() - seen.add(18) - result = find_pii_in_text(original_text, start_index, pii, seen) - assert(result[0] == 36), "found pii does not start at correct location" - assert(result[1] == 41), "found pii does not end at correct location" - - -def test_find_pii_in_text_prefix_to_ignore(): - original_text = "the samovar belongs to sam" - start_index = 0 - pii = "sam" - seen = set() - result = find_pii_in_text(original_text, start_index, pii, seen) - assert(result[0] == 23), "found pii does not start at correct location" - assert(result[1] == 26), "found pii does not end at correct location" - - -def test_find_pii_in_text_suffix_to_ignore(): - original_text = "He stopped ed from jumping" - start_index = 0 - pii = "ed" - seen = set() - result = find_pii_in_text(original_text, start_index, pii, seen) - assert(result[0] == 11), "found pii does not start at correct location" - assert(result[1] == 13), "found pii does not end at correct location" - - -def test_anonymize_pii(): - data = { - "My name is Peter Parker. I live in Queens, NYC. I work at the Daily Bugle.": { - "DATE_TIME": [], - "LOC": ["Queens", "NYC"], - "NRP": [], - "ORG": ["the Daily Bugle"], - "PER": ["Peter Parker"], - } - } - out = anonymize_pii_for_output(data) - assert( - out["text"] == "My name is [PER]. I live in [LOC], [LOC]. I work at [ORG]." - ), "text anonymized incorrectly" diff --git a/cmd/datafog-api/main.go b/cmd/datafog-api/main.go new file mode 100644 index 0000000..0e43fed --- /dev/null +++ b/cmd/datafog-api/main.go @@ -0,0 +1,149 @@ +package main + +import ( + "context" + "errors" + "log" + "net/http" + "os" + "os/signal" + "strconv" + "strings" + "syscall" + "time" + + "github.com/datafog/datafog-api/internal/policy" + "github.com/datafog/datafog-api/internal/receipts" + "github.com/datafog/datafog-api/internal/server" + "github.com/datafog/datafog-api/internal/shim" +) + +func main() { + policyPath := getenv("DATAFOG_POLICY_PATH", "config/policy.json") + receiptPath := getenv("DATAFOG_RECEIPT_PATH", "datafog_receipts.jsonl") + apiToken := getenv("DATAFOG_API_TOKEN", "") + addr := getenv("DATAFOG_ADDR", ":8080") + rateLimitRPS := getenvInt("DATAFOG_RATE_LIMIT_RPS", 0) + shutdownTimeout := getenvDuration("DATAFOG_SHUTDOWN_TIMEOUT", 10*time.Second) + enableDemo := getenv("DATAFOG_ENABLE_DEMO", "") != "" || hasFlag("--enable-demo") + eventsPath := getenv("DATAFOG_EVENTS_PATH", "datafog_events.ndjson") + + policyData, err := policy.LoadPolicyFromFile(policyPath) + if err != nil { + log.Fatalf("load policy: %v", err) + } + + store, err := receipts.NewReceiptStore(receiptPath) + if err != nil { + log.Fatalf("init receipts: %v", err) + } + + eventSink := shim.NewNDJSONDecisionEventSink(eventsPath) + + h := server.New(policyData, store, log.Default(), apiToken, rateLimitRPS) + h.SetEventReader(eventSink) + + var handler http.Handler + if enableDemo { + // Create a shim gate backed by a local HTTP decision client + client := shim.NewHTTPDecisionClient("http://127.0.0.1"+addr, apiToken) + gate := shim.NewGate(client, shim.WithEventSink(eventSink)) + + demoHTMLPath := getenv("DATAFOG_DEMO_HTML", "docs/demo.html") + demo, err := server.NewDemoHandler(gate, h, demoHTMLPath) + if err != nil { + log.Fatalf("init demo: %v", err) + } + defer demo.Cleanup() + + handler = h.HandlerWithDemo(demo) + log.Printf("demo mode enabled — /demo/exec, /demo/write-file, /demo/read-file available") + } else { + handler = h.Handler() + } + + srv := &http.Server{ + Addr: addr, + Handler: handler, + ReadTimeout: getenvDuration("DATAFOG_READ_TIMEOUT", 5*time.Second), + ReadHeaderTimeout: getenvDuration("DATAFOG_READ_HEADER_TIMEOUT", 2*time.Second), + WriteTimeout: getenvDuration("DATAFOG_WRITE_TIMEOUT", 10*time.Second), + IdleTimeout: getenvDuration("DATAFOG_IDLE_TIMEOUT", 30*time.Second), + MaxHeaderBytes: 1 << 20, // 1 MiB + ErrorLog: log.Default(), + } + + log.Printf("datafog-api listening on %s", addr) + done := make(chan error, 1) + go func() { + done <- srv.ListenAndServe() + }() + + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(sig) + + select { + case err := <-done: + if !errors.Is(err, http.ErrServerClosed) { + log.Fatalf("server failed: %v", err) + } + case <-sig: + log.Printf("shutdown signal received") + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + + if err := srv.Shutdown(ctx); err != nil { + log.Printf("graceful shutdown failed: %v", err) + if closeErr := srv.Close(); closeErr != nil && !errors.Is(closeErr, http.ErrServerClosed) { + log.Printf("forced close failed: %v", closeErr) + } + } + if err := <-done; err != nil && !errors.Is(err, http.ErrServerClosed) { + log.Printf("server stopped with error: %v", err) + } + } +} + +func getenv(key, fallback string) string { + if value := os.Getenv(key); value != "" { + return value + } + return fallback +} + +func getenvDuration(key string, fallback time.Duration) time.Duration { + value := os.Getenv(key) + if value == "" { + return fallback + } + parsed, err := time.ParseDuration(value) + if err != nil || parsed <= 0 { + log.Printf("invalid duration for %s=%q, using fallback %s", key, value, fallback) + return fallback + } + return parsed +} + +func getenvInt(key string, fallback int) int { + value := strings.TrimSpace(os.Getenv(key)) + if value == "" { + return fallback + } + + parsed, err := strconv.Atoi(value) + if err != nil || parsed < 0 { + log.Printf("invalid integer for %s=%q, using fallback %d", key, value, fallback) + return fallback + } + return parsed +} + +func hasFlag(flag string) bool { + for _, arg := range os.Args[1:] { + if arg == flag { + return true + } + } + return false +} diff --git a/cmd/datafog-api/main_test.go b/cmd/datafog-api/main_test.go new file mode 100644 index 0000000..b3c26ab --- /dev/null +++ b/cmd/datafog-api/main_test.go @@ -0,0 +1,98 @@ +package main + +import ( + "testing" + "time" +) + +func TestGetenvDuration(t *testing.T) { + t.Run("fallback_when_missing", func(t *testing.T) { + t.Setenv("DATAFOG_READ_TIMEOUT", "") + got := getenvDuration("DATAFOG_READ_TIMEOUT", 5*time.Second) + if got != 5*time.Second { + t.Fatalf("expected fallback duration, got %s", got) + } + }) + + t.Run("parse_valid_duration", func(t *testing.T) { + t.Setenv("DATAFOG_READ_TIMEOUT", "7s") + got := getenvDuration("DATAFOG_READ_TIMEOUT", 5*time.Second) + if got != 7*time.Second { + t.Fatalf("expected 7s, got %s", got) + } + }) + + t.Run("fallback_on_invalid_duration", func(t *testing.T) { + t.Setenv("DATAFOG_READ_TIMEOUT", "bad") + got := getenvDuration("DATAFOG_READ_TIMEOUT", 5*time.Second) + if got != 5*time.Second { + t.Fatalf("expected fallback duration, got %s", got) + } + }) + + t.Run("fallback_on_negative_duration", func(t *testing.T) { + t.Setenv("DATAFOG_READ_TIMEOUT", "-1s") + got := getenvDuration("DATAFOG_READ_TIMEOUT", 5*time.Second) + if got != 5*time.Second { + t.Fatalf("expected fallback duration, got %s", got) + } + }) + + t.Run("parse_subsecond", func(t *testing.T) { + t.Setenv("DATAFOG_READ_TIMEOUT", "250ms") + got := getenvDuration("DATAFOG_READ_TIMEOUT", 5*time.Second) + if got != 250*time.Millisecond { + t.Fatalf("expected 250ms, got %s", got) + } + }) + + t.Run("parse_shutdown_timeout", func(t *testing.T) { + t.Setenv("DATAFOG_SHUTDOWN_TIMEOUT", "2s") + got := getenvDuration("DATAFOG_SHUTDOWN_TIMEOUT", 10*time.Second) + if got != 2*time.Second { + t.Fatalf("expected 2s, got %s", got) + } + }) +} + +func TestGetenvInt(t *testing.T) { + t.Run("fallback_when_missing", func(t *testing.T) { + t.Setenv("DATAFOG_RATE_LIMIT_RPS", "") + got := getenvInt("DATAFOG_RATE_LIMIT_RPS", 0) + if got != 0 { + t.Fatalf("expected fallback int, got %d", got) + } + }) + + t.Run("parse_valid_int", func(t *testing.T) { + t.Setenv("DATAFOG_RATE_LIMIT_RPS", "15") + got := getenvInt("DATAFOG_RATE_LIMIT_RPS", 0) + if got != 15 { + t.Fatalf("expected 15, got %d", got) + } + }) + + t.Run("fallback_on_invalid_int", func(t *testing.T) { + t.Setenv("DATAFOG_RATE_LIMIT_RPS", "bad") + got := getenvInt("DATAFOG_RATE_LIMIT_RPS", 3) + if got != 3 { + t.Fatalf("expected fallback int, got %d", got) + } + }) + + t.Run("fallback_on_negative_int", func(t *testing.T) { + t.Setenv("DATAFOG_RATE_LIMIT_RPS", "-1") + got := getenvInt("DATAFOG_RATE_LIMIT_RPS", 3) + if got != 3 { + t.Fatalf("expected fallback int, got %d", got) + } + }) + + t.Run("parse_zero", func(t *testing.T) { + t.Setenv("DATAFOG_RATE_LIMIT_RPS", "0") + got := getenvInt("DATAFOG_RATE_LIMIT_RPS", 3) + if got != 0 { + t.Fatalf("expected 0, got %d", got) + } + }) +} diff --git a/cmd/datafog-shim/adapters.go b/cmd/datafog-shim/adapters.go new file mode 100644 index 0000000..80832e1 --- /dev/null +++ b/cmd/datafog-shim/adapters.go @@ -0,0 +1,23 @@ +package main + +import "github.com/datafog/datafog-api/internal/adapters" + +type adapterSpec = adapters.Spec + +var defaultAdapters = adapters.DefaultAdapters + +func resolveAdapter(raw string, command string) string { + return adapters.Resolve(raw, command) +} + +func canonicalAdapter(value string) (string, bool) { + return adapters.Canonical(value) +} + +func normalizeAdapter(raw string) string { + return adapters.Normalize(raw) +} + +func knownAdapters() []adapterSpec { + return adapters.KnownAdapters() +} diff --git a/cmd/datafog-shim/main.go b/cmd/datafog-shim/main.go new file mode 100644 index 0000000..e833867 --- /dev/null +++ b/cmd/datafog-shim/main.go @@ -0,0 +1,668 @@ +package main + +import ( + "bufio" + "context" + "flag" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strings" + + "github.com/datafog/datafog-api/internal/shim" +) + +const ( + defaultPolicyURL = "http://localhost:8080" + shimMarker = "# datafog-shim-wrapper" + shimMetaPrefix = "# DATAFOG_SHIM_" +) + +type shimRuntimeConfig struct { + policyURL string + apiToken string + mode string + eventSink string + shimDir string + sensitive bool +} + +type managedShimMetadata struct { + Command string + Adapter string + Target string + Mode string + PolicyURL string + EventSink string +} + +func main() { + if err := run(os.Args[1:]); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run(argv []string) error { + flags := flag.NewFlagSet("datafog-shim", flag.ContinueOnError) + policyURL := flags.String("policy-url", "", "base URL for datafog API (for example http://localhost:8080)") + apiToken := flags.String("api-token", "", "API token for policy decisions") + mode := flags.String("mode", "", "enforcement mode: enforced|observe") + eventSink := flags.String("event-sink", "", "path for NDJSON decision event sink") + shimDir := flags.String("shim-dir", "", "directory for installed adapter shims") + sensitive := flags.Bool("sensitive", false, "mark shimmed actions as sensitive") + if err := flags.Parse(argv); err != nil { + return err + } + + cfg, err := resolveRuntimeConfig(shimRuntimeConfig{ + policyURL: *policyURL, + apiToken: *apiToken, + mode: *mode, + eventSink: *eventSink, + shimDir: *shimDir, + sensitive: *sensitive, + }) + if err != nil { + return err + } + + args := flags.Args() + if len(args) == 0 { + return fmt.Errorf("missing command: adapters|hooks|shell|run|read-file|write-file\n\n%s", usage()) + } + + ctx := context.Background() + cmd := args[0] + switch cmd { + case "shell": + return runShell(ctx, cfg, args[1:]) + case "read-file": + return runReadFile(ctx, cfg, args[1:]) + case "write-file": + return runWriteFile(ctx, cfg, args[1:]) + case "run": + return runCommandAdapter(ctx, cfg, args[1:]) + case "hooks": + return runHooks(ctx, cfg, args[1:]) + case "adapters": + return runAdapters(ctx, args[1:]) + default: + return fmt.Errorf("unknown command %q\n\n%s", cmd, usage()) + } +} + +func resolveRuntimeConfig(input shimRuntimeConfig) (shimRuntimeConfig, error) { + cfg := shimRuntimeConfig{ + policyURL: coalesce(input.policyURL, os.Getenv("DATAFOG_SHIM_POLICY_URL"), defaultPolicyURL), + apiToken: coalesce(input.apiToken, os.Getenv("DATAFOG_SHIM_API_TOKEN")), + mode: coalesce(input.mode, os.Getenv("DATAFOG_SHIM_MODE"), string(shim.ModeEnforced)), + eventSink: coalesce(input.eventSink, os.Getenv("DATAFOG_SHIM_EVENT_SINK")), + shimDir: coalesce(input.shimDir, os.Getenv("DATAFOG_SHIM_DIR"), defaultShimDir()), + sensitive: input.sensitive, + } + + parsedMode, err := parseMode(cfg.mode) + if err != nil { + return shimRuntimeConfig{}, err + } + cfg.mode = string(parsedMode) + return cfg, nil +} + +func parseMode(raw string) (shim.EnforcementMode, error) { + switch strings.ToLower(strings.TrimSpace(raw)) { + case "", string(shim.ModeEnforced): + return shim.ModeEnforced, nil + case string(shim.ModeObserve): + return shim.ModeObserve, nil + default: + return "", fmt.Errorf("invalid mode %q (expected enforced or observe)", raw) + } +} + +func coalesce(values ...string) string { + for _, value := range values { + if strings.TrimSpace(value) != "" { + return strings.TrimSpace(value) + } + } + return "" +} + +func newGate(cfg shimRuntimeConfig) *shim.Gate { + client := shim.NewHTTPDecisionClient(cfg.policyURL, cfg.apiToken) + mode := shim.ModeEnforced + if m, err := parseMode(cfg.mode); err == nil { + mode = m + } + opts := []shim.GateOption{ + shim.WithMode(mode), + } + if strings.TrimSpace(cfg.eventSink) != "" { + opts = append(opts, shim.WithEventSink(shim.NewNDJSONDecisionEventSink(cfg.eventSink))) + } + return shim.NewGate(client, opts...) +} + +func runShell(ctx context.Context, cfg shimRuntimeConfig, args []string) error { + if len(args) < 1 { + return fmt.Errorf("shell command is required") + } + command := args[0] + shellArgs := args[1:] + gate := newGate(cfg) + decision, output, err := gate.ExecuteShell(ctx, command, shellArgs, "", nil, cfg.sensitive) + if err != nil { + return err + } + if len(output) > 0 { + if _, writeErr := os.Stdout.Write(output); writeErr != nil { + return writeErr + } + } + if decision.ReceiptID != "" { + fmt.Fprintf(os.Stderr, "receipt=%s decision=%s\n", decision.ReceiptID, decision.Decision) + } + return nil +} + +func runReadFile(ctx context.Context, cfg shimRuntimeConfig, args []string) error { + if len(args) < 1 { + return fmt.Errorf("read-file path is required") + } + path := args[0] + gate := newGate(cfg) + decision, data, err := gate.ReadFile(ctx, path, "", nil, cfg.sensitive) + if err != nil { + return err + } + if _, writeErr := os.Stdout.Write(data); writeErr != nil { + return writeErr + } + if !strings.HasSuffix(string(data), "\n") { + if _, writeErr := os.Stdout.Write([]byte("\n")); writeErr != nil { + return writeErr + } + } + if decision.ReceiptID != "" { + fmt.Fprintf(os.Stderr, "receipt=%s decision=%s\n", decision.ReceiptID, decision.Decision) + } + return nil +} + +func runWriteFile(ctx context.Context, cfg shimRuntimeConfig, args []string) error { + if len(args) < 2 { + return fmt.Errorf("write-file requires ") + } + path := args[0] + content := strings.Join(args[1:], " ") + gate := newGate(cfg) + decision, err := gate.WriteFile(ctx, path, []byte(content), 0o600, "", nil, cfg.sensitive) + if err != nil { + return err + } + fmt.Fprintf(os.Stderr, "wrote %d bytes to %s receipt=%s decision=%s\n", len(content), path, decision.ReceiptID, decision.Decision) + return nil +} + +func runCommandAdapter(ctx context.Context, cfg shimRuntimeConfig, args []string) error { + flags := flag.NewFlagSet("run", flag.ContinueOnError) + adapter := flags.String("adapter", "", "tool adapter name") + target := flags.String("target", "", "binary or command to execute") + overrideMode := flags.String("mode", "", "enforcement mode: enforced|observe") + policyURL := flags.String("policy-url", "", "base URL for datafog API (for example http://localhost:8080)") + apiToken := flags.String("api-token", "", "API token for policy decisions") + eventSink := flags.String("event-sink", "", "path for NDJSON decision event sink") + sensitive := flags.Bool("sensitive", false, "mark this action as sensitive") + if err := flags.Parse(args); err != nil { + return err + } + + var err error + cfg, err = resolveRuntimeConfig(shimRuntimeConfig{ + policyURL: coalesce(*policyURL, cfg.policyURL), + apiToken: coalesce(*apiToken, cfg.apiToken), + mode: coalesce(*overrideMode, cfg.mode), + eventSink: coalesce(*eventSink, cfg.eventSink), + shimDir: cfg.shimDir, + sensitive: *sensitive || cfg.sensitive, + }) + if err != nil { + return err + } + + runArgs := flags.Args() + targetPath := strings.TrimSpace(*target) + if targetPath == "" { + if len(runArgs) == 0 { + return fmt.Errorf("run requires --target or a command") + } + targetPath = runArgs[0] + runArgs = runArgs[1:] + } + if targetPath == "" { + return fmt.Errorf("run target is required") + } + + adapterName := resolveAdapter(*adapter, targetPath) + if adapterName == "" { + return fmt.Errorf("run requires --adapter or an identifiable target command") + } + + gate := newGate(cfg) + decision, output, err := gate.ExecuteCommand(ctx, adapterName, targetPath, runArgs, "", nil, cfg.sensitive) + if err != nil { + return err + } + if len(output) > 0 { + if _, writeErr := os.Stdout.Write(output); writeErr != nil { + return writeErr + } + } + if decision.ReceiptID != "" { + fmt.Fprintf(os.Stderr, "receipt=%s decision=%s\n", decision.ReceiptID, decision.Decision) + } + return nil +} + +func runHooks(_ context.Context, cfg shimRuntimeConfig, args []string) error { + if len(args) == 0 { + return fmt.Errorf("missing hooks subcommand: install|list|uninstall\n\n%s", usage()) + } + switch args[0] { + case "install": + return runHooksInstall(cfg, args[1:]) + case "list": + return runHooksList(cfg, args[1:]) + case "uninstall": + return runHooksUninstall(cfg, args[1:]) + default: + return fmt.Errorf("unknown hooks subcommand %q\n\n%s", args[0], usage()) + } +} + +func runHooksInstall(cfg shimRuntimeConfig, argv []string) error { + flags := flag.NewFlagSet("hooks install", flag.ContinueOnError) + adapter := flags.String("adapter", "", "adapter name to report in policy") + target := flags.String("target", "", "binary path to wrap (resolved from command if omitted)") + force := flags.Bool("force", false, "overwrite unmanaged shim at same name") + overrideMode := flags.String("mode", "", "override enforcement mode for this shim") + overridePolicyURL := flags.String("policy-url", "", "override policy URL for this shim") + overrideEventSink := flags.String("event-sink", "", "override event sink path for this shim") + shimDir := flags.String("shim-dir", "", "directory for generated shim") + if err := flags.Parse(argv); err != nil { + return err + } + + args := flags.Args() + if len(args) != 1 { + return fmt.Errorf("hooks install expects one command name") + } + + installCfg := cfg + installCfg.mode = coalesce(*overrideMode, cfg.mode) + installCfg.policyURL = coalesce(*overridePolicyURL, cfg.policyURL) + installCfg.eventSink = coalesce(*overrideEventSink, cfg.eventSink) + if *shimDir != "" { + installCfg.shimDir = *shimDir + } + installCfg.shimDir = filepath.Clean(installCfg.shimDir) + if installCfg.shimDir == "" { + return fmt.Errorf("shim directory is required") + } + + command := strings.TrimSpace(args[0]) + if command == "" { + return fmt.Errorf("command name is required") + } + adapterName := resolveAdapter(*adapter, command) + if adapterName == "" { + return fmt.Errorf("unable to infer adapter name") + } + + targetPath := strings.TrimSpace(*target) + if targetPath == "" { + targetPath = command + } + resolvedTarget, err := resolveTargetBinary(targetPath) + if err != nil { + return err + } + + shimBinary, err := os.Executable() + if err != nil { + return fmt.Errorf("unable to locate datafog-shim executable: %w", err) + } + shimBinary, err = filepath.Abs(shimBinary) + if err != nil { + return fmt.Errorf("unable to resolve shim binary path: %w", err) + } + + shimPath, err := installShimScript(shimBinary, installCfg, command, adapterName, resolvedTarget, *force) + if err != nil { + return err + } + fmt.Printf("installed shim for %s at %s\n", command, shimPath) + return nil +} + +func runHooksList(cfg shimRuntimeConfig, argv []string) error { + flags := flag.NewFlagSet("hooks list", flag.ContinueOnError) + shimDir := flags.String("shim-dir", "", "directory for generated shims") + if err := flags.Parse(argv); err != nil { + return err + } + if *shimDir != "" { + cfg.shimDir = *shimDir + } + cfg.shimDir = filepath.Clean(cfg.shimDir) + if cfg.shimDir == "" { + return fmt.Errorf("shim directory is required") + } + + shims, err := listManagedShims(cfg.shimDir) + if err != nil { + return err + } + if len(shims) == 0 { + fmt.Printf("no managed shims found in %s\n", cfg.shimDir) + return nil + } + for _, m := range shims { + fmt.Printf("%s -> target=%s adapter=%s mode=%s policy=%s\n", m.Command, m.Target, m.Adapter, m.Mode, m.PolicyURL) + } + return nil +} + +func runAdapters(_ context.Context, args []string) error { + if len(args) != 1 || args[0] != "list" { + return fmt.Errorf("adapters command currently supports: list") + } + + for _, adapter := range knownAdapters() { + fmt.Printf("%s\n", adapter.Canonical) + fmt.Printf(" aliases: %s\n", strings.Join(adapter.Aliases, ", ")) + fmt.Printf(" description: %s\n\n", adapter.Description) + } + return nil +} + +func runHooksUninstall(cfg shimRuntimeConfig, argv []string) error { + flags := flag.NewFlagSet("hooks uninstall", flag.ContinueOnError) + shimDir := flags.String("shim-dir", "", "directory for generated shims") + force := flags.Bool("force", false, "remove unmanaged file if name matches command") + if err := flags.Parse(argv); err != nil { + return err + } + if *shimDir != "" { + cfg.shimDir = *shimDir + } + cfg.shimDir = filepath.Clean(cfg.shimDir) + args := flags.Args() + if len(args) != 1 { + return fmt.Errorf("hooks uninstall expects one command name") + } + command := strings.TrimSpace(args[0]) + if command == "" { + return fmt.Errorf("command name is required") + } + shimPath := shimScriptPath(cfg.shimDir, command) + _, managed, err := readShimMetadata(shimPath) + if err != nil { + return err + } + if !managed && !*force { + return fmt.Errorf("file %s is not a managed shim; use --force to remove", shimPath) + } + return os.Remove(shimPath) +} + +func defaultShimDir() string { + if override := strings.TrimSpace(os.Getenv("DATAFOG_SHIM_DIR")); override != "" { + return override + } + home := os.Getenv("HOME") + if home == "" { + home = os.Getenv("USERPROFILE") + } + if home == "" { + return filepath.Join(os.TempDir(), "datafog-shims") + } + return filepath.Join(home, ".datafog", "shims") +} + +func resolveTargetBinary(raw string) (string, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", fmt.Errorf("target binary is required") + } + if strings.ContainsRune(raw, filepath.Separator) || strings.HasPrefix(raw, ".") { + abs, err := filepath.Abs(raw) + if err != nil { + return "", fmt.Errorf("resolve target path %q: %w", raw, err) + } + info, err := os.Stat(abs) + if err != nil { + return "", fmt.Errorf("target binary not found: %w", err) + } + if info.IsDir() { + return "", fmt.Errorf("target binary cannot be a directory: %s", abs) + } + return abs, nil + } + if runtime.GOOS == "windows" && !strings.ContainsRune(raw, filepath.Separator) && !strings.HasSuffix(strings.ToLower(raw), ".exe") { + raw = raw + ".exe" + } + bin, err := exec.LookPath(raw) + if err != nil { + return "", fmt.Errorf("target binary not found %q: %w", raw, err) + } + return filepath.Clean(bin), nil +} + +func installShimScript(shimBinary string, cfg shimRuntimeConfig, command, adapter, target string, force bool) (string, error) { + adapter = resolveAdapter(adapter, command) + cfg.mode = coalesce(cfg.mode, string(shim.ModeEnforced)) + cfg.shimDir = coalesce(cfg.shimDir, defaultShimDir()) + shimPath := shimScriptPath(cfg.shimDir, command) + + mode, err := parseMode(cfg.mode) + if err != nil { + return "", err + } + + if mode == "" { + mode = shim.ModeEnforced + } + + if err := os.MkdirAll(cfg.shimDir, 0o755); err != nil { + return "", fmt.Errorf("create shim directory %q: %w", cfg.shimDir, err) + } + + metadata, managed, err := readShimMetadata(shimPath) + if err != nil && !os.IsNotExist(err) { + return "", err + } + if err == nil && !managed && !force { + return "", fmt.Errorf("cannot overwrite unmanaged file %s; use --force", shimPath) + } + if metadata.Command == command && metadata.Adapter == adapter && metadata.Target == target && metadata.Mode == string(mode) { + // idempotent overwrite allowed + } + + content := buildShimScript( + shimBinary, + command, + adapter, + target, + string(mode), + cfg.policyURL, + cfg.eventSink, + ) + + if err := os.WriteFile(shimPath, []byte(content), 0o755); err != nil { + return "", fmt.Errorf("write shim %q: %w", shimPath, err) + } + return shimPath, nil +} + +func listManagedShims(dir string) ([]managedShimMetadata, error) { + dir = filepath.Clean(dir) + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + + managed := make([]managedShimMetadata, 0, len(entries)) + for _, entry := range entries { + if entry.IsDir() { + continue + } + meta, isManaged, err := readShimMetadata(filepath.Join(dir, entry.Name())) + if err != nil { + return nil, err + } + if !isManaged { + continue + } + if meta.Command == "" { + meta.Command = entry.Name() + } + managed = append(managed, meta) + } + + sort.SliceStable(managed, func(i, j int) bool { + return managed[i].Command < managed[j].Command + }) + return managed, nil +} + +func readShimMetadata(path string) (managedShimMetadata, bool, error) { + var meta managedShimMetadata + fd, err := os.Open(path) + if err != nil { + return meta, false, err + } + defer fd.Close() + + scanner := bufio.NewScanner(fd) + isManaged := false + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + if line == shimMarker { + isManaged = true + continue + } + if !strings.HasPrefix(line, shimMetaPrefix) { + continue + } + line = strings.TrimSpace(strings.TrimPrefix(line, shimMetaPrefix)) + parts := strings.SplitN(line, "=", 2) + if len(parts) != 2 { + continue + } + key := strings.TrimSpace(parts[0]) + val := strings.TrimSpace(parts[1]) + switch key { + case "COMMAND": + meta.Command = val + case "ADAPTER": + meta.Adapter = val + case "TARGET": + meta.Target = val + case "MODE": + meta.Mode = val + case "POLICY_URL": + meta.PolicyURL = val + case "EVENT_SINK": + meta.EventSink = val + } + } + if err := scanner.Err(); err != nil { + return managedShimMetadata{}, false, err + } + return meta, isManaged, nil +} + +func buildShimScript(shimBinary, command, adapter, target, mode, policyURL, eventSink string) string { + lines := []string{ + "#!/bin/sh", + "set -eu", + shimMarker, + "# DATAFOG_SHIM_COMMAND=" + command, + "# DATAFOG_SHIM_ADAPTER=" + adapter, + "# DATAFOG_SHIM_TARGET=" + target, + "# DATAFOG_SHIM_MODE=" + mode, + "# DATAFOG_SHIM_POLICY_URL=" + policyURL, + "# DATAFOG_SHIM_EVENT_SINK=" + eventSink, + "", + "SHIM_BINARY=" + shQuote(shimBinary), + "SHIM_MODE=" + shQuote(mode), + "SHIM_POLICY_URL=" + shQuote(policyURL), + "SHIM_EVENT_SINK=" + shQuote(eventSink), + "if [ -n \"${DATAFOG_SHIM_MODE:-}\" ]; then", + " SHIM_MODE=\"$DATAFOG_SHIM_MODE\"", + "fi", + "if [ -n \"${DATAFOG_SHIM_POLICY_URL:-}\" ]; then", + " SHIM_POLICY_URL=\"$DATAFOG_SHIM_POLICY_URL\"", + "fi", + "if [ -n \"${DATAFOG_SHIM_EVENT_SINK:-}\" ]; then", + " SHIM_EVENT_SINK=\"$DATAFOG_SHIM_EVENT_SINK\"", + "fi", + "", + `exec "$SHIM_BINARY" run \`, + ` --adapter "` + shellEscape(adapter) + `" \`, + ` --target "` + shellEscape(target) + `" \`, + ` --mode "$SHIM_MODE" \`, + ` --policy-url "$SHIM_POLICY_URL" \`, + ` --event-sink "$SHIM_EVENT_SINK" \`, + ` --api-token "${DATAFOG_SHIM_API_TOKEN:-}" \`, + ` -- \`, + ` "$@"`, + "", + } + return strings.Join(lines, "\n") +} + +func shellEscape(value string) string { + value = strings.TrimSpace(value) + return strings.ReplaceAll(value, `"`, `\"`) +} + +func shQuote(value string) string { + value = strings.ReplaceAll(value, `'`, `'\''`) + return "'" + value + "'" +} + +func shimScriptPath(dir, command string) string { + name := command + if runtime.GOOS == "windows" && !strings.HasSuffix(strings.ToLower(name), ".cmd") { + name += ".cmd" + } + return filepath.Join(dir, name) +} + +func usage() string { + text := strings.TrimSpace(` +usage: + datafog-shim --policy-url=http://localhost:8080 shell [args...] + datafog-shim --policy-url=http://localhost:8080 run [--adapter ] --target [args...] + datafog-shim --policy-url=http://localhost:8080 read-file + datafog-shim --policy-url=http://localhost:8080 write-file + datafog-shim adapters list + datafog-shim hooks install [--adapter ] [--target ] + datafog-shim hooks list + datafog-shim hooks uninstall +`) + return text +} diff --git a/cmd/datafog-shim/main_test.go b/cmd/datafog-shim/main_test.go new file mode 100644 index 0000000..7d09228 --- /dev/null +++ b/cmd/datafog-shim/main_test.go @@ -0,0 +1,235 @@ +package main + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/datafog/datafog-api/internal/shim" +) + +func TestParseMode(t *testing.T) { + t.Run("default", func(t *testing.T) { + mode, err := parseMode("") + if err != nil { + t.Fatalf("expected parse success, got %v", err) + } + if mode != shim.ModeEnforced { + t.Fatalf("expected default enforced, got %q", mode) + } + }) + + t.Run("observe", func(t *testing.T) { + mode, err := parseMode("observe") + if err != nil { + t.Fatalf("expected parse success, got %v", err) + } + if mode != shim.ModeObserve { + t.Fatalf("expected observe, got %q", mode) + } + }) + + t.Run("invalid", func(t *testing.T) { + if _, err := parseMode("invalid"); err == nil { + t.Fatal("expected parse error") + } + }) +} + +func TestResolveRuntimeConfig(t *testing.T) { + t.Setenv("DATAFOG_SHIM_POLICY_URL", "http://env:8080") + t.Setenv("DATAFOG_SHIM_MODE", string(shim.ModeObserve)) + + cfg, err := resolveRuntimeConfig(shimRuntimeConfig{}) + if err != nil { + t.Fatalf("expected config resolve, got %v", err) + } + if cfg.policyURL != "http://env:8080" { + t.Fatalf("expected env policy URL, got %q", cfg.policyURL) + } + if cfg.mode != string(shim.ModeObserve) { + t.Fatalf("expected observe mode, got %q", cfg.mode) + } + if cfg.shimDir == "" { + t.Fatal("expected shim directory fallback") + } +} + +func TestResolveTargetBinary(t *testing.T) { + root := t.TempDir() + bin := filepath.Join(root, "tool") + if err := os.WriteFile(bin, []byte(""), 0o755); err != nil { + t.Fatalf("write file: %v", err) + } + + got, err := resolveTargetBinary(bin) + if err != nil { + t.Fatalf("expected absolute resolve, got %v", err) + } + if got != bin { + abs, _ := filepath.Abs(bin) + if got != abs { + t.Fatalf("unexpected resolved path: %q", got) + } + } + + t.Run("pathLookup", func(t *testing.T) { + path := t.TempDir() + name := "lookupme" + if runtime.GOOS == "windows" { + name += ".exe" + } + commandBin := filepath.Join(path, name) + if err := os.WriteFile(commandBin, []byte(""), 0o755); err != nil { + t.Fatalf("write path command: %v", err) + } + t.Setenv("PATH", path+string(filepath.ListSeparator)+os.Getenv("PATH")) + + got, err := resolveTargetBinary("lookupme") + if err != nil { + t.Fatalf("expected path resolve, got %v", err) + } + if got != commandBin && got != filepath.Clean(commandBin) { + t.Fatalf("unexpected target resolve result: %q", got) + } + }) +} + +func TestBuildShimScript(t *testing.T) { + script := buildShimScript( + "/opt/datafog/datafog-shim", + "git", + "git", + "/usr/bin/git", + string(shim.ModeObserve), + "http://localhost:8080", + "/tmp/events.ndjson", + ) + if !strings.Contains(script, shimMarker) { + t.Fatalf("script missing shim marker") + } + if !strings.Contains(script, "# DATAFOG_SHIM_ADAPTER=git") { + t.Fatalf("script missing adapter metadata") + } + if !strings.Contains(script, "# DATAFOG_SHIM_TARGET=/usr/bin/git") { + t.Fatalf("script missing target metadata") + } + if !strings.Contains(script, `--mode "$SHIM_MODE"`) { + t.Fatalf("script missing runtime mode wiring") + } +} + +func TestInstallListAndUninstallShim(t *testing.T) { + root := t.TempDir() + shimDir := filepath.Join(root, "shims") + targetDir := filepath.Join(root, "targets") + if err := os.MkdirAll(targetDir, 0o755); err != nil { + t.Fatalf("mkdir targets: %v", err) + } + + targetBinary := filepath.Join(targetDir, "git") + if err := os.WriteFile(targetBinary, []byte(""), 0o755); err != nil { + t.Fatalf("write target binary: %v", err) + } + fakeShimBinary := filepath.Join(root, "datafog-shim") + if err := os.WriteFile(fakeShimBinary, []byte("#!/bin/sh\necho shim\n"), 0o755); err != nil { + t.Fatalf("write shim binary: %v", err) + } + + cfg := shimRuntimeConfig{ + policyURL: "http://localhost:8080", + mode: string(shim.ModeEnforced), + shimDir: shimDir, + } + + shimPath, err := installShimScript(fakeShimBinary, cfg, "git", "git", targetBinary, false) + if err != nil { + t.Fatalf("install shim failed: %v", err) + } + + shimPath = filepath.Clean(shimPath) + if runtime.GOOS != "windows" { + if got := shimPath; got != filepath.Clean(shimScriptPath(shimDir, "git")) { + t.Fatalf("unexpected shim path %q", got) + } + } + + found, managed, err := readShimMetadata(shimPath) + if err != nil { + t.Fatalf("read metadata: %v", err) + } + if !managed { + t.Fatal("expected managed shim") + } + if found.Adapter != "vcs" { + t.Fatalf("expected adapter vcs, got %q", found.Adapter) + } + + list, err := listManagedShims(shimDir) + if err != nil { + t.Fatalf("list managed shims: %v", err) + } + if len(list) != 1 { + t.Fatalf("expected one managed shim, got %d", len(list)) + } + if list[0].Command != "git" { + t.Fatalf("expected listed command git, got %q", list[0].Command) + } + + uninstallCfg := cfg + uninstallCfg.shimDir = shimDir + if err := runHooksUninstall(uninstallCfg, []string{"git"}); err != nil { + t.Fatalf("uninstall shim: %v", err) + } + + if _, statErr := os.Stat(shimPath); !os.IsNotExist(statErr) { + t.Fatalf("expected shim removed") + } +} + +func TestAdapterResolution(t *testing.T) { + if adapter := resolveAdapter("", "/usr/bin/git"); adapter != "vcs" { + t.Fatalf("expected git to resolve to vcs, got %q", adapter) + } + if adapter := resolveAdapter("", "/usr/local/bin/git"); adapter != "vcs" { + t.Fatalf("expected absolute git path to resolve to vcs, got %q", adapter) + } + if adapter := resolveAdapter("", "/usr/bin/docker"); adapter != "container" { + t.Fatalf("expected docker to resolve to container, got %q", adapter) + } + if adapter := resolveAdapter(" Git ", ""); adapter != "vcs" { + t.Fatalf("expected explicit git alias to resolve to vcs, got %q", adapter) + } + if adapter := resolveAdapter(" gh ", ""); adapter != "vcs" { + t.Fatalf("expected explicit gh alias to resolve to vcs, got %q", adapter) + } + if adapter := resolveAdapter(" gogcli ", ""); adapter != "vcs" { + t.Fatalf("expected explicit gogcli alias to resolve to vcs, got %q", adapter) + } + if adapter := resolveAdapter("customTool", "/usr/bin/git"); adapter != "customtool" { + t.Fatalf("expected unknown adapter to normalize only, got %q", adapter) + } + if adapter := resolveAdapter("", "/usr/bin/customcommand"); adapter != "customcommand" { + t.Fatalf("expected unknown command path to normalize only, got %q", adapter) + } +} + +func TestKnownAdaptersAreDeterministic(t *testing.T) { + adapters := knownAdapters() + if len(adapters) == 0 { + t.Fatalf("expected known adapters list to be populated") + } + + foundGit := false + for _, adapter := range adapters { + if adapter.Canonical == "vcs" { + foundGit = true + break + } + } + if !foundGit { + t.Fatalf("expected vcs canonical adapter") + } +} diff --git a/config/policy.json b/config/policy.json new file mode 100644 index 0000000..0d33e26 --- /dev/null +++ b/config/policy.json @@ -0,0 +1,87 @@ +{ + "policy_id": "datafog-mvp", + "policy_version": "v2026-02-24-2", + "description": "MVP policy with action + entity gating and platform adapter rules", + "updated_at": "2026-02-24T00:00:00Z", + "rules": [ + { + "id": "deny-shell-api-key", + "description": "Deny shell actions when API key is present in payload", + "priority": 100, + "effect": "deny", + "match": { + "action_types": ["shell.exec"] + }, + "entity_requirements": ["api_key"] + }, + { + "id": "allow-claude-help", + "description": "Allow claude help command", + "priority": 100, + "effect": "allow", + "match": { + "action_types": ["command.exec"], + "adapters": ["claude"], + "commands": ["--help"] + } + }, + { + "id": "allow-codex-help", + "description": "Allow codex help command", + "priority": 100, + "effect": "allow", + "match": { + "action_types": ["command.exec"], + "adapters": ["codex"], + "commands": ["--help"] + } + }, + { + "id": "deny-agent-shell-exec", + "description": "Block shell-style invocation from AI agent adapters", + "priority": 90, + "effect": "deny", + "match": { + "action_types": ["command.exec"], + "adapters": ["claude", "codex"], + "commands": ["-lc"] + } + }, + { + "id": "redact-file-write", + "description": "Redact PII before writing files", + "priority": 80, + "effect": "allow_with_redaction", + "match": { + "action_types": ["file.write"] + } + }, + { + "id": "redact-file-read", + "description": "Redact PII when reading files", + "priority": 80, + "effect": "allow_with_redaction", + "match": { + "action_types": ["file.read"] + } + }, + { + "id": "allow-shell", + "description": "Allow shell commands when no dangerous entities detected", + "priority": 10, + "effect": "allow", + "match": { + "action_types": ["shell.exec"] + } + }, + { + "id": "allow-annotate", + "description": "Allow annotation-only action to support non-side-effect checks", + "priority": 10, + "effect": "allow", + "match": { + "action_types": ["annotation.default", "annotation.custom"] + } + } + ] +} diff --git a/docs/DATA.md b/docs/DATA.md new file mode 100644 index 0000000..d8c4e5c --- /dev/null +++ b/docs/DATA.md @@ -0,0 +1,28 @@ +--- +title: "Data" +use_when: "Capturing data model and data-change safety rules for this repo (schemas, migrations, backfills, integrity, and operational safety)." +--- + +## Data Model + +- Source of truth for schemas (ORM models, migrations, schema dump files) and where they live. +- Entity ownership boundaries (what owns IDs, who can write which tables/collections). + +## Migrations + +- Migration rules (forward-only vs reversible, locking/online migration expectations, index/constraint strategy). +- Validation steps for schema changes (commands and what to check). + +## Backfills And Data Fixes + +- How to run backfills safely (idempotence, batching, checkpoints). +- How to verify correctness and how to roll back (or compensate) if needed. + +## Integrity And Consistency + +- Constraints and invariants that must remain true (unique keys, foreign keys, referential rules). +- Concurrency expectations (transactions/isolation, retry policies) where relevant. + +## Sensitive Data Notes + +- Pointers to where sensitive fields live and how they must be handled (logging/redaction, retention, deletion). diff --git a/docs/DESIGN.md b/docs/DESIGN.md new file mode 100644 index 0000000..c70b22c --- /dev/null +++ b/docs/DESIGN.md @@ -0,0 +1,17 @@ +--- +title: "Design" +use_when: "Documenting UI/UX design principles, visual direction, and interaction standards for this repo." +--- + +## Design Principles +- Clarity over cleverness; make the primary action obvious. +- Consistency beats novelty; reuse patterns unless there is a strong reason not to. +- Accessible by default (contrast, focus, keyboard). + +## Visual Direction +- Use design tokens (colors, spacing, typography) to keep the UI cohesive. +- Prefer a small, intentional palette and a consistent type scale. + +## Interaction Standards +- Every async action has loading, success, and error states with clear messaging. +- Forms validate inline and preserve user input; errors explain how to recover. diff --git a/docs/DOMAIN_DOCS.md b/docs/DOMAIN_DOCS.md new file mode 100644 index 0000000..7890143 --- /dev/null +++ b/docs/DOMAIN_DOCS.md @@ -0,0 +1,30 @@ +# Domain Docs Registry + +Reference for agents: what domain docs exist, how to detect relevant content, and when to create or update them. Domain docs are deployed at bootstrap with baseline guidance. Flesh them out with real, repo-specific content on demand. + +## Domain Docs + +| Doc | Path | Purpose | Auto-Detect Signals | Seed Question | +|---|---|---|---|---| +| DESIGN.md | `docs/DESIGN.md` | Design principles, visual direction, interaction standards | — | What are your core design principles? | +| DATA.md | `docs/DATA.md` | Data model and data-change safety rules (migrations/backfills/integrity) | `db/`, migrations, ORM schema files, backfill scripts | What are your data model and migration/backfill safety rules? | +| FRONTEND.md | `docs/FRONTEND.md` | Frontend stack, conventions, component architecture | `package.json` (react/vue/angular/svelte), `next.config.*`, `vite.config.*`, `tsconfig.json` | What's your frontend stack and key conventions? | +| PRODUCT_SENSE.md | `docs/PRODUCT_SENSE.md` | Target users, key outcomes, decision heuristics | — | Who are your target users and what outcomes matter most? | +| RELIABILITY.md | `docs/RELIABILITY.md` | Uptime targets, failure modes, operational guardrails | Dockerfile, health check routes, CI config | What are your reliability requirements? | +| SECURITY.md | `docs/SECURITY.md` | Threat model, auth, data sensitivity, compliance | Auth deps, middleware files, env var references | What security concerns apply? | +| OBSERVABILITY.md | `docs/OBSERVABILITY.md` | Logging, metrics, traces, health checks, agent access | Logging libs (winston/pino/structlog/slog), `/metrics`, opentelemetry/jaeger config, `/healthz` | What observability tools do you use? | +| core-beliefs.md | `docs/design-docs/core-beliefs.md` | Non-negotiable engineering beliefs | — | What are 2-3 non-negotiable engineering beliefs? | + +## When to Create or Update + +- **he-plan**: Identify relevant/missing domain docs during planning, then create/populate them at end-of-`he-plan` after final plan approval and before transition +- **he-implement**: If implementation reveals a missing, wrong, or incomplete domain doc, create or update it in-place and note in Revision Notes +- **he-learn**: Post-release policy updates from lessons learned +- **he-doc-gardening**: Flag stale domain docs for refresh + +## How to Create or Update + +1. Check if the domain doc file exists (bootstrap deploys all baseline docs) +2. If the doc has only baseline guidance (template defaults): replace with real, repo-specific content using auto-detect signals and current context +3. If it has real content: append or revise — never overwrite working policies without replacing them with something better +4. Preserve section structure (headings stay, content fills in) diff --git a/docs/FRONTEND.md b/docs/FRONTEND.md new file mode 100644 index 0000000..c7315ca --- /dev/null +++ b/docs/FRONTEND.md @@ -0,0 +1,24 @@ +--- +title: "Frontend" +use_when: "Documenting frontend stack, conventions, component architecture, performance budgets, and accessibility requirements for this repo." +--- + +## Stack +- Define supported browsers/platforms and the minimum accessibility target. +- Prefer a small set of core dependencies and consistent build tooling across the app. + +## Conventions +- Keep components small and named by what they do; avoid "utils soup" without ownership. +- Centralize shared UI primitives; avoid duplicating patterns across pages. + +## Component Architecture +- Separate UI rendering from data fetching/mutations where practical. +- Prefer explicit data flow and local state; introduce global state only with a clear boundary. + +## Performance +- Avoid unnecessary client work: minimize re-renders, split code on route/feature boundaries, and lazy-load heavy modules. +- Measure before optimizing; keep a short list of performance budgets that matter to users. + +## Accessibility +- Keyboard navigation works for all interactive controls; focus states are visible. +- Use semantic HTML first; ARIA is for filling gaps, not replacing semantics. diff --git a/docs/OBSERVABILITY.md b/docs/OBSERVABILITY.md new file mode 100644 index 0000000..d346b79 --- /dev/null +++ b/docs/OBSERVABILITY.md @@ -0,0 +1,25 @@ +--- +title: "Observability" +use_when: "Documenting logging, metrics, tracing, and health check conventions for this repo, including how agents can access signals to self-verify behavior." +--- + +## Logging Strategy +- Prefer structured logs with consistent fields (service, env, request_id/trace_id, user_id when safe). +- Never log secrets; be deliberate about PII. +- Log at boundaries and on errors; avoid noisy per-loop logging in hot paths. + +## Metrics +- Track the golden signals: latency, traffic, errors, saturation. +- Prefer histograms for latency; keep label cardinality low. + +## Traces +- Propagate trace context across service boundaries. +- Trace the critical paths (requests, background jobs) with stable span names. + +## Health Checks +- Health checks are fast and deterministic; readiness reflects dependency availability when needed. +- Document expected status codes and what "unhealthy" means operationally. + +## Agent Access +- Provide at least one concrete way to query each signal (logs, metrics, traces) without tribal knowledge. +- Include 1-2 copy-pastable examples per signal once the stack is known (commands, URLs, or queries). diff --git a/docs/PLANS.md b/docs/PLANS.md new file mode 100644 index 0000000..3e2d0cc --- /dev/null +++ b/docs/PLANS.md @@ -0,0 +1,171 @@ +# Agent Plans: + +This document describes the requirements for a plan ("Plan"), a design document that a coding agent can follow to deliver a working feature or system change. Treat the reader as a complete beginner to this repository: they have only the current working tree and the single Plan file you provide. There is no memory of prior plans and no external context. + +## How to use Plans and PLANS.md + +When authoring an executable specification (Plan), follow PLANS.md _to the letter_. If it is not in your context, refresh your memory by reading the entire PLANS.md file. Be thorough in reading (and re-reading) source material to produce an accurate specification. When creating a spec, start from the skeleton and flesh it out as you do your research. + +When implementing an executable specification (Plan), do not prompt the user for "next steps"; simply proceed to the next milestone. Keep all sections up to date, add or split entries in the list at every stopping point to affirmatively state the progress made and next steps. Resolve ambiguities autonomously, and commit frequently. + +When discussing an executable specification (Plan), record decisions in a log in the spec for posterity; it should be unambiguously clear why any change to the specification was made. Plans are living documents, and it should always be possible to restart from _only_ the Plan and no other work. + +When researching a design with challenging requirements or significant unknowns, use milestones to implement proof of concepts, "toy implementations", etc., that allow validating whether the user's proposal is feasible. Read the source code of libraries by finding or acquiring them, research deeply, and include prototypes to guide a fuller implementation. + +## Requirements + +NON-NEGOTIABLE REQUIREMENTS: + +* Every Plan must be fully self-contained. Self-contained means that in its current form it contains all knowledge and instructions needed for a novice to succeed. +* Every Plan is a living document. Contributors are required to revise it as progress is made, as discoveries occur, and as design decisions are finalized. Each revision must remain fully self-contained. +* Every Plan must enable a complete novice to implement the feature end-to-end without prior knowledge of this repo. +* Every Plan must produce a demonstrably working behavior, not merely code changes to "meet a definition". +* Every Plan must define every term of art in plain language or do not use it. + +Purpose and intent come first. Begin by explaining, in a few sentences, why the work matters from a user's perspective: what someone can do after this change that they could not do before, and how to see it working. Then guide the reader through the exact steps to achieve that outcome, including what to edit, what to run, and what they should observe. + +The agent executing your plan can list files, read files, search, run the project, and run tests. It does not know any prior context and cannot infer what you meant from earlier milestones. Repeat any assumption you rely on. Do not point to external blogs or docs; if knowledge is required, embed it in the plan itself in your own words. If a Plan builds upon a prior Plan and that file is checked in, incorporate it by reference. If it is not, you must include all relevant context from that plan. + +## Formatting + +Format and envelope are simple and strict. Each Plan must be one single fenced code block labeled as `md` that begins and ends with triple backticks. Do not nest additional triple-backtick code fences inside; when you need to show commands, transcripts, diffs, or code, present them as indented blocks within that single fence. Use indentation for clarity rather than code fences inside a Plan to avoid prematurely closing the Plan's code fence. Use two newlines after every heading, use # and ## and so on, and correct syntax for ordered and unordered lists. + +When writing a Plan to a Markdown (.md) file where the content of the file *is only* the single Plan, you should omit the triple backticks. + +Write in plain prose. Prefer sentences over lists. Avoid checklists, tables, and long enumerations unless brevity would obscure meaning. Checklists are permitted only in the `Progress` section, where they are mandatory. Narrative sections must remain prose-first. + +## Guidelines + +Self-containment and plain language are paramount. If you introduce a phrase that is not ordinary English ("daemon", "middleware", "RPC gateway", "filter graph"), define it immediately and remind the reader how it manifests in this repository (for example, by naming the files or commands where it appears). Do not say "as defined previously" or "according to the architecture doc." Include the needed explanation here, even if you repeat yourself. + +Avoid common failure modes. Do not rely on undefined jargon. Do not describe "the letter of a feature" so narrowly that the resulting code compiles but does nothing meaningful. Do not outsource key decisions to the reader. When ambiguity exists, resolve it in the plan itself and explain why you chose that path. Err on the side of over-explaining user-visible effects and under-specifying incidental implementation details. + +Anchor the plan with observable outcomes. State what the user can do after implementation, the commands to run, and the outputs they should see. Acceptance should be phrased as behavior a human can verify ("after starting the server, navigating to [http://localhost:8080/health](http://localhost:8080/health) returns HTTP 200 with body OK") rather than internal attributes ("added a HealthCheck struct"). If a change is internal, explain how its impact can still be demonstrated (for example, by running tests that fail before and pass after, and by showing a scenario that uses the new behavior). + +Specify repository context explicitly. Name files with full repository-relative paths, name functions and modules precisely, and describe where new files should be created. If touching multiple areas, include a short orientation paragraph that explains how those parts fit together so a novice can navigate confidently. When running commands, show the working directory and exact command line. When outcomes depend on environment, state the assumptions and provide alternatives when reasonable. + +Be idempotent and safe. Write the steps so they can be run multiple times without causing damage or drift. If a step can fail halfway, include how to retry or adapt. If a migration or destructive operation is necessary, spell out backups or safe fallbacks. Prefer additive, testable changes that can be validated as you go. + +Validation is not optional. Include instructions to run tests, to start the system if applicable, and to observe it doing something useful. Describe comprehensive testing for any new features or capabilities. Include expected outputs and error messages so a novice can tell success from failure. Where possible, show how to prove that the change is effective beyond compilation (for example, through a small end-to-end scenario, a CLI invocation, or an HTTP request/response transcript). State the exact test commands appropriate to the project’s toolchain and how to interpret their results. + +Capture evidence. When your steps produce terminal output, short diffs, or logs, include them inside the single fenced block as indented examples. Keep them concise and focused on what proves success. If you need to include a patch, prefer file-scoped diffs or small excerpts that a reader can recreate by following your instructions rather than pasting large blobs. + +## Milestones + +Milestones are narrative, not bureaucracy. If you break the work into milestones, introduce each with a brief paragraph that describes the scope, what will exist at the end of the milestone that did not exist before, the commands to run, and the acceptance you expect to observe. Keep it readable as a story: goal, work, result, proof. Progress and milestones are distinct: milestones tell the story, progress tracks granular work. Both must exist. Never abbreviate a milestone merely for the sake of brevity, do not leave out details that could be crucial to a future implementation. + +Each milestone must be independently verifiable and incrementally implement the overall goal of the plan. + +## Living plans and design decisions + +* Plans are living documents. As you make key design decisions, update the plan to record both the decision and the thinking behind it. Record all decisions in the `Decision Log` section. +* Plans must contain and maintain a `Progress` section, a `Surprises & Discoveries` section, a `Decision Log`, and an `Outcomes & Retrospective` section. These are not optional. +* When you discover optimizer behavior, performance tradeoffs, unexpected bugs, or inverse/unapply semantics that shaped your approach, capture those observations in the `Surprises & Discoveries` section with short evidence snippets (test output is ideal). +* If you change course mid-implementation, document why in the `Decision Log` and reflect the implications in `Progress`. Plans are guides for the next contributor as much as checklists for you. +* At completion of a major task or the full plan, write an `Outcomes & Retrospective` entry summarizing what was achieved, what remains, and lessons learned. +* Plans must include explicit workflow handoff sections so later phases have a stable contract: + * `## Pull Request` (populated by PR-opening workflow) + * `## Review Findings` (populated by `he-review`) + * `## Verify/Release Decision` (populated by `he-verify-release`) + +# Prototyping milestones and parallel implementations + +It is acceptable—-and often encouraged—-to include explicit prototyping milestones when they de-risk a larger change. Examples: adding a low-level operator to a dependency to validate feasibility, or exploring two composition orders while measuring optimizer effects. Keep prototypes additive and testable. Clearly label the scope as “prototyping”; describe how to run and observe results; and state the criteria for promoting or discarding the prototype. + +Prefer additive code changes followed by subtractions that keep tests passing. Parallel implementations (e.g., keeping an adapter alongside an older path during migration) are fine when they reduce risk or enable tests to continue passing during a large migration. Describe how to validate both paths and how to retire one safely with tests. When working with multiple new libraries or feature areas, consider creating spikes that evaluate the feasibility of these features _independently_ of one another, proving that the external library performs as expected and implements the features we need in isolation. + +## Skeleton of a Good Plan + + # + + This Plan is a living document. The sections `Progress`, `Surprises & Discoveries`, `Decision Log`, and `Outcomes & Retrospective` must be kept up to date as work proceeds. + + If PLANS.md file is checked into the repo, reference the path to that file here from the repository root and note that this document must be maintained in accordance with PLANS.md. + + ## Purpose / Big Picture + + Explain in a few sentences what someone gains after this change and how they can see it working. State the user-visible behavior you will enable. + + ## Progress + + Use a list with checkboxes to summarize granular steps. Every stopping point must be documented here, even if it requires splitting a partially completed task into two (“done” vs. “remaining”). This section must always reflect the actual current state of the work. + + - [x] (2025-10-01 13:00Z) Example completed step. + - [ ] Example incomplete step. + - [ ] Example partially completed step (completed: X; remaining: Y). + + Use timestamps to measure rates of progress. + + ## Surprises & Discoveries + + Document unexpected behaviors, bugs, optimizations, or insights discovered during implementation. Provide concise evidence. + + - Observation: … + Evidence: … + + ## Decision Log + + Record every decision made while working on the plan in the format: + + - Decision: … + Rationale: … + Date/Author: … + + ## Outcomes & Retrospective + + Summarize outcomes, gaps, and lessons learned at major milestones or at completion. Compare the result against the original purpose. + + ## Context and Orientation + + Describe the current state relevant to this task as if the reader knows nothing. Name the key files and modules by full path. Define any non-obvious term you will use. Do not refer to prior plans. + + ## Plan of Work + + Describe, in prose, the sequence of edits and additions. For each edit, name the file and location (function, module) and what to insert or change. Keep it concrete and minimal. + + ## Concrete Steps + + State the exact commands to run and where to run them (working directory). When a command generates output, show a short expected transcript so the reader can compare. This section must be updated as work proceeds. + + ## Validation and Acceptance + + Describe how to start or exercise the system and what to observe. Phrase acceptance as behavior, with specific inputs and outputs. If tests are involved, say "run and expect passed; the new test fails before the change and passes after>". + + ## Idempotence and Recovery + + If steps can be repeated safely, say so. If a step is risky, provide a safe retry or rollback path. Keep the environment clean after completion. + + ## Artifacts and Notes + + Include the most important transcripts, diffs, or snippets as indented examples. Keep them concise and focused on what proves success. + + ## Interfaces and Dependencies + + Be prescriptive. Name the libraries, modules, and services to use and why. Specify the types, traits/interfaces, and function signatures that must exist at the end of the milestone. Prefer stable names and paths such as `crate::module::function` or `package.submodule.Interface`. E.g.: + + In crates/foo/planner.rs, define: + + pub trait Planner { + fn plan(&self, observed: &Observed) -> Vec; + } + + ## Pull Request + + This section is the stable handoff contract to the PR/CI phase. Record: + + - pr: + - branch: + - commit: + - ci: + + ## Review Findings + + Populated by review workflow (e.g. `he-review`). Consolidate findings here with priorities and locations. + + ## Verify/Release Decision + + Populated by verify/release workflow (e.g. `he-verify-release`). Record GO/NO-GO plus evidence and rollback. + +If you follow the guidance above, a single, stateless agent -- or a human novice -- can read your Plan from top to bottom and produce a working, observable result. That is the bar: SELF-CONTAINED, SELF-SUFFICIENT, NOVICE-GUIDING, OUTCOME-FOCUSED. + +When you revise a plan, you must ensure your changes are comprehensively reflected across all sections, including the living document sections, and you must write a note at the bottom of the plan describing the change and the reason why. Plans must describe not just the what but the why for almost everything. diff --git a/docs/PRODUCT_SENSE.md b/docs/PRODUCT_SENSE.md new file mode 100644 index 0000000..91a4238 --- /dev/null +++ b/docs/PRODUCT_SENSE.md @@ -0,0 +1,20 @@ +--- +title: "Product Sense" +use_when: "Capturing target users, success outcomes, decision heuristics, and quality criteria for this repo." +--- + +## Target Users +- Name the primary user and the primary job-to-be-done; list any secondary users explicitly. +- Call out non-users (who this is not for) to reduce scope creep. + +## Key Outcomes +- Define 1-3 outcomes that matter and how you will measure them (even if qualitative). +- Prefer metrics tied to user time, reliability, and task completion. + +## Decision Heuristics +- Prefer shipping a smaller, complete slice over a broad, partial feature. +- Optimize for reducing user effort and reducing operational burden. + +## Quality Criteria +- Clear error messages and recovery paths; no silent failures. +- Sensible defaults and empty states; predictable navigation. diff --git a/docs/RELIABILITY.md b/docs/RELIABILITY.md new file mode 100644 index 0000000..385adee --- /dev/null +++ b/docs/RELIABILITY.md @@ -0,0 +1,62 @@ +--- +title: "Reliability" +use_when: "Capturing reliability goals, failure modes, monitoring, and operational guardrails for this repo." +--- + +## Reliability goals (MVP) + +- Primary flow (`POST /v1/scan`): 99.9% availability, p95 latency below 250ms at steady load. +- Policy and redaction consistency (`POST /v1/decide`, `POST /v1/transform`, `POST /v1/anonymize`): 99.5% availability, p95 latency below 350ms. +- Health signal (`GET /health`): 99.99% availability for readiness/liveness checks. + +Definition of degraded: + +- Availability below target for 5-minute windows. +- p95 latency sustained > 1.5x target for 10 minutes. +- Error rate > 1% for any public endpoint. + +## Failure Modes + +Top failures and controls: + +- Policy file missing or invalid JSON: + - Signal: `policy_load_failed_total` increases, `/health` may degrade. + - Blast radius: all scan/decide/transform calls fail. + - Recovery: roll back to last known-good `config/policy.json`, fix schema, redeploy. + +- Receipt path write failure: + - Signal: request-level `receipt_write_failed` metric spikes, partial request successes. + - Blast radius: observability of decisions degrades first; policy logic still runs. + - Recovery: fix filesystem permissions, point to healthy `DATAFOG_RECEIPT_PATH`, restart. + +- Rate limit configuration too low or malformed: + - Signal: sudden `429` rise and client-side retries. + - Blast radius: throughput reduction for bursty clients. + - Recovery: validate and tune `DATAFOG_RATE_LIMIT_RPS`, deploy config change. + +- Bad deployment image or env drift: + - Signal: crash/restart loop, increased non-2xx responses. + - Blast radius: endpoint unavailability. + - Recovery: rollback image/version and redeploy after diff review. + +## Monitoring + +Minimum signal set: + +- Error rate by endpoint and status code. +- p95/p99 latency per endpoint. +- `/health` pass/fail and startup duration. +- `DATAFOG_RATE_LIMIT_RPS` rejections. +- Receipt persistence success rate. + +Alert rules: + +- Page if SLO burn reaches 10% remaining over 10 minutes. +- Page on crash loop, persistent readiness failure, or error budget burn above threshold. +- Warn on sustained latency regression above 2x target for two consecutive intervals. + +## Operational Guardrails + +- Keep configuration centralized and immutable per release (`policy`, env vars, receipt path). +- Every change must include a verified rollback command or known Git point-in-time for the container image and config map. +- Prefer controlled rollout with canaries for policy schema changes and rate-limit changes. diff --git a/docs/SECURITY.md b/docs/SECURITY.md new file mode 100644 index 0000000..61fb0f4 --- /dev/null +++ b/docs/SECURITY.md @@ -0,0 +1,71 @@ +--- +title: "Security" +use_when: "Capturing security expectations for datafog-api: threat model, auth/authorization, data sensitivity, compliance, and required controls." +--- + +## Scope and threat model + +This service is an HTTP API that accepts untrusted text payloads and returns structured inspection or redaction decisions. + +Assume public internet exposure for all request endpoints unless deployment policy specifies internal networking. + +## Assets + +- PII and policy-sensitive text submitted for scanning and transformation. +- API token and runtime configuration (`DATAFOG_API_TOKEN`, timeout and path settings). +- Receipt storage (`DATAFOG_RECEIPT_PATH`) containing per-request decisions. +- Container images and Git history. + +## Trust boundaries + +- Boundary 1: client ↔ datafog-api over HTTP. +- Boundary 2: operator/admin ↔ runtime environment (container image, host, secrets). +- Boundary 3: in-process policy and redaction engine ↔ local filesystem. + +## Authentication and authorization + +- API token is optional and configured via `DATAFOG_API_TOKEN`. +- When enabled, all non-public endpoints require `Authorization: Bearer ` or `X-API-Key: `. +- Keep default deployment policy at least as strict as token-required if endpoint access is not intentionally internal. +- Use per-environment tokens; rotate on incidents and after any suspected leak. + +## Input and output handling + +- Never trust request bodies; all inputs are validated by request schema and handler-level parsing. +- Error messages must avoid leaking request secrets or raw internal stack traces. +- No secrets should be echoed in responses, logs, or receipts. + +## Data handling and retention + +- Treat text payloads and findings as confidential. +- `DATAFOG_RECEIPT_PATH` stores action receipts and should be writable only to the minimal directory required by deployment. +- Store receipts with minimal retention where possible; if retention policy is externalized, define purge windows in deployment docs. + +## Operational controls + +- Do not include secrets in repository history, container args, or logs. +- Use process isolation and least privilege in deployment: + - `USER 65532` in container image. + - `readOnlyRootFilesystem`, dropped capabilities, and no privilege escalation in orchestration. +- Enforce transport security (TLS termination before API pods/services if not done inside service). +- Keep rate limit enabled (`DATAFOG_RATE_LIMIT_RPS`) in multi-tenant or public exposures. + +## Compliance expectations + +- If policy data includes regulated PII categories, classify that data and map obligations before enabling production rollout. +- Maintain a documented decision log for retention/erasure support and policy schema evolution. + +## Controls and hardening checks + +- Dependency and static security checks are part of the production hardening phase: + - `gosec` for common Go security smells. + - `govulncheck` for module vulnerability visibility. +- In CI, these checks are hard-failing (`exit non-zero`) on reported findings. + +## Incident response baseline + +If compromise or data exposure is suspected: + +1. Revoke `DATAFOG_API_TOKEN` and issue new token. +2. Rotate any service credentials and block traffic by policy as needed. +3. Freeze release of mutated policy files until integrity checks are revalidated. diff --git a/docs/contracts/datafog-api-contract.md b/docs/contracts/datafog-api-contract.md new file mode 100644 index 0000000..8e7f800 --- /dev/null +++ b/docs/contracts/datafog-api-contract.md @@ -0,0 +1,335 @@ +# datafog-api Contract + +## Version + +- API revision: `v1` +- Service revision tracked via policy file fields: `policy_id`, `policy_version` + +## Base URL + +- Default local: `http://localhost:8080` +- JSON request and response payloads for all API routes. + +## Cross-cutting response model + +All responses use JSON and include `Content-Type: application/json`. + +`X-Request-ID` is always returned on every response. +- If `x-request-id` is provided by the caller, the same value is reflected in the response header and error `request_id` payload. +- If missing, the service generates a request id and returns it in `X-Request-ID` and error payloads where present. + +POST requests require `Content-Type: application/json` (charset may be supplied with standard media type syntax). +A request body larger than 1 MiB (`1048576` bytes) is rejected with `request_too_large`. + +If `DATAFOG_API_TOKEN` is configured, every request must include either: + +- `Authorization: Bearer ` +- `X-API-Key: ` + +If `DATAFOG_RATE_LIMIT_RPS` is greater than `0`, requests are subject to a service-wide token-bucket request cap with response `429` and `rate_limited` on excess bursts. + +### Standard error response + +```json +{ + "error": { + "code": "invalid_request", + "message": "descriptive error message", + "request_id": "optional request id propagation", + "details": "optional low-level details" + } +} +``` + +### Standard codes + +- `invalid_request` (400) +- `method_not_allowed` (405) +- `not_found` (404) +- `unauthorized` (401) +- `rate_limited` (429) +- `idempotency_conflict` (409) +- `unsupported_media_type` (415) +- `request_too_large` (413) +- `encode_error` (500) +- `hash_error` (500) +- `receipt_error` (500) +- `internal_error` (500) + +## Endpoints + +### `GET /health` + +Returns process health and policy metadata. + +```json +{ + "status": "ok", + "policy_id": "string", + "policy_version": "string", + "started_at": "RFC3339 timestamp" +} +``` + +### `GET /v1/policy/version` + +Returns policy identity. + +```json +{ + "policy_id": "string", + "policy_version": "string" +} +``` + +### `GET /metrics` + +Returns coarse-grained service telemetry for operations and routing. + +```json +{ + "total_requests": 42, + "error_requests": 3, + "by_status": { + "200": 30, + "400": 1, + "404": 2, + "500": 0 + }, + "by_path": { + "/health": 20, + "/v1/scan": 5, + "/v1/decide": 3, + "/_not_found": 3 + }, + "by_method": { + "GET": 14, + "POST": 28 + }, + "started_at": "RFC3339 timestamp", + "uptime_seconds": 12.34 +} +``` + +`by_status`, `by_path`, and `by_method` include counters for completed requests observed before each `/metrics` call. `/metrics` request details appear on subsequent polling. + +### `POST /v1/scan` + +Scans free text and returns deterministic findings. + +#### Request + +```json +{ + "text": "string (required)", + "entity_types": ["optional", "list", "of", "entities"], + "request_id": "optional opaque id", + "trace_id": "optional correlation id", + "idempotency_key": "optional key for replay-safe dedupe" +} +``` + +#### Response 200 + +```json +{ + "request_id": "string", + "trace_id": "string", + "findings": [ + { + "entity_type": "email|phone|ssn|api_key|credit_card", + "value": "string", + "start": 0, + "end": 5, + "confidence": 0.0 + } + ], + "policy_version": "string", + "policy_id": "string" +} +``` + +### `POST /v1/decide` + +Evaluates action policy against findings and returns a deterministic decision. + +#### Request + +```json +{ + "action": { + "type": "string (required)", + "tool": "optional", + "resource": "optional", + "command": "optional", + "args": ["optional", "args"], + "sensitive": false + }, + "text": "optional input text; if no findings are supplied", + "findings": [ + { + "entity_type": "email|phone|ssn|api_key|credit_card", + "value": "string", + "start": 0, + "end": 5, + "confidence": 0.0 + } + ], + "request_id": "optional opaque id", + "trace_id": "optional correlation id", + "tenant_id": "optional tenant context", + "actor_id": "optional actor context", + "session_id": "optional session context", + "idempotency_key": "optional key for replay-safe dedupe" +} +``` + +#### Response 200 + +```json +{ + "request_id": "string", + "trace_id": "string", + "decision": "allow|allow_with_redaction|transform|deny", + "receipt_id": "string", + "policy_version": "string", + "policy_id": "string", + "matched_rules": ["rule_id"], + "transform_plan": [ + { "entity_type": "email", "mode": "mask|tokenize|anonymize|redact" } + ], + "findings": [ + { + "entity_type": "email|phone|ssn|api_key|credit_card", + "value": "string", + "start": 0, + "end": 5, + "confidence": 0.0 + } + ], + "reason": "optional reason for deny/fallback" +} +``` + +### `POST /v1/transform` + +Transforms text based on per-entity transforms. + +#### Request + +```json +{ + "text": "string (required)", + "findings": [], + "mode": "mask|tokenize|anonymize|redact", + "entity_modes": { + "email": "mask", + "phone": "tokenize" + }, + "request_id": "optional opaque id", + "trace_id": "optional correlation id", + "idempotency_key": "optional key for replay-safe dedupe" +} + +`transform` accepts only the documented modes (`mask`, `tokenize`, `anonymize`, `redact`) in both `mode` and `entity_modes` values. +Invalid transform mode values result in `400` with `code: invalid_request`. +`entity_modes` must not contain empty keys. +``` + +#### Response 200 + +```json +{ + "request_id": "string", + "trace_id": "string", + "output": "string", + "policy_id": "string", + "policy_version": "string", + "stats": { + "entities_transformed": 1, + "modes_applied": "entity:mode,entity:mode" + } +} +``` + +### `POST /v1/anonymize` + +Applies irreversible anonymization for detected entities. + +#### Request + +```json +{ + "text": "string (required)", + "findings": [], + "request_id": "optional opaque id", + "trace_id": "optional correlation id", + "idempotency_key": "optional key for replay-safe dedupe" +} +``` + +#### Response 200 + +```json +{ + "request_id": "string", + "trace_id": "string", + "output": "string", + "policy_id": "string", + "policy_version": "string", + "stats": { + "entities_transformed": 1, + "modes_applied": "email:anonymize" + } +} +``` + +### `GET /v1/receipts/{id}` + +Returns persisted decision receipts. + +```json +{ + "receipt_id": "string", + "timestamp": "RFC3339 timestamp", + "request_id": "string", + "trace_id": "string", + "tenant_id": "string", + "actor_id": "string", + "session_id": "string", + "policy_version": "string", + "policy_id": "string", + "action_hash": "sha256 hex", + "input_hash": "sha256 hex", + "sanitized_summary": "optional json string summary", + "decision": "allow|allow_with_redaction|transform|deny", + "action": { + "type": "string", + "tool": "string", + "resource": "string", + "command": "string", + "args": ["string"], + "sensitive": false + }, + "matched_rules": ["rule_id"], + "findings": [ + { + "entity_type": "email|phone|ssn|api_key|credit_card", + "value": "string", + "start": 0, + "end": 5, + "confidence": 0.0 + } + ], + "transform_plan": [ + { "entity_type": "email", "mode": "mask|tokenize|anonymize|redact" } + ], + "reason": "optional" +} +``` + +## Idempotency + +- Supported endpoints: `POST /v1/scan`, `POST /v1/decide`, `POST /v1/transform`, `POST /v1/anonymize`. +- Replaying the same idempotency key and identical semantic payload returns the same status and body. +- Reusing a key with different payloads returns `409` and `code: idempotency_conflict`. diff --git a/docs/demo.html b/docs/demo.html new file mode 100644 index 0000000..bad9b1e --- /dev/null +++ b/docs/demo.html @@ -0,0 +1,574 @@ + + + + + +DataFog — Scenario Explorer + + + + +
+

DataFog Scenario Explorer

+
+
Connecting...
+ +
+
+ +
+ + + + diff --git a/docs/design-docs/core-beliefs.md b/docs/design-docs/core-beliefs.md new file mode 100644 index 0000000..ec1057c --- /dev/null +++ b/docs/design-docs/core-beliefs.md @@ -0,0 +1,17 @@ +# Core Beliefs + +Document the product and engineering beliefs that guide roadmap, architecture, and delivery decisions. + + + +## Belief 1 + +- Statement: +- Why it matters: +- Tradeoffs: + +## Belief 2 + +- Statement: +- Why it matters: +- Tradeoffs: diff --git a/docs/design-docs/index.md b/docs/design-docs/index.md new file mode 100644 index 0000000..ff2d88e --- /dev/null +++ b/docs/design-docs/index.md @@ -0,0 +1,8 @@ +# Design Docs Index + +Design rationale and deep dives live here. + +## Documents + +- `core-beliefs.md` + diff --git a/docs/generated/README.md b/docs/generated/README.md new file mode 100644 index 0000000..e10a4d3 --- /dev/null +++ b/docs/generated/README.md @@ -0,0 +1,36 @@ +# Generated Context + +This directory holds generated reference context that agents create to reason about the codebase. Files here are auto-generated snapshots — not hand-authored documentation. + +## When to Create + +Create a generated context file when a skill (`he-implement`, `he-review`, `he-doc-gardening`) discovers relevant project infrastructure during its workflow. Discovery signals and corresponding context files: + +| Discovery Signal | Context to Create | Example Filename | +|---|---|---| +| Database migrations or schema files exist | Schema snapshot | `db-schema.md` | +| Route definitions or API framework detected | API endpoint index | `api-schema.md` | +| UI component hierarchy (React, Vue, etc.) | Component tree map | `component-tree.md` | +| Complex module dependency structure | Dependency graph | `dependency-graph.md` | + +This is not exhaustive — create whatever context helps agents reason about the project. The key rule: only create files for infrastructure that actually exists. + +## Format Contract + +Every generated file must include: + +``` +- last_updated: YYYY-MM-DD HH:MM +``` + +The `he-docs-lint` CI gate checks this timestamp on all files in this directory (except README.md and memory.md). + +## Rules + +- **Do not** create files for infrastructure the project does not have. +- **Do not** manually edit generated files — regenerate them from source. +- **Do** regenerate when the underlying source changes (migrations added, routes modified, etc.). + +## memory.md + +`memory.md` is a separate concept: it is a scratchpad for observations and patterns discovered during work, processed by `he-learn`. It is not auto-generated context and is not subject to the `last_updated` requirement. diff --git a/docs/generated/api-schema.md b/docs/generated/api-schema.md new file mode 100644 index 0000000..bd565fe --- /dev/null +++ b/docs/generated/api-schema.md @@ -0,0 +1,23 @@ +- last_updated: 2026-02-23 12:00 + +# API Schema Snapshot + +Discovered from `internal/server/server.go` route registration. + +## Routes + +- `GET /health` +- `GET /v1/policy/version` +- `POST /v1/scan` +- `POST /v1/decide` +- `POST /v1/transform` +- `POST /v1/anonymize` +- `GET /v1/receipts/{id}` +- `GET /metrics` + +## Route behavior + +- Method validation returns `method_not_allowed` on unsupported methods. +- Unknown routes return `not_found`. +- All successful and error responses are JSON with `Content-Type: application/json`. +- Every response includes `X-Request-ID`; callers may provide `x-request-id` for propagation. diff --git a/docs/generated/memory.md b/docs/generated/memory.md new file mode 100644 index 0000000..9f1fa92 --- /dev/null +++ b/docs/generated/memory.md @@ -0,0 +1 @@ +# Memory diff --git a/docs/plans/README.md b/docs/plans/README.md new file mode 100644 index 0000000..05dfd0b --- /dev/null +++ b/docs/plans/README.md @@ -0,0 +1,15 @@ +# Plans + +Active plans: +- `docs/plans/active/-plan.md` (`plan_mode: lightweight|execution`) + +Completed plans: +- `docs/plans/completed/-plan.md` + +All plan files must start with YAML frontmatter and follow `docs/PLANS.md` with required living sections, especially: + +- `## Progress` (timestamped checkbox list) +- `## Surprises & Discoveries` +- `## Decision Log` +- `## Outcomes & Retrospective` +- `## Revision Notes` diff --git a/docs/plans/tech-debt-tracker.md b/docs/plans/tech-debt-tracker.md new file mode 100644 index 0000000..ddd2b8c --- /dev/null +++ b/docs/plans/tech-debt-tracker.md @@ -0,0 +1,22 @@ +# Tech Debt Tracker + +General-purpose deferred-work queue. Review findings, cleanup tasks, improvement ideas — anything we want to address later but shouldn't block now. Any skill can append to this file. + +Treat this file as append-and-update: do not delete historical rows unless duplicated by mistake. When status changes, update both the index table row and the detail entry. + +## Status Semantics + +- `new`: captured, not yet scheduled. +- `queued`: prioritized for a future slug. +- `in_progress`: being addressed in an active plan. +- `resolved`: fixed, evidence linked. +- `wont_fix`: consciously accepted with documented rationale. + +## Index + +| ID | Date | Priority | Source | Status | Summary | +|---|---|---|---|---|---| + +## Detail Entries + + diff --git a/docs/runbooks/address-review-findings.md b/docs/runbooks/address-review-findings.md new file mode 100644 index 0000000..137d38e --- /dev/null +++ b/docs/runbooks/address-review-findings.md @@ -0,0 +1,30 @@ +--- +title: "Address Review Findings" +use_when: "You have review findings in an active plan and need a consistent process to fix, re-run review, and document what changed." +called_from: + - he-review + - he-implement +--- + +# Address Review Findings + +This runbook is repo-specific and **additive only**. It must not waive or override any gates enforced by skills. + +## Workflow + +1. Triage findings by priority. +2. For each `critical`/`high`, do one of: + - fix it (preferred), or + - escalate per `he-review` SKILL.md § Escalation if behavior is ambiguous or risk is unclear. +3. For `medium`/`low`, either: + - fix it, or + - accept it explicitly in the plan with rationale and follow-up link. +4. Update evidence: + - rerun the most relevant tests + - update `Artifacts and Notes` with new proof +5. Update `Progress`, `Decision Log`, and `Revision Notes` in the plan to reflect what changed and why. +6. Re-run `he-review` if the change materially altered behavior or implementation. + +## Re-entry Rules + +See `he-review` SKILL.md § Re-entry Rules for the canonical gates (design-level issues and material behavior changes). diff --git a/docs/runbooks/ci-failures.md b/docs/runbooks/ci-failures.md new file mode 100644 index 0000000..16a984e --- /dev/null +++ b/docs/runbooks/ci-failures.md @@ -0,0 +1,46 @@ +--- +title: "Remediate CI Failures" +use_when: "A verify/release gate fails due to build/test/lint failures locally or in CI; you need a consistent triage and stop/escalate policy." +called_from: + - he-verify-release + - he-implement +--- + +# Remediate CI Failures + +This runbook is repo-specific and **additive only**. It must not waive or override any gates enforced by skills. + +Treat CI failures as signal. The goal is not to make CI green by any means; it is to restore correctness with minimal, root-cause fixes. + +## Triage Order + +1. Confirm you are testing the right thing (branch, commit, env). +2. Identify failure class: + - deterministic test failure + - flaky test + - lint/format/typecheck + - build/tooling regression +3. Reduce to the smallest reproducer command. + +## Deterministic Failures + +- Add or adjust a real unit/e2e test when the failure indicates a missing assertion. +- Fix the underlying behavior; avoid "just loosen the test" unless the test is truly wrong. + +## Flaky Failures + +- If you can reproduce locally, fix like deterministic. +- If you cannot reproduce: + - mark as `judgment required` and escalate with evidence per the calling skill's § Escalation + - do not disable tests silently + +## Tooling Failures + +- Keep changes minimal and reversible. +- Prefer pinning/fixing the tool invocation over broad refactors. + +## Required Evidence + +- Command used to reproduce +- Short failure output excerpt +- Command/output showing the fix diff --git a/docs/runbooks/code-review.md b/docs/runbooks/code-review.md new file mode 100644 index 0000000..3ed0df6 --- /dev/null +++ b/docs/runbooks/code-review.md @@ -0,0 +1,34 @@ +--- +title: "Code Review" +use_when: "Running he-review to perform structured review fanout, write Review Findings into the active plan, and decide whether the work can proceed to verify/release." +called_from: + - he-review +--- + +# Code Review + +This runbook is repo-specific and **additive only**. It must not waive or override any gates enforced by skills. + +The skill `he-review` enforces stable gates (fanout, findings format, and priority blocking). This document carries the details that change per project. Inputs: active plan (`docs/plans/active/-plan.md` with `## Review Findings`) and current branch diff/test evidence. + +## Output + +Populate `## Review Findings` with: + +- a prioritized list of findings (see `docs/runbooks/review-findings.md`) +- accepted medium/low items (explicitly called out) +- any required re-entry decision (`he-implement` vs `he-plan`) + +## What Review Must Cover (Customize Per Repo) + +Keep this list short and concrete: + +- correctness and edge cases in the changed area +- tests: coverage of new behavior and regression prevention +- user-visible behavior (if applicable) with evidence +- security/data boundaries (if applicable) +- performance or reliability impact (if applicable) + +## Escalation + +If review requires judgment (risk unclear, expected behavior ambiguous, flaky failures), stop and escalate per `he-review` SKILL.md § Escalation. diff --git a/docs/runbooks/datafog-claude-agent-ux.md b/docs/runbooks/datafog-claude-agent-ux.md new file mode 100644 index 0000000..e54843b --- /dev/null +++ b/docs/runbooks/datafog-claude-agent-ux.md @@ -0,0 +1,122 @@ +--- +title: "Datafog + Claude Code Agent UX setup" +use_when: "A developer wants a minimal, reliable setup flow for enforcing Datafog policy around Claude Code agent actions." +called_from: + - he-implement + - he-spec + - he-review +--- + +# Datafog + Claude Code Agent UX setup + +This runbook captures the user experience for creating a secure workflow that +adds policy checkpoints to Claude Code without changing how users run Claude Code day-to-day. + +The end state is: +- a normal `claude` command still works, +- side-effect actions are checked against policy before execution, +- policy decisions and enforcement mode are explicit and reversible, +- setup can be validated in under two minutes. + +## Why this UX is efficient + +The onboarding experience should minimize manual glue code. A user should not have to know policy internals before trying the feature. + +The flow is: + +1. Start policy service. +2. Run one bootstrap command. +3. Source one generated env file. +4. Keep PATH updated with shim directory. + +This makes "I have policy-aware coding agent behavior" a predictable, low-friction sequence. + +## Prerequisites + +- `datafog-api` running and reachable (for example `http://localhost:8080`). +- `claude` binary on PATH or available by absolute path. +- `go` installed for shim build (first run only). +- Shell startup files are optional; dry-run mode can be used first. + +## Fast setup flow + +From repository root, run: + +```sh +chmod +x scripts/claude-datafog-setup.sh +./scripts/claude-datafog-setup.sh --policy-url http://localhost:8080 +``` + +The script: + +- Builds `datafog-shim` (if needed), +- Installs a managed shim named `claude`, +- Writes a helper env file `~/.datafog/claude-datafog.env`, +- Shows a minimal activation checklist. + +## Activation steps + +After bootstrap: + +```sh +source ~/.datafog/claude-datafog.env +export PATH="$HOME/.datafog/shims:$PATH" +``` + +Expected behavior after this is that running `which claude` should resolve to the shim path in `~/.datafog/shims`. + +## Verification + +Run: + +```sh +DATAFOG_SHIM_API_TOKEN="" claude --help +``` + +With policy defaults in place, if there is a matching policy rule for the command action metadata: +- allow/allow_with_redaction: command executes and emits decision info, +- deny/transform: command is blocked with a visible `PolicyDecisionError` in shim output. + +Audit evidence can be checked from sink: + +```sh +tail -f ~/.datafog/decisions.ndjson +``` + +Expected NDJSON events include action type, tool `claude`, decision, and request IDs. + +## Optional hardening knobs + +- `--mode observe` for non-blocking rollout. +- `--mode enforced` for hard blocking. +- `--api-token` to enforce tokened policy API requests. +- `--install-git` to additionally gate `git` through the same shim family. + +## Recovery and escape hatch + +If a user is blocked during onboarding, run in observe mode to collect logs: + +```sh +./scripts/claude-datafog-setup.sh --policy-url http://localhost:8080 --mode observe +``` + +If needed, remove generated files: + +```sh +rm -f ~/.datafog/claude-datafog.env +datafog-shim hooks uninstall claude --force +``` + +## Reuse for repeated installs + +Keep a local alias in shell rc: + +```sh +alias datafog-claude-setup='cd /path/to/datafog-api && ./scripts/claude-datafog-setup.sh --policy-url http://localhost:8080' +``` + +Then iterate with: + +```sh +datafog-claude-setup +``` diff --git a/docs/runbooks/datafog-codex-agent-ux.md b/docs/runbooks/datafog-codex-agent-ux.md new file mode 100644 index 0000000..81ae3f9 --- /dev/null +++ b/docs/runbooks/datafog-codex-agent-ux.md @@ -0,0 +1,124 @@ +--- +title: "Datafog + Codex Agent UX setup" +use_when: "A developer wants a minimal, reliable setup flow for enforcing Datafog policy around OpenAI Codex agent actions." +called_from: + - he-implement + - he-spec + - he-review +--- + +# Datafog + Codex Agent UX setup + +This runbook captures the target user experience for creating a secure workflow that +adds policy checkpoints to OpenAI Codex without changing how users run Codex day-to-day. + +The end state is: +- a normal `codex` command still works, +- every side-effect action passes through `datafog-shim` before execution, +- policy decisions and enforcement mode are explicit and reversible, +- setup can be validated in under two minutes. + +## Why this UX is efficient + +The onboarding experience should minimize manual glue code. A user should not have to learn policy JSON or low-level policy API endpoints before they can safely try the feature. + +The flow is intentionally: + +1. Start policy service. +2. Run one bootstrap command. +3. Source one generated env file. +4. Keep PATH updated with shim dir. + +This makes "I have policy-aware coding agent behavior" a predictable sequence rather than a long shell script to memorize. + +## Prerequisites + +- `datafog-api` running and reachable (for example `http://localhost:8080`). +- `codex` binary on PATH or available by absolute path. +- `go` installed for shim build (first run only). +- Write access to shell startup files is optional: the runbook can stay in dry-run mode first. + +## Fast setup flow + +From repository root, run: + +```sh +chmod +x scripts/codex-datafog-setup.sh +./scripts/codex-datafog-setup.sh --policy-url http://localhost:8080 +``` + +The script: + +- Builds `datafog-shim` (if needed), +- Installs a managed shim named `codex`, +- Writes a helper env file `~/.datafog/codex-datafog.env`, +- Shows a minimal activation checklist. + +## Activation steps + +After bootstrap: + +```sh +source ~/.datafog/codex-datafog.env +export PATH="$HOME/.datafog/shims:$PATH" +``` + +Expected behavior after this is that running `which codex` should resolve to the shim path in `~/.datafog/shims`. + +## Verification + +Run: + +```sh +DATAFOG_SHIM_API_TOKEN="" codex --help +``` + +With policy defaults in place, if there is a matching policy rule for the command action metadata: +- allow/allow_with_redaction: command executes and emits decision info, +- deny/transform: command is blocked with a visible `PolicyDecisionError` in the shim output. + +Audit evidence can be checked by reading the configured sink: + +```sh +tail -f ~/.datafog/decisions.ndjson +``` + +Expected NDJSON events include action type, tool `codex`, decision, and request IDs. + +## Optional hardening knobs + +- `--mode observe` for non-blocking rollout. +- `--mode enforced` for hard blocking. +- `--api-token` to enforce tokened policy API requests. +- `--install-git` to additionally gate `git` through the same shim family. + +## Recovery and escape hatch + +If a user is blocked during onboarding, run the shim in observe mode to collect logs: + +```sh +./scripts/codex-datafog-setup.sh --policy-url http://localhost:8080 --mode observe +``` + +If needed, remove only generated files: + +```sh +rm -f ~/.datafog/codex-datafog.env +datafog-shim hooks uninstall codex --force +``` + +## Reuse for repeated installs + +Keep a local alias in shell rc: + +```sh +alias datafog-codex-setup='cd /path/to/datafog-api && ./scripts/codex-datafog-setup.sh --policy-url http://localhost:8080' +``` + +Then iterate with: + +```sh +datafog-codex-setup +``` + +This preserves a predictable bootstrap habit and makes team onboarding copy-pastable. diff --git a/docs/runbooks/merge-change.md b/docs/runbooks/merge-change.md new file mode 100644 index 0000000..00a2405 --- /dev/null +++ b/docs/runbooks/merge-change.md @@ -0,0 +1,28 @@ +--- +title: "Merge Change" +use_when: "You have a GO decision and need the minimum merge gate (checks/approvals/evidence) before merging to the main branch." +called_from: + - he-verify-release +--- + +# Merge Change + +This runbook is repo-specific and **additive only**. It must not waive or override any gates enforced by skills. + +This runbook captures the repo-specific merge gate. Keep it short and make it objective where possible. + +## Preconditions + +See `he-github` SKILL.md § Merge for the canonical merge gate. Add repo-specific preconditions below. + +## Merge Checklist (Customize Per Repo) + +- Required approvals obtained +- Required checks passing +- Versioning/release notes updated (if applicable) +- Post-merge verification steps queued (see `docs/runbooks/verify-release.md`) + +## Post-Merge + +- Run the post-release checks documented in the plan +- If any regression is found, open a follow-up and record it in learnings diff --git a/docs/runbooks/pull-request.md b/docs/runbooks/pull-request.md new file mode 100644 index 0000000..d835a8e --- /dev/null +++ b/docs/runbooks/pull-request.md @@ -0,0 +1,45 @@ +--- +title: "Pull Request" +use_when: "You need to open or update a PR that links the initiative plan and evidence, and you want a consistent PR hygiene/checks workflow." +called_from: + - he-github + - he-implement + - he-review +--- + +# Pull Request + +This runbook is repo-specific and **additive only**. It must not waive or override any gates enforced by skills. + +This runbook describes repo-specific PR conventions (title/body conventions, labels, reviewers, and required checks). + +## Preflight + +- `git status --short --branch` +- `git diff` +- `gh auth status` + +## Create Or Update PR (Customize Per Repo) + +Recommended `gh` flow: + +- Push: + - `git push -u origin HEAD` +- Create: + - `gh pr create --fill` +- Update: + - `gh pr edit --body-file ` + +## Required Links In PR Description + +- Spec: `docs/specs/-spec.md` +- Plan: `docs/plans/active/-plan.md` +- Evidence (if any): `docs/artifacts//...` + +## Checks + +- View checks: + - `gh pr checks` +- View a failing run: + - `gh run view --log-failed` + diff --git a/docs/runbooks/record-evidence.md b/docs/runbooks/record-evidence.md new file mode 100644 index 0000000..7ded5f4 --- /dev/null +++ b/docs/runbooks/record-evidence.md @@ -0,0 +1,43 @@ +--- +title: "Record Evidence" +use_when: "You need screenshots or short recordings as proof of failure and proof of resolution, especially for UI or behavior changes." +called_from: + - he-video + - he-verify-release + - he-implement +--- + +# Record Evidence + +This runbook is repo-specific and **additive only**. It must not waive or override any gates enforced by skills. + +Evidence should be easy to review, easy to find, and tied to an artifact (plan/PR) so it does not get lost. + +## What To Capture + +- Failure evidence: what is broken, with a minimal reproduction +- Resolution evidence: the same reproduction after the fix +- Any relevant logs or error output (short) + +## Where To Put It + +- Link evidence from: + - `docs/plans/active/-plan.md` under `Artifacts and Notes` and `Verify/Release Decision` + - the PR description (if one exists) + +## Naming Convention + +Use predictable names so evidence is searchable: + +- `-failure.` +- `-resolution.` + +If multiple clips exist: + +- `-failure-1.`, `-resolution-1.` + +## Minimum Bar + +- If you claim a bug exists, there is at least one artifact showing it. +- If you claim it is fixed, there is at least one artifact showing the fix under the same scenario. +- Prefer short clips (10-60s) over long walkthroughs. diff --git a/docs/runbooks/reproduce-bug.md b/docs/runbooks/reproduce-bug.md new file mode 100644 index 0000000..789b709 --- /dev/null +++ b/docs/runbooks/reproduce-bug.md @@ -0,0 +1,42 @@ +--- +title: "Reproduce Bug" +use_when: "You have a bug report and need a minimal, reliable reproduction with evidence before implementing a fix." +called_from: + - he-implement + - he-video +--- + +# Reproduce Bug + +This runbook is repo-specific and **additive only**. It must not waive or override any gates enforced by skills. + +The goal is a smallest-possible reproducer you can run repeatedly to prove the bug exists and prove it is fixed. + +## Repro Checklist + +1. Write down the expected behavior vs observed behavior in plain language. +2. Reduce to one of: + - a single command (unit/integration test, script, request), or + - a single UI flow script (agent-browser), or + - a single minimal fixture (input file, request payload). +3. Make it deterministic: + - pin any randomness, time, or external dependencies when possible + - record env/config assumptions + +## Evidence Capture + +- For UI/behavior: capture a short `failure` video via `he-video`. +- For non-UI: capture terminal output (command + short excerpt) and link it in the plan. + +## Test Strategy (Preferred) + +- Add a real unit or e2e test that fails on the current state. +- Avoid mock-only tests unless the repo explicitly documents an exception. + +## Plan Updates + +Update `docs/plans/active/-plan.md`: + +- `Progress`: add/mark the repro artifact as complete only when repeatable +- `Artifacts and Notes`: link the repro command/script and evidence paths + diff --git a/docs/runbooks/respond-to-feedback.md b/docs/runbooks/respond-to-feedback.md new file mode 100644 index 0000000..8b55ecf --- /dev/null +++ b/docs/runbooks/respond-to-feedback.md @@ -0,0 +1,42 @@ +--- +title: "Respond To Feedback" +use_when: "A PR has review comments or requested changes, and you need a consistent loop to address them with evidence and minimal diffs." +called_from: + - he-github + - he-review + - he-implement +--- + +# Respond To Feedback + +This runbook is repo-specific and **additive only**. It must not waive or override any gates enforced by skills. + +Treat feedback as new requirements. The objective is to address comments with the smallest correct change and keep the plan/evidence accurate. + +## Triage + +1. Group comments by theme (correctness, security/data, architecture, taste). +2. Identify which comments require code changes vs explanation-only. +3. For any comment that is ambiguous or high risk, escalate per `he-review` SKILL.md § Escalation. + +## Commands (Recommended) + +- Read comments: + - `gh pr view --comments` +- Re-check CI: + - `gh pr checks` +- Pull failed logs: + - `gh run view --log-failed` + +## Fix Loop + +1. Make the root-cause fix. +2. Update tests/e2e evidence as needed. +3. Update the active plan: + - `Progress` items + - `Review Findings` (if you’re tracking findings there) + - `Artifacts and Notes` +4. Push and re-check (when approved): + - `git push` + - `gh pr checks` + diff --git a/docs/runbooks/review-findings.md b/docs/runbooks/review-findings.md new file mode 100644 index 0000000..4d70ecb --- /dev/null +++ b/docs/runbooks/review-findings.md @@ -0,0 +1,31 @@ +--- +title: "Review Findings" +use_when: "Writing or interpreting review findings in docs/plans/active/-plan.md under the Review Findings section." +called_from: + - he-review +--- + +# Review Findings + +This runbook is repo-specific and **additive only**. It must not waive or override any gates enforced by skills. + +Review findings must be actionable and verifiable. The goal is to let a future reader fix issues without rediscovering context. + +## Required Fields + +Each finding includes: + +- priority: `critical|high|medium|low` +- location: file path + symbol or short pointer +- issue summary: what is wrong +- required action: what must change or what proof is missing +- owner: who is responsible (team/name/agent) + +## Priority Rubric, No-Mocks Policy, Mandatory Coverage + +Canonical definitions live in `he-review` SKILL.md. Add repo-specific examples or exceptions below — do not redefine the severity levels or gate rules. + +## Acceptance Rules + +- Unresolved `critical` or `high` blocks progression to verify/release. +- `medium` and `low` can proceed only if explicitly accepted in writing in the plan. diff --git a/docs/runbooks/update-agents-md.md b/docs/runbooks/update-agents-md.md new file mode 100644 index 0000000..a24e0d1 --- /dev/null +++ b/docs/runbooks/update-agents-md.md @@ -0,0 +1,59 @@ +--- +title: "Update AGENTS.md" +use_when: "Creating or updating a project's AGENTS.md (agent instructions, conventions, and workflows)." +called_from: + - he-bootstrap + - he-learn + - he-doc-gardening +--- + +# Update AGENTS.md + +This runbook is repo-specific and **additive only**. It must not waive or override any gates enforced by skills. + +AGENTS.md is the agent-facing README: a predictable place to put the few repo-specific instructions an agent needs to work effectively. + +When `he-bootstrap` runs in a repo that already has `AGENTS.md`, it appends a managed block once using: + +- `` +- `` + +Do not edit outside your repo's intended scope when touching this managed block. + +## What To Optimize For + +- Keep it short and stable (a map, not an encyclopedia). +- Put only high-leverage, repo-specific guidance here: build/test commands, conventions, and hard constraints. +- Add rules over time when you observe repeated failure modes; do not try to predict everything up front. + +## What To Put Elsewhere + +- Long procedures, checklists, and evolving processes: `docs/runbooks/.md` and link from AGENTS.md. +- One-off migrations or multi-hour work: a plan/spec doc under `docs/` (not in AGENTS.md). + +## Minimum Sections (Good Starting Point) + +- Setup commands (install, dev, test, lint) in copy-pastable form. +- Repo map (where the important stuff lives; key entrypoints). +- Conventions (formatting, naming, dependency rules, boundaries). +- Safety and verification (what not to do; how to prove the change works here). +- Runbook index (links into `docs/runbooks/` for process). + +## Rules Of Thumb When Editing AGENTS.md + +- If it changes often, it probably belongs in a runbook, not AGENTS.md. +- Prefer "When X, do Y" over vague guidance. +- Make requirements verifiable (a command, a file path, an expected output). +- Avoid duplicating information already in `docs/`; link instead. +- Keep any `he-bootstrap` managed block concise and link-first to avoid disrupting existing user conventions. + +## Quick Update Checklist + +1. Confirm scope: are you editing the right AGENTS.md for the files you are touching (root vs nested)? +2. Keep it minimal: can you replace paragraphs with a link to a runbook? +3. Verify paths/commands exist: + +```sh +rg -n "docs/runbooks|PLANS\\.md|Runbooks|Setup|test|lint" AGENTS.md +find docs/runbooks -type f -maxdepth 2 -name "*.md" -print +``` diff --git a/docs/runbooks/update-domain-docs.md b/docs/runbooks/update-domain-docs.md new file mode 100644 index 0000000..239704c --- /dev/null +++ b/docs/runbooks/update-domain-docs.md @@ -0,0 +1,42 @@ +--- +title: "Update Domain Docs" +use_when: "A change introduces new product/engineering policy (security, reliability, frontend, observability, design) that should be captured as durable guidance for future work." +called_from: + - he-plan + - he-implement + - he-learn + - he-doc-gardening +--- + +# Update Domain Docs + +This runbook is repo-specific and **additive only**. It must not waive or override any gates enforced by skills. + +Domain docs live under `docs/` and capture stable, repo-specific policy. Update them when you learn something that will prevent future bugs, regressions, or confusion. + +## What Counts As A Domain-Doc Change + +- A recurring decision rule ("we always do X when Y"). +- A new constraint or boundary (security model, data sensitivity, performance guardrails). +- An operational expectation (SLOs, monitoring, alerts, rollback rules). +- A UI/UX standard or accessibility requirement. + +If it's a one-off procedure or checklist, prefer a runbook in `docs/runbooks/`. + +## Where To Put It + +- The registry: `docs/DOMAIN_DOCS.md` (what exists and why). +- The doc itself (when present): `docs/SECURITY.md`, `docs/RELIABILITY.md`, `docs/FRONTEND.md`, `docs/OBSERVABILITY.md`, `docs/DESIGN.md`, `docs/PRODUCT_SENSE.md`. + +## How To Update (Minimum) + +1. Add the smallest rule that will prevent the problem from recurring (short, testable language). +2. Include a concrete anchor: + - a file path, command, config key, or observable behavior. +3. Avoid long procedures; link to a runbook if needed. +4. If the change implies enforcement, note the guardrail candidate (lint/test/CI gate) so it can be promoted later. + +## When To Do This + +- During `he-learn`, when converting "what happened" into durable prevention. +- During review, if you discover undocumented constraints the next contributor will trip over. diff --git a/docs/runbooks/validate-current-state.md b/docs/runbooks/validate-current-state.md new file mode 100644 index 0000000..a298df2 --- /dev/null +++ b/docs/runbooks/validate-current-state.md @@ -0,0 +1,41 @@ +--- +title: "Validate Current State" +use_when: "Starting an initiative and you need to confirm you understand the current behavior, repo state, and baseline signals before changing code." +called_from: + - he-workflow + - he-implement +--- + +# Validate Current State + +This runbook is repo-specific and **additive only**. It must not waive or override any gates enforced by skills. + +This runbook defines the minimum baseline checks before claiming you understand "what's broken" (or "what exists") today. + +## Repo Baseline + +- Confirm you are in the intended workspace (worktree/branch): + - `git status --short --branch` +- Confirm clean-ish state (or record intentional local changes): + - `git diff` +- Confirm remote + default branch context: + - `git remote -v` + +## Behavior Baseline (Customize Per Repo) + +Record the exact commands used and a short excerpt of the output in the active plan. + +- Boot the app/service: + - `` +- Run the fastest “is it alive” check: + - `` +- Run targeted tests for the area (if they exist): + - `` + +## Evidence + +Link evidence from `docs/plans/active/-plan.md` under: + +- `Surprises & Discoveries` (what you observed) +- `Artifacts and Notes` (logs, screenshots, recordings) + diff --git a/docs/runbooks/verify-release.md b/docs/runbooks/verify-release.md new file mode 100644 index 0000000..99a6fe4 --- /dev/null +++ b/docs/runbooks/verify-release.md @@ -0,0 +1,69 @@ +--- +title: "Verify/Release" +use_when: "Running he-verify-release to decide GO/NO-GO with evidence, rollback readiness, and post-release checks recorded in the active plan." +called_from: + - he-verify-release +--- + +# Verify/Release + +This runbook is repo-specific and **additive only**. It must not waive or override any gates enforced by skills. + +The skill `he-verify-release` enforces the stable invariants; this document carries the details that change per project. Inputs: active plan (`docs/plans/active/-plan.md` with `## Verify/Release Decision`) and review findings (populated by `he-review`). + +## Output + +Fill in `## Verify/Release Decision` with: + +- decision: `GO` or `NO-GO` +- date: +- open findings by priority (if any): +- evidence: links/paths to test output and E2E artifacts +- rollback: exact steps or pointers +- post-release checks: exact checks/queries/URLs +- owner: + +## Verification Ladder (Customize Per Repo) + +Define the repo's minimum ladder here. Keep it short and ordered. + +1. Fast checks: format/lint/typecheck (if applicable) +2. Targeted tests for changed area +3. Full relevant suite (unit/e2e) +4. Manual/E2E scenario (required for user-visible changes) + +Document the exact commands for this repo: + + # From repo root: + + +## Evidence Requirements + +- Prefer evidence that a reviewer can reproduce (commands + short transcripts). +- For UI changes, include screenshots or a short recording (see `docs/runbooks/record-evidence.md`). +- For regressions, include a "before vs after" behavior description in plain language. + +## Rollback And Recovery + +Record the rollback plan for this repo: + +- What to revert (commit/flag/config) +- How to detect failure +- How to restore service/data (if relevant) + +## Post-Release Checks + +Record the minimum set of checks to run after merge/release: + +- health checks / smoke path +- key metrics / dashboards (if any) +- error logs / alerts (if any) + +## Escalation + +If any of these apply, stop and escalate per `he-verify-release` SKILL.md § Escalation: + +- Unclear risk to users/data +- Flaky or non-deterministic failures +- Rollback steps are missing or untested +- Evidence is incomplete but time pressure exists diff --git a/docs/scalar.html b/docs/scalar.html new file mode 100644 index 0000000..d752d2f --- /dev/null +++ b/docs/scalar.html @@ -0,0 +1,397 @@ + + + + DataFog API Reference + + + + + + + + diff --git a/docs/specs/2026-02-24-feat-interactive-demo-ui-spec.md b/docs/specs/2026-02-24-feat-interactive-demo-ui-spec.md new file mode 100644 index 0000000..c320bb6 --- /dev/null +++ b/docs/specs/2026-02-24-feat-interactive-demo-ui-spec.md @@ -0,0 +1,89 @@ +--- +slug: 2026-02-24-feat-interactive-demo-ui +plan_mode: lightweight +spike_recommended: no +status: active +owner: sidmohan +created: 2026-02-24 +--- + +# Interactive Demo UI + +## Purpose + +Provide a single-page interactive playground that lets anyone with the server running on localhost see the DataFog API in action — scan PII, view policy decisions, and compare all four transform modes — without leaving the browser or reading docs first. + +## Scope + +### In scope + +- Single self-contained HTML file (`docs/demo.html`), no build tools or dependencies beyond a CDN-hosted minimal CSS (or inline styles) +- Pre-populated sample text containing all 5 entity types (email, phone, SSN, API key, credit card) +- Editable text input so users can paste their own content +- **Scan panel**: call `POST /v1/scan`, display detected entities with type badges, confidence scores, and highlighted positions in the source text +- **Decide panel**: configurable action type/tool/command fields, call `POST /v1/decide`, display decision (allow / deny / transform / allow_with_redaction), matched rules, and transform plan +- **Transform panel**: toggle between all 4 modes (mask, tokenize, anonymize, redact), call `POST /v1/transform`, show transformed output with a visual diff against the original +- One-click "Run All" button that executes the full pipeline (scan → decide → transform) sequentially and populates all panels +- Connection status indicator (hits `GET /health` on load, shows server version and policy info) +- Works against `http://localhost:8080` (server must already have CORS support — already shipped) + +### Boundaries + +- No authentication UI — demo assumes no `DATAFOG_API_TOKEN` is set (local dev default) +- No receipt viewer — receipts are an audit concern, not a demo concern +- No server-side changes — the HTML file is purely a client; the API is unchanged +- No build step, no npm, no bundler — vanilla HTML/CSS/JS only +- No persistent state — page reload resets everything +- Mobile responsiveness is nice-to-have, not required + +## Requirements + +| ID | Requirement | Priority | +|----|-------------|----------| +| R1 | Page loads with sample text pre-filled containing at least one of each entity type (email, phone, SSN, API key, credit card) | must | +| R2 | User can edit the text freely before running any operation | must | +| R3 | "Scan" button calls `POST /v1/scan` and renders findings as highlighted spans in the text with entity type labels and confidence | must | +| R4 | "Decide" button calls `POST /v1/decide` with user-configurable action fields (type, tool, command) and displays the decision, matched rules, reason, and any transform plan | must | +| R5 | "Transform" section lets the user pick a mode (mask / tokenize / anonymize / redact) or per-entity modes, calls `POST /v1/transform`, and shows the transformed output | must | +| R6 | "Run All" button executes scan → decide → transform in sequence, populating all panels with a single click | must | +| R7 | Each API call shows a loading state and elapsed time | should | +| R8 | Errors from the API are displayed inline with the error code and message | must | +| R9 | On page load, `GET /health` is called; connection status + policy version shown in a header bar | should | +| R10 | Visual diff between original text and transformed output (e.g., side-by-side or inline highlights showing what changed) | should | + +## Success Criteria + +1. A new user can open `docs/demo.html` in a browser, click "Run All", and see scan results, a policy decision, and transformed text within 3 seconds (given a running server). +2. All 5 entity types are visually distinguishable in the scan results. +3. Switching transform modes re-runs the transform and updates the output without re-scanning. +4. The file is a single `demo.html` with zero external dependencies beyond optional CDN CSS — works offline if the CDN is cached. +5. `go test ./...` continues to pass (no server-side changes). + +## Constraints + +- Must work on Chrome, Edge, Firefox (current versions) +- No server-side changes — client-only HTML file +- File size should stay under 30 KB to keep it easy to review in a PR +- Must handle server-down gracefully (show connection error, don't break the page) + +## Priority + +**high** — this is the first thing someone sees when evaluating the project locally. + +## Initial Milestone Candidates + +| ID | Milestone | Observable outcome | Risk | +|----|-----------|--------------------|------| +| M1 | Page skeleton + health check | HTML file loads, shows server connection status and policy version in header | Low | +| M2 | Scan panel | Editable text area, "Scan" button, findings rendered with highlights and badges | Low | +| M3 | Decide panel | Action fields (type/tool/command), "Decide" button, decision + matched rules displayed | Low | +| M4 | Transform panel | Mode selector, "Transform" button, output with visual diff | Medium — diff rendering | +| M5 | Run All pipeline + polish | Sequential execution, loading states, timing, error handling, final styling pass | Low | + +## Handoff + +After approval, proceed to `he-plan` for implementation planning. No spike needed — the API contract is stable, CORS is already in place, and the scope is well-bounded. + +## Revision Notes + +- v1: Initial spec from interactive session. diff --git a/docs/specs/2026-02-24-feat-v2-mvp-complete-spec.md b/docs/specs/2026-02-24-feat-v2-mvp-complete-spec.md new file mode 100644 index 0000000..3ecdb2f --- /dev/null +++ b/docs/specs/2026-02-24-feat-v2-mvp-complete-spec.md @@ -0,0 +1,168 @@ +--- +slug: 2026-02-24-feat-v2-mvp-complete +plan_mode: execution +spike_recommended: no +status: active +owner: sidmohan +created: 2026-02-24 +--- + +# DataFog API v2 — Complete MVP + +## Purpose + +Close every gap between the current v2 branch (a working but incomplete policy gating prototype) and a shippable MVP that can detect PII at parity with datafog-python, enforce policy decisions with real data transformation, demonstrate end-to-end execution in a live demo, and integrate cleanly into Claude Code and OpenAI Codex via the existing adapter branches. + +## Current State (Honest Baseline) + +| Component | What works | What's missing | +|---|---|---| +| Scanner | 5 regex patterns, static confidence | No NER, no IP/date/zip, no PERSON/ORG, no Luhn, no multi-engine | +| Policy engine | Priority-sorted string-equality rules | No pattern matching, adapter registry not wired in | +| Transforms | mask/redact/tokenize/anonymize on API | Shim ignores transform plans — only does allow/deny | +| Shim | Real command gating via exec, calls /v1/decide | Doesn't apply redaction before allowing, events write-only | +| Receipts | JSONL to disk, survives restart | No querying beyond by-ID, no rotation | +| Demo | Static HTML calling API, "what-if" only | No real command execution, no shim integration | +| Platform adapters | v2-claude and v2-codex branches exist | Not merged, need base branch to be complete first | + +## Scope + +### In scope + +**WS1 — PII Detection Parity** +- Expand entity types to 10+: email, phone, SSN, credit card, API key, IP address (IPv4), date (common formats), zip code (US 5-digit/ZIP+4), PERSON, ORGANIZATION/LOCATION +- Regex engine for structured entities with real validation (Luhn check for credit cards, IP range validation) +- Go-native NER engine for unstructured entities (PERSON, ORG, LOCATION) — dictionary/heuristic-based or lightweight model, no Python/cgo dependency +- Multi-engine cascade: regex (fast, always available) → NER (when enabled) +- Graceful degradation: NER unavailable → regex-only with warning +- Three anonymization strategies: redact (`[REDACTED]`), replace/pseudonymize (`[PERSON_A1B2C3]`), hash (SHA256) +- Selective entity filtering on all scan/transform endpoints +- Confidence scores: 1.0 for regex matches, configurable threshold for NER + +**WS2 — Enforcement Gap Closure** +- Shim applies transform plans on `allow_with_redaction` decisions: stdin/file content redacted before command executes +- Wire adapter registry into policy rules (rules can match on canonical adapter names, not just raw strings) +- Events endpoint: `GET /v1/events` — query decision events with filters (time range, decision type, adapter) +- Disk-backed idempotency cache (survives restart) +- Receipt rotation/archival (configurable max size, auto-rotate) + +**WS3 — Interactive Demo with Real Execution** +- Demo server: new Go HTTP handler (can be a mode of the existing binary or separate `cmd/datafog-demo`) +- `POST /demo/exec` — run a shell command through the shim, return decision + stdout/stderr +- `POST /demo/write-file` — write content through the shim to a sandbox, return decision + result +- `POST /demo/read-file` — read a file through the shim, return decision + content (or redacted content) +- Sandboxed temp directory for all demo file operations +- Updated `docs/demo.html` UI: command input, file operation panel, real pipeline visualization +- Shows: input → scan findings → policy decision → enforcement action → real output + +**WS4 — Platform Integration Readiness** +- Ensure adapter registry includes `claude` and `codex` as recognized adapters with aliases +- Policy rules can target `tools: ["claude", "codex"]` and `adapters: ["claude", "codex"]` +- Base branch supports the hook installation pattern used by both v2-claude and v2-codex +- Validate that both setup scripts (`scripts/claude-datafog-setup.sh`, `scripts/codex-datafog-setup.sh`) work against the updated base +- Merge or rebase v2-claude and v2-codex onto the completed v2 branch + +### Boundaries + +- No Python dependencies or cgo — pure Go +- No ML model training or fine-tuning — NER is dictionary/heuristic or pre-built lookup +- No OCR or image-based PII detection (datafog-python has this but it's out of scope for Go API MVP) +- No distributed processing (Spark equivalent) +- No persistent database — JSONL/file-based storage is acceptable for MVP +- No authentication UI — API token via env var is sufficient +- No cloud deployment automation — local/Docker is sufficient +- Demo sandbox is ephemeral — no persistent demo state +- Platform adapter testing requires actual Claude/Codex binaries installed (can't be automated in CI without them) + +## Requirements + +| ID | Requirement | Priority | Workstream | +|----|-------------|----------|------------| +| R1 | Detect 10+ entity types: email, phone, SSN, credit card, API key, IP address, date, zip code, PERSON, ORGANIZATION | must | WS1 | +| R2 | Credit card detection includes Luhn validation to reduce false positives | must | WS1 | +| R3 | IP address detection validates range (0-255 per octet) | must | WS1 | +| R4 | Go-native NER for PERSON/ORG/LOCATION without Python or cgo | must | WS1 | +| R5 | Multi-engine cascade: regex first, NER second, configurable via env var | must | WS1 | +| R6 | Three anonymization strategies: redact, replace (pseudonymize with entity-typed tokens), hash (SHA256) | must | WS1 | +| R7 | Selective entity filtering: caller can specify which entity types to detect/transform | must | WS1 | +| R8 | Backward compatible: existing /v1/scan, /v1/decide, /v1/transform, /v1/anonymize contracts unchanged | must | WS1 | +| R9 | Shim applies transform plans on allow_with_redaction — content is actually redacted before command executes | must | WS2 | +| R10 | Adapter registry wired into policy: rules can match on canonical adapter names | must | WS2 | +| R11 | Events endpoint: GET /v1/events with time range and decision type filters | should | WS2 | +| R12 | Disk-backed idempotency cache survives process restart | should | WS2 | +| R13 | Receipt store supports rotation (configurable max entries or file size) | should | WS2 | +| R14 | Demo server exposes /demo/exec, /demo/write-file, /demo/read-file endpoints | must | WS3 | +| R15 | Demo operations run through the shim gate (real policy enforcement, not simulated) | must | WS3 | +| R16 | Demo file operations use a sandboxed temp directory (auto-cleaned) | must | WS3 | +| R17 | Demo UI shows full pipeline: input → findings → decision → enforcement → output | must | WS3 | +| R18 | Adapter registry includes claude and codex with correct aliases | must | WS4 | +| R19 | Policy rules support adapter-based matching for claude and codex | must | WS4 | +| R20 | v2-claude and v2-codex bootstrap scripts work against updated v2 base | must | WS4 | +| R21 | All existing tests continue to pass; new functionality has test coverage | must | All | + +## Success Criteria + +1. `go test ./...` passes with 0 failures on the completed branch. +2. Scanning text containing all 10+ entity types returns correct findings with appropriate confidence scores. +3. Running `datafog-shim shell git push` with PII in the working directory triggers `allow_with_redaction` and the PII is actually redacted in the output — not just flagged. +4. The demo UI at `docs/demo.html` can execute a real command, show the real decision, and display real stdout/blocked output. +5. `datafog-shim hooks install claude` and `datafog-shim hooks install codex` both succeed and create correct wrapper scripts. +6. The PR from v2 to dev passes CI (gofmt, go vet, go test, gosec). + +## Constraints + +- Go 1.22+ (module spec), CI uses 1.24+ +- No external service dependencies (self-contained binary) +- API contract backward compatible (new fields allowed, existing fields stable) +- File size for demo.html under 50KB (was 30KB, allowing growth for new features) +- Demo server must not expose shell execution without explicit opt-in flag (security) + +## Risks + +| Risk | Impact | Mitigation | +|------|--------|------------| +| Go-native NER accuracy for PERSON/ORG may be significantly lower than spaCy/GLiNER | Medium | Start with dictionary + heuristic approach; document accuracy tradeoff; design for pluggable engine so ML can be added later | +| Shim transform-on-enforcement may break piped command workflows | High | Observe mode as default for first release; enforce mode opt-in | +| Demo shell execution endpoint is a security surface | High | Require explicit `--enable-demo` flag; sandbox all operations; never expose on non-localhost | +| Merging v2-claude/v2-codex may have conflicts with WS1-WS3 changes | Low | Merge adapter branches last after base is stable | + +## Priority + +**critical** — this is the MVP gate for the entire DataFog v2 product line. Both adapter branches are blocked on this. + +## Initial Milestone Candidates + +| ID | Milestone | Observable outcome | Risk | +|----|-----------|--------------------|------| +| M1 | Scanner parity | 10+ entity types detected, Luhn/IP validation, regex engine complete | Low | +| M2 | NER engine | PERSON/ORG/LOCATION detected via Go-native approach, cascade works | Medium — accuracy | +| M3 | Anonymization strategies | replace + hash modes added alongside existing redact/mask/tokenize/anonymize | Low | +| M4 | Shim enforcement | allow_with_redaction actually redacts, transform plans applied | Medium — piping | +| M5 | Policy + adapter wiring | Adapter registry in rules, claude/codex adapters registered | Low | +| M6 | Events + persistence | Events endpoint, disk-backed idempotency, receipt rotation | Low | +| M7 | Demo server + UI | Real execution demo with command/file panels, sandboxed ops | Medium — security | +| M8 | Platform integration | v2-claude and v2-codex merged/rebased, setup scripts validated | Low | +| M9 | PR + CI | All tests pass, PR to dev created, CI green | Low | + +## Key Decisions + +1. **Go-native NER over cgo/Python**: Accepting lower accuracy for PERSON/ORG in exchange for single-binary deployment and zero external dependencies. Can be upgraded later with a pluggable engine interface. +2. **Demo requires explicit opt-in**: `--enable-demo` flag prevents accidental shell execution exposure. Demo endpoints only bind on localhost. +3. **Workstream sequencing**: WS1 (scanner) → WS2 (enforcement) → WS3 (demo) → WS4 (adapters). Each builds on the previous. + +## Reference Artifacts + +- DataFog Python repo: https://github.com/DataFog/datafog-python (v4.3.0, PII detection baseline) +- Existing API contract: `docs/contracts/datafog-api-contract.md` +- v2-claude branch: `origin/codex/v2-claude` +- v2-codex branch: `origin/codex/v2-codex` +- Current demo UI: `docs/demo.html` +- Honest v2 audit: conversation context (2026-02-24) + +## Handoff + +After approval, proceed directly to `he-plan` then `he-implement`. No spike needed — all unknowns are resolvable during planning. The implementer has full autonomy to commit, push, and create a PR to dev. + +## Revision Notes + +- v1: Initial spec from comprehensive audit of v2 state, datafog-python research, and adapter branch analysis. diff --git a/docs/specs/2026-02-24-feat-v2.1-ner-sidecar-spec.md b/docs/specs/2026-02-24-feat-v2.1-ner-sidecar-spec.md new file mode 100644 index 0000000..c3bcb2d --- /dev/null +++ b/docs/specs/2026-02-24-feat-v2.1-ner-sidecar-spec.md @@ -0,0 +1,152 @@ +--- +slug: 2026-02-24-feat-v2.1-ner-sidecar +plan_mode: lightweight +spike_recommended: no +status: draft +owner: sidmohan +created: 2026-02-24 +--- + +# feat: Optional NER sidecar for GLiNER2-grade entity detection + +## Purpose + +The v2 shim uses regex + heuristic NER (48-name dictionary, 19 org suffixes, 46-location dictionary). This covers the primary threat model — secrets and structured PII leaking through AI agents — at near-zero latency. But it misses unstructured PII: arbitrary person names ("Satya Nadella"), orgs without suffixes ("Microsoft"), and locations not in the dictionary ("Springfield"). + +Teams in healthcare, finance, and HR need stronger NER for compliance. Rather than bloating the shim binary with a 900MB+ ONNX model and cgo dependency, we add an **optional sidecar** NER service. When configured, the shim sends content to it for enhanced detection. When not configured, the shim falls back to the existing heuristic NER with zero overhead. + +## Scope + +### In scope + +- **Sidecar HTTP contract** — `/v1/ner` endpoint accepting text, returning entity spans in the same `ScanFinding` format the shim already uses +- **Shim integration** — new `DATAFOG_NER_ENDPOINT` config; when set, the shim calls the sidecar during scan phase and merges results with regex findings +- **GLiNER2 reference implementation** — Python container running `urchade/gliner_large-v2` behind a thin FastAPI/Flask server, with Dockerfile +- **Graceful degradation** — sidecar unavailable → fall back to heuristic NER, log warning, continue (same pattern as observe mode) +- **Entity type pass-through** — the shim tells the sidecar which entity types to detect (person, organization, location, or custom labels), leveraging GLiNER2's zero-shot capability +- **Latency budget** — configurable timeout per sidecar call (default 200ms); if exceeded, use heuristic results + +### Boundaries + +- No embedding the model in the Go binary (ruled out — cgo, binary size, memory) +- No changes to the regex engine or existing heuristic NER +- No training or fine-tuning of GLiNER2 +- No mandatory dependency — the sidecar is fully optional +- No gRPC — HTTP/JSON only for simplicity; gRPC is a future optimization +- No batching in v2.1 — one request per scan call; batching is a future optimization + +## Requirements + +| ID | Requirement | Priority | +|----|-------------|----------| +| R1 | Shim reads `DATAFOG_NER_ENDPOINT` and `DATAFOG_NER_TIMEOUT` from env | must | +| R2 | When endpoint is set, `ScanText` sends content to sidecar and merges NER findings with regex findings | must | +| R3 | When endpoint is unset or empty, behavior is identical to v2 (no sidecar call) | must | +| R4 | Sidecar timeout defaults to 200ms; on timeout, fall back to heuristic NER | must | +| R5 | Sidecar request includes entity type filter so only requested types are inferred | should | +| R6 | Sidecar response uses the same `ScanFinding` schema (entity_type, value, start, end, confidence) | must | +| R7 | Duplicate entity spans (same start/end from both regex and sidecar) are deduplicated, keeping highest confidence | should | +| R8 | Reference GLiNER2 sidecar ships as a Dockerfile in `sidecar/ner/` | must | +| R9 | Sidecar has its own `/health` endpoint for readiness probes | should | +| R10 | Shim logs sidecar latency per call at debug level | should | +| R11 | Demo scenario explorer gets a "NER sidecar" scenario showing enhanced detection | nice | + +## Sidecar API Contract + +### `POST /v1/ner` + +**Request:** +```json +{ + "text": "Meeting with Satya Nadella at Microsoft HQ in Redmond", + "entity_types": ["person", "organization", "location"] +} +``` + +**Response:** +```json +{ + "findings": [ + {"entity_type": "person", "value": "Satya Nadella", "start": 13, "end": 26, "confidence": 0.92}, + {"entity_type": "organization", "value": "Microsoft", "start": 30, "end": 39, "confidence": 0.89}, + {"entity_type": "location", "value": "Redmond", "start": 46, "end": 53, "confidence": 0.87} + ] +} +``` + +### `GET /health` + +Returns `{"status": "ok", "model": "gliner_large-v2"}`. + +## Integration Architecture + +``` +┌─────────────┐ ┌──────────────┐ ┌─────────────────┐ +│ AI Agent │────▶│ datafog-shim │────▶│ datafog-api │ +│ (Claude/Codex)│ │ (Go binary) │ │ (policy engine) │ +└─────────────┘ └──────┬───────┘ └─────────────────┘ + │ + ┌──────▼───────┐ + │ NER sidecar │ ← optional + │ (GLiNER2) │ + └──────────────┘ +``` + +**Scan flow when sidecar is configured:** + +1. `ScanText` runs regex phase (fast, <1ms) +2. `ScanText` runs heuristic NER phase (fast, <1ms) +3. `ScanText` calls sidecar with text + entity filter (50-200ms) +4. Merge: deduplicate overlapping spans, keep highest confidence +5. Return combined findings + +**Scan flow when sidecar is not configured (default):** + +1. `ScanText` runs regex phase +2. `ScanText` runs heuristic NER phase +3. Return findings (identical to v2 behavior) + +## Success Criteria + +1. `DATAFOG_NER_ENDPOINT` unset → all existing tests pass with zero behavior change +2. `DATAFOG_NER_ENDPOINT` set → "Satya Nadella" detected as person, "Microsoft" as organization +3. Sidecar down → shim logs warning, falls back to heuristic, does not block execution +4. Sidecar timeout (>200ms) → same fallback behavior +5. Reference sidecar container starts and passes `/health` check +6. End-to-end: agent writes file containing "Meeting with Satya Nadella at Microsoft" → sidecar detects entities → policy applies redaction + +## Constraints + +- Shim binary size must not increase by more than 1KB (HTTP client code only) +- No cgo, no ONNX in the Go binary +- Sidecar is Python — acceptable since it's a separate container, not a Go dependency +- Must work with existing policy rules (transform plans, entity requirements) + +## Risks + +| Risk | Impact | Mitigation | +|------|--------|------------| +| Sidecar latency makes agent interactions sluggish | High | 200ms timeout + fallback; async/fire-and-forget mode for observe-only deployments | +| GLiNER2 model accuracy on code-adjacent text | Medium | Code context is unusual for NER models; test with realistic coding payloads | +| Sidecar memory footprint (~1-2GB) | Medium | Document minimum requirements; make it optional | +| Network failure between shim and sidecar | Low | Already solved — same graceful degradation as observe mode | + +## Priority + +Medium — the regex engine covers the primary threat model. This is for teams with compliance requirements around unstructured PII. + +## Initial Milestone Candidates + +| ID | Milestone | Observable outcome | +|----|-----------|-------------------| +| M1 | Sidecar HTTP contract + reference GLiNER2 container | `docker run` → `/health` ok, `/v1/ner` returns findings for test text | +| M2 | Shim NER client + scan integration | `DATAFOG_NER_ENDPOINT` set → `ScanText` returns sidecar findings merged with regex | +| M3 | Graceful degradation + timeout | Sidecar killed → shim continues with heuristic NER, no errors propagated | + +## Handoff + +Spec is ready for planning. No spike needed — the integration pattern (HTTP call with timeout + fallback) already exists in the shim's policy client. The new work is the sidecar container and the scan-phase integration. + +## Revision Notes + +- 2026-02-24: Initial draft diff --git a/docs/specs/README.md b/docs/specs/README.md new file mode 100644 index 0000000..4673c3e --- /dev/null +++ b/docs/specs/README.md @@ -0,0 +1,5 @@ +# Specs + +Store initiative specs here using one file per slug. +Each spec must start with YAML frontmatter and set `plan_mode: lightweight|execution`. +Store spike findings separately in `docs/spikes/`. diff --git a/docs/specs/datafog-api-mvp-spec.md b/docs/specs/datafog-api-mvp-spec.md new file mode 100644 index 0000000..5342b42 --- /dev/null +++ b/docs/specs/datafog-api-mvp-spec.md @@ -0,0 +1,169 @@ +--- +slug: datafog-api-mvp +plan_mode: execution +status: active +owner: sidmohan +created: 2026-02-23 +--- + +# datafog-api Go MVP Specification + +## Purpose +Datafog API v2 will be a single Go service that owns policy decisioning and privacy transformations for agent action execution. It provides a stable API (`/v1/scan`, `/v1/decide`, `/v1/transform`, `/v1/anonymize`, `/v1/receipts/{id}`, `/v1/policy/version`, `/health`) and deterministic receipts for every side-effect decision. + +## Scope +- In scope + - Canonical policy schema, evaluator, and evaluation outcomes + - Canonical entity definitions and confidence-scored detection + - Request-scoped redaction/anonymization transforms + - Receipt/audit trail with traceability IDs + - Health/version endpoints for operational integration + - Golden tests + reproducible behavior tests +- Out of scope + - Full enterprise UI + - Native Python/TS client SDKs in this MVP release + - Multi-tenant authN/Z UI flows + +## User-visible behavior +- Agents and services call `/v1/scan` to get entity findings. +- Agents and services call `/v1/decide` before any side-effect action. +- `decide` returns one of `allow`, `allow_with_redaction`, `transform`, or `deny` and always includes `policy_version`, `receipt_id`, and rule trace. +- If evaluator cannot make a safe decision, response defaults to `deny` with `reason` explaining failure. +- `/v1/transform` and `/v1/anonymize` return sanitized payloads using policy-bound modes. +- `/v1/receipts/{id}` returns immutable decision/transform evidence for audit. +- `/health` returns service status, policy version, and startup timestamp. + +## Functional requirements + +### 1) Action and policy model +- Define canonical request models with stable fields: + - `action` metadata: `type`, `tool`, `resource`, `command`, `args`, `sensitive`. + - `context` metadata: `tenant_id`, `actor_id`, `session_id`, `trace_id`, `request_id`. +- Define policy rule model with deterministic matching: + - `id`, `description`, `priority`, `effect` + - `match` on `action.type`, optional `resource`, optional `tool`, optional `resource_prefix`, optional `commands`, and optional `args` (list of required args). + - `entity_requirements` for findings-driven gating + - `require_sensitive_only` to scope a rule to sensitive-marked actions + - `transform` list when effect is `transform`. +- Priority and conflict resolution: + - evaluate all matching rules, then apply deterministic precedence: `deny` first, then `transform`, then `allow_with_redaction`, then `allow`. + - unknown or empty action types fail closed to `deny`. + +### 2) Canonical detectors +- Add built-in entity detectors for MVP: + - `email`, `phone`, `ssn`, `api_key`, `credit_card`. +- All detections are deterministic with `[start, end)` offsets and confidence score. +- Detectors must never panic on empty or malformed UTF-8 strings. + +### 3) Decision endpoint (`/v1/decide`) +- `POST /v1/decide` +- Request pipeline: + 1. validate input + 2. run scan + 3. evaluate policy + 4. generate `receipt_id` + 5. persist receipt +- Decision response includes: + - `decision` + - `policy_version` + - `receipt_id` + - `matched_rules` + - `findings` + - `transform_plan` if required + +### 4) Transform endpoints +- `POST /v1/transform` applies masking or redaction strategies. +- `POST /v1/anonymize` applies irreversible pseudonymization for configured fields. +- Both include summary with operation counts and changed span count. + +### 5) Receipt and audit +- Persist receipts in-process via file-backed append log `datafog_receipts.jsonl`. +- Receipt includes: action hash, input hash, policy id/version, decision, rule ids, timestamps, and optional sanitized summary. +- Receipts are immutable by API contract (write-once append only). + +### 6) Versioning and rollout +- Return policy version from a pinned local policy snapshot file. +- API returns `policy_version` in all relevant responses. + +### 7) Error model +- Standard JSON error object with `code`, `message`, `request_id`, and `details`. +- Invalid payloads return 400; internal failures return 500. + +## Non-functional requirements +- Deterministic behavior for same request + same policy snapshot. +- Deterministic idempotent replays: + - repeatable responses for the same `idempotency_key` + request content. + - `409 conflict` with `idempotency_conflict` when the key is reused with different payload. +- Receipts must include immutable integrity metadata: + - `action_hash` for the action object. + - `input_hash` for action+context sans transport and request metadata. + - `sanitized_summary` for transform decisions when transform plan is applied. +- Latency targets for local path: p95 under 200ms for `decide` and `transform` on moderate text payloads. +- Configurable policy file path and store path via environment variables. +- Basic structured logs with no raw payload or secret material. + +## Production hardening addendum + +### Immediate production-readiness requirements + +- Add deterministic graceful shutdown behavior. + - Handle `SIGINT` and `SIGTERM`. + - Drain in-flight requests within a bounded timeout before process exit. + - Provide fallback forced close if graceful shutdown fails. +- Add service-readiness and service-liveness boundaries. + - Keep `/health` as liveness baseline. + - Add readiness semantics if deployment requires startup dependency checks. +- Add transport hardening defaults. + - Enforce request timeout, read-header timeout, write timeout, idle timeout, and header limits. +- Add operational controls for bounded resource use. + - Limit body sizes at transport boundaries. + - Cap idempotency cache/map growth and lifecycle. + - **Done (v2):** Added optional service-wide request rate limiting via `DATAFOG_RATE_LIMIT_RPS` (0=disabled), returning `429 rate_limited`. + +### Additional production readiness work before release + +- Add authN/Z at the edge (or strict allowlist + service mesh policy), with explicit deny-by-default. + - **Done (v2):** API token enforcement via `DATAFOG_API_TOKEN` (Bearer or `X-API-Key`) and `401 unauthorized` response code. +- Add metrics quality improvements: + - Per-endpoint latency distributions, saturation/error-rate alarms. + - Route-level and code-path attribution for policy/transform load. +- Add security hardening checks in CI: + - Dependency vulnerability scanning. + - Dependency/license/supply-chain policy checks. +- Add operational guardrails: + - Non-root container execution and read-only root filesystem. + - Explicit `DATAFOG_*` config schema docs and examples. +- Add resiliency tests: + - Slow client / partial body / invalid content-type / malformed UTF-8 / interrupted shutdown scenarios. + +## Task list for MVP implementation + +1. Repo hardening + - Move to `go.mod` module layout. + - Remove Python runtime entrypoints. + - Add build/test config and Go Dockerfile. + +2. Core domain and policy contract + - Add request/response structs for scan/decide/transform/anonymize/receipt. + - Add canonical `Decision`, `Receipt`, `Rule`, `Entity` models. + +3. Detector engine + - Implement deterministic regex detectors. + - Add tests for email/phone/SSN/API-key/credit-card. + +4. Policy evaluator + - Implement rule model and precedence logic. + - Add tests for deny, transform, allow, allow_with_redaction. + +5. Runtime and endpoints + - Implement HTTP router and handlers for 6 endpoints. + - Add request/validation and error handling tests. + +6. Receipt store + - Implement append-only receipt writer and read API. + - Add retrieval tests and error path for missing receipts. + +7. Integration acceptance + - Add end-to-end test for `decide` + `transform` + `/v1/receipts/{id}`. + - Confirm startup docs and API examples. + - Add coverage for idempotency conflict and replay behavior. diff --git a/docs/specs/datafog-claude-ux-spec.md b/docs/specs/datafog-claude-ux-spec.md new file mode 100644 index 0000000..9e2341c --- /dev/null +++ b/docs/specs/datafog-claude-ux-spec.md @@ -0,0 +1,93 @@ +--- +slug: datafog-claude-agent-ux +plan_mode: execution +status: active +owner: sidmohan +created: 2026-02-23 +--- + +# datafog + Claude Setup UX Specification + +## Purpose + +Define a one-command-ish onboarding flow so a developer can make a Claude Code workflow +policy-aware quickly and reliably, while keeping Claude command usage familiar. + +The user experience should answer: + +- "How do I start without rewriting my current workflow?" +- "How do I verify policy controls are active on Claude actions?" + +## User story + +As a developer: + +1. I want to keep using `claude` for normal prompts and agent tasks. +2. I want policy enforcement to apply to side-effect actions before execution. +3. I want to review decisions and adjust policy without low-level troubleshooting. + +## Desired setup experience + +Setup flow: + +- step 1: run policy service, +- step 2: run one bootstrap helper, +- step 3: source one env file and update PATH, +- step 4: confirm status with one Claude command and one decision log line. + +This should take less than five minutes in a clean machine. + +## Acceptance goals + +- Setup can be completed by running `scripts/claude-datafog-setup.sh` from repo root. +- The flow should be deterministic even if the shim is already installed. +- Setup should not require editing system files automatically; explicit activation steps are shown. +- Policy mode must be switchable between `enforced` and `observe` without reinstalling shim. +- A user can recover quickly with: + - `--dry-run` to preview, + - `--mode observe` for no-blocking validation, + - `datafog-shim hooks uninstall claude --force` to remove the managed wrapper. + +## Interaction flow + +- Prerequisite: `datafog-api` is running at a known endpoint and returns `/health`. +- User runs bootstrap helper: + - `./scripts/claude-datafog-setup.sh --policy-url http://localhost:8080` +- Helper resolves the `claude` binary and installs managed wrapper with `datafog-shim hooks install --adapter claude`. +- Helper writes `~/.datafog/claude-datafog.env`. +- User sources env + PATH and runs: + - `claude --help` +- On first usage, user sees either allow/deny/transform behavior and decision event in sink. + +## Interaction improvements + +- Use adapter inference so explicit `--adapter` is not required when using normal Claude invocation. +- Provide `adapters` introspection so policy authors can align rules with canonical action families. +- Keep the managed wrapper command namespace visible (`claude`, with optional `git` if opted). +- Permit fast mode switching via env var in `~/.datafog/claude-datafog.env`: + - `DATAFOG_SHIM_MODE=observe`. + +## Scope for this release + +In-scope: + +- Claude bootstrap helper script. +- Runbook for onboarding and rollback. +- README and docs spec updates. + +Out of scope: + +- Native policy authoring UX. +- Cross-agent orchestration platform. +- GUI installer package for all shells. + +## Metrics of success + +- Time-to-first-policy-aware action: + - target <= 5 minutes. +- First-run friction: + - no manual edits required for PATH/env file creation. +- Visibility: + - at least one decision event appears in NDJSON after first controlled action. +- Recovery time: + - user can switch to observe mode with one env file line change. diff --git a/docs/specs/datafog-codex-ux-spec.md b/docs/specs/datafog-codex-ux-spec.md new file mode 100644 index 0000000..46503a8 --- /dev/null +++ b/docs/specs/datafog-codex-ux-spec.md @@ -0,0 +1,92 @@ +--- +slug: datafog-codex-agent-ux +plan_mode: execution +status: active +owner: sidmohan +created: 2026-02-23 +--- + +# datafog + OpenAI Codex Setup UX Specification + +## Purpose + +Define a single-command onboarding flow that lets a user make a new Codex-driven coding workflow policy-aware quickly and safely, while keeping Codex command usage familiar. + +The user experience should answer two things immediately: + +- "How do I start without rewriting existing agent habits?" +- "How do I know policy controls are active and acting on Codex actions?" + +## User story + +As a developer: + +1. I want to keep using `codex` for normal prompts and task execution. +2. I want policy enforcement to apply to risky actions before they hit tools. +3. I want to review decisions and adjust policy without dropping into low-level networking details. + +## Desired setup experience + +The workflow should feel like: + +- step 1: run policy service, +- step 2: run one bootstrap command, +- step 3: source a generated env file and update PATH, +- step 4: confirm status with one `codex` command and one decision sink line. + +This should take less than five minutes in a clean machine state. + +## Acceptance goals + +- Setup can be completed by running `scripts/codex-datafog-setup.sh` from repo root. +- Command completion should be deterministic even if the shim is already installed. +- The setup should not require editing system files automatically; it should produce explicit manual activation steps. +- Policy mode must be switchable between `enforced` and `observe` without reinstalling the shim. +- A new user can recover quickly with: + - `--dry-run` to preview, + - `--mode observe` for non-blocking validation, + - `datafog-shim hooks uninstall codex --force` to remove the managed wrapper. + +## Interaction flow + +- Prerequisite: `datafog-api` is running at a known endpoint and returns `/health`. +- User runs bootstrap helper: + - `./scripts/codex-datafog-setup.sh --policy-url http://localhost:8080` +- Helper resolves binary path for `codex`, installs managed wrapper using `datafog-shim hooks install --adapter codex`. +- Helper writes `~/.datafog/codex-datafog.env`. +- User sources env + PATH and runs: + - `codex --help` +- On first execution, user sees either allow/deny output or policy decision message in command output and a sink event in `~/.datafog/decisions.ndjson`. + +## Interaction improvements + +- Use `run` adapters inferred by name so explicit `--adapter` is not required for common binaries like `codex`. +- Provide an explicit `adapters list` command so policy authors can align policy rules with shim-inferred canonical families. +- Keep a default shim command namespace visible (`codex`, with optional `git` if opted). +- Offer policy-mode switching through environment to avoid binary reinstall: + - set `DATAFOG_SHIM_MODE=observe` in `~/.datafog/codex-datafog.env`. + +## Scope for this release + +In-scope: + +- CLI bootstrap helper. +- Runbook for onboarding and rollback. +- Documentation updates to `README.md`. + +Out of scope: + +- Deep policy authoring UX inside this repo. +- Multi-agent federation or centralized policy profile UI. +- Non-shell platform-specific installer packaging. + +## Metrics of successful UX + +- Time-to-first-policy-aware action: + - target: <= 5 minutes. +- First-run friction: + - zero manual edits required for PATH/env file creation. +- Visibility: + - decision event appears for at least one invoked action. +- Recovery time: + - user can disable enforcement with observe mode within one edit to one env file line. diff --git a/docs/specs/index.md b/docs/specs/index.md new file mode 100644 index 0000000..3c0a1a9 --- /dev/null +++ b/docs/specs/index.md @@ -0,0 +1,7 @@ +# Specs Index + +Use this index to track initiative specs in `docs/specs`. + +## Active Specs + +- datafog-api-mvp: `docs/specs/datafog-api-mvp-spec.md` diff --git a/docs/spikes/README.md b/docs/spikes/README.md new file mode 100644 index 0000000..e004978 --- /dev/null +++ b/docs/spikes/README.md @@ -0,0 +1,8 @@ +# Spikes + +Time-boxed investigations for uncertain/risky initiatives. +Use the pattern `docs/spikes/-spike.md`. + +Spike docs should start with YAML frontmatter (see `docs/PLANS.md` for the artifact contract). + +Recommended sections: `Context`, `Validation Goal`, `Approach`, `Findings`, `Decisions`, `Recommendation`, `Impact on Upstream Docs`, `Spike Code`, `Remaining Unknowns`, `Time Spent`, and append-only `Revision Notes`. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..d3cf85f --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/datafog/datafog-api + +go 1.22 diff --git a/internal/adapters/registry.go b/internal/adapters/registry.go new file mode 100644 index 0000000..6bf15bc --- /dev/null +++ b/internal/adapters/registry.go @@ -0,0 +1,167 @@ +package adapters + +import ( + "path/filepath" + "runtime" + "sort" + "strings" +) + +// Spec describes a recognized adapter category with canonical name and aliases. +type Spec struct { + Canonical string + Aliases []string + Description string +} + +// DefaultAdapters is the built-in adapter registry. +var DefaultAdapters = []Spec{ + { + Canonical: "vcs", + Aliases: []string{"git", "gh", "gogcli", "hub", "mercurial", "hg", "svn", "bzr", "fossil", "stgit"}, + Description: "Version-control clients and helpers", + }, + { + Canonical: "shell", + Aliases: []string{"sh", "bash", "zsh", "fish", "csh", "tcsh", "cmd", "powershell", "pwsh"}, + Description: "Shell and command interpreters", + }, + { + Canonical: "container", + Aliases: []string{"docker", "podman", "nerdctl", "crictl", "buildah"}, + Description: "Container runtimes and tooling", + }, + { + Canonical: "kubernetes", + Aliases: []string{"kubectl", "helm", "oc", "k9s"}, + Description: "Kubernetes and cluster control tooling", + }, + { + Canonical: "cloud_aws", + Aliases: []string{"aws", "aws2", "sam", "cdk", "eksctl"}, + Description: "AWS CLIs and wrappers", + }, + { + Canonical: "cloud_gcp", + Aliases: []string{"gcloud", "gsutil", "bq", "gke"}, + Description: "Google Cloud CLIs and wrappers", + }, + { + Canonical: "cloud_azure", + Aliases: []string{"az", "azure"}, + Description: "Azure CLIs and wrappers", + }, + { + Canonical: "package_manager", + Aliases: []string{"npm", "pnpm", "yarn", "pip", "pip3", "poetry", "cargo", "go", "mvn", "gradle", "ruby", "gem"}, + Description: "Package manager and language ecosystem CLIs", + }, + { + Canonical: "database", + Aliases: []string{"psql", "mysql", "mariadb", "sqlite3", "mongo", "mongosh", "redis-cli"}, + Description: "Datastore and SQL/NoSQL command interfaces", + }, + { + Canonical: "http", + Aliases: []string{"curl", "wget", "http", "https"}, + Description: "HTTP/API request tooling", + }, + { + Canonical: "claude", + Aliases: []string{"claude", "claude-code", "claude-cli"}, + Description: "Anthropic Claude Code AI coding assistant", + }, + { + Canonical: "codex", + Aliases: []string{"codex", "openai-codex", "codex-cli"}, + Description: "OpenAI Codex AI coding assistant", + }, +} + +var canonicalByAlias = map[string]string{} + +func init() { + for _, spec := range DefaultAdapters { + canonicalByAlias[strings.ToLower(spec.Canonical)] = spec.Canonical + for _, alias := range spec.Aliases { + canonicalByAlias[strings.ToLower(alias)] = spec.Canonical + } + } +} + +// Resolve maps a raw adapter name or command to its canonical adapter name. +func Resolve(raw string, command string) string { + normalized := Normalize(raw) + if normalized == "" { + normalized = Normalize(command) + } + if normalized == "" { + return "" + } + if canonical, ok := Canonical(normalized); ok { + return canonical + } + return normalized +} + +// Canonical returns the canonical adapter name for a given value. +func Canonical(value string) (string, bool) { + canonical, ok := canonicalByAlias[strings.ToLower(value)] + return canonical, ok +} + +// Normalize strips path, extension, and lowercases the adapter name. +func Normalize(raw string) string { + raw = strings.TrimSpace(raw) + if raw == "" { + return "" + } + raw = filepath.Base(raw) + raw = strings.TrimSuffix(strings.ToLower(raw), ".exe") + if runtime.GOOS == "windows" { + raw = strings.TrimSuffix(strings.ToLower(raw), ".cmd") + raw = strings.TrimSuffix(strings.ToLower(raw), ".bat") + } + return strings.TrimSpace(raw) +} + +// KnownAdapters returns all registered adapters sorted by canonical name. +func KnownAdapters() []Spec { + result := make([]Spec, 0, len(DefaultAdapters)) + result = append(result, DefaultAdapters...) + sort.Slice(result, func(i, j int) bool { + return result[i].Canonical < result[j].Canonical + }) + return result +} + +// MatchesAdapter checks if a given tool/adapter name matches any of the +// target adapter names (checking both canonical names and aliases). +func MatchesAdapter(tool string, targets []string) bool { + if len(targets) == 0 { + return true + } + toolCanonical, _ := Canonical(strings.ToLower(tool)) + toolLower := strings.ToLower(tool) + + for _, target := range targets { + targetLower := strings.ToLower(target) + targetCanonical, _ := Canonical(targetLower) + + // Direct match + if toolLower == targetLower { + return true + } + // Canonical match + if toolCanonical != "" && toolCanonical == targetCanonical { + return true + } + if toolCanonical != "" && toolCanonical == targetLower { + return true + } + if toolLower == targetCanonical { + return true + } + } + return false +} diff --git a/internal/adapters/registry_test.go b/internal/adapters/registry_test.go new file mode 100644 index 0000000..6e4e4ee --- /dev/null +++ b/internal/adapters/registry_test.go @@ -0,0 +1,85 @@ +package adapters + +import "testing" + +func TestResolveClaudeAdapter(t *testing.T) { + tests := []struct { + raw string + command string + expected string + }{ + {"claude", "", "claude"}, + {"claude-code", "", "claude"}, + {"claude-cli", "", "claude"}, + {"", "claude", "claude"}, + } + for _, tt := range tests { + got := Resolve(tt.raw, tt.command) + if got != tt.expected { + t.Errorf("Resolve(%q, %q) = %q, want %q", tt.raw, tt.command, got, tt.expected) + } + } +} + +func TestResolveCodexAdapter(t *testing.T) { + tests := []struct { + raw string + command string + expected string + }{ + {"codex", "", "codex"}, + {"openai-codex", "", "codex"}, + {"codex-cli", "", "codex"}, + {"", "codex", "codex"}, + } + for _, tt := range tests { + got := Resolve(tt.raw, tt.command) + if got != tt.expected { + t.Errorf("Resolve(%q, %q) = %q, want %q", tt.raw, tt.command, got, tt.expected) + } + } +} + +func TestMatchesAdapter(t *testing.T) { + tests := []struct { + name string + tool string + targets []string + want bool + }{ + {"empty targets matches anything", "git", nil, true}, + {"direct match", "claude", []string{"claude"}, true}, + {"alias match", "claude-code", []string{"claude"}, true}, + {"canonical match", "git", []string{"vcs"}, true}, + {"no match", "curl", []string{"claude", "codex"}, false}, + {"codex match", "codex-cli", []string{"codex"}, true}, + {"multiple targets", "claude", []string{"claude", "codex"}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := MatchesAdapter(tt.tool, tt.targets) + if got != tt.want { + t.Errorf("MatchesAdapter(%q, %v) = %v, want %v", tt.tool, tt.targets, got, tt.want) + } + }) + } +} + +func TestKnownAdaptersContainsClaudeAndCodex(t *testing.T) { + all := KnownAdapters() + foundClaude, foundCodex := false, false + for _, a := range all { + if a.Canonical == "claude" { + foundClaude = true + } + if a.Canonical == "codex" { + foundCodex = true + } + } + if !foundClaude { + t.Error("expected claude adapter in registry") + } + if !foundCodex { + t.Error("expected codex adapter in registry") + } +} diff --git a/internal/models/models.go b/internal/models/models.go new file mode 100644 index 0000000..a72f00e --- /dev/null +++ b/internal/models/models.go @@ -0,0 +1,187 @@ +package models + +import "time" + +type Decision string + +const ( + DecisionAllow Decision = "allow" + DecisionDeny Decision = "deny" + DecisionTransform Decision = "transform" + DecisionAllowWithRedaction Decision = "allow_with_redaction" +) + +type TransformMode string + +const ( + TransformModeMask TransformMode = "mask" + TransformModeTokenize TransformMode = "tokenize" + TransformModeAnonymize TransformMode = "anonymize" + TransformModeRedact TransformMode = "redact" + TransformModeReplace TransformMode = "replace" + TransformModeHash TransformMode = "hash" +) + +type ScanFinding struct { + EntityType string `json:"entity_type"` + Value string `json:"value"` + Start int `json:"start"` + End int `json:"end"` + Confidence float64 `json:"confidence"` +} + +type ScanRequest struct { + Text string `json:"text"` + EntityTypes []string `json:"entity_types,omitempty"` + RequestID string `json:"request_id,omitempty"` + TraceID string `json:"trace_id,omitempty"` + IdempotencyKey string `json:"idempotency_key,omitempty"` +} + +type ScanResponse struct { + RequestID string `json:"request_id"` + TraceID string `json:"trace_id,omitempty"` + Findings []ScanFinding `json:"findings"` + PolicyVersion string `json:"policy_version"` + PolicyID string `json:"policy_id"` +} + +type ActionMeta struct { + Type string `json:"type"` + Tool string `json:"tool,omitempty"` + Resource string `json:"resource,omitempty"` + Command string `json:"command,omitempty"` + Args []string `json:"args,omitempty"` + Sensitive bool `json:"sensitive,omitempty"` +} + +type DecideRequest struct { + RequestID string `json:"request_id,omitempty"` + TraceID string `json:"trace_id,omitempty"` + TenantID string `json:"tenant_id,omitempty"` + ActorID string `json:"actor_id,omitempty"` + SessionID string `json:"session_id,omitempty"` + Action ActionMeta `json:"action"` + Text string `json:"text,omitempty"` + Findings []ScanFinding `json:"findings,omitempty"` + IdempotencyKey string `json:"idempotency_key,omitempty"` +} + +type DecideResponse struct { + RequestID string `json:"request_id,omitempty"` + TraceID string `json:"trace_id,omitempty"` + Decision Decision `json:"decision"` + ReceiptID string `json:"receipt_id"` + PolicyVersion string `json:"policy_version"` + PolicyID string `json:"policy_id"` + MatchedRules []string `json:"matched_rules"` + TransformPlan []TransformStep `json:"transform_plan,omitempty"` + Findings []ScanFinding `json:"findings"` + Reason string `json:"reason,omitempty"` +} + +type TransformRequest struct { + Text string `json:"text"` + Findings []ScanFinding `json:"findings,omitempty"` + Mode TransformMode `json:"mode,omitempty"` + EntityModes map[string]TransformMode `json:"entity_modes,omitempty"` + RequestID string `json:"request_id,omitempty"` + TraceID string `json:"trace_id,omitempty"` + IdempotencyKey string `json:"idempotency_key,omitempty"` +} + +type TransformResponse struct { + RequestID string `json:"request_id"` + TraceID string `json:"trace_id,omitempty"` + Output string `json:"output"` + PolicyID string `json:"policy_id"` + PolicyVersion string `json:"policy_version"` + Stats TransformStats `json:"stats"` +} + +type TransformStep struct { + EntityType string `json:"entity_type"` + Mode TransformMode `json:"mode"` +} + +type TransformStats struct { + EntitiesTransformed int `json:"entities_transformed"` + ModesApplied string `json:"modes_applied"` +} + +type AnonymizeRequest struct { + Text string `json:"text"` + Findings []ScanFinding `json:"findings,omitempty"` + RequestID string `json:"request_id,omitempty"` + TraceID string `json:"trace_id,omitempty"` + IdempotencyKey string `json:"idempotency_key,omitempty"` +} + +type PolicyRequestContext struct { + PolicyID string + PolicyVersion string + ActiveRules []Rule +} + +type HealthResponse struct { + Status string `json:"status"` + PolicyID string `json:"policy_id"` + PolicyVersion string `json:"policy_version"` + StartedAt string `json:"started_at"` +} + +type Receipt struct { + ReceiptID string `json:"receipt_id"` + Timestamp time.Time `json:"timestamp"` + RequestID string `json:"request_id"` + TraceID string `json:"trace_id"` + TenantID string `json:"tenant_id"` + ActorID string `json:"actor_id"` + SessionID string `json:"session_id"` + PolicyVersion string `json:"policy_version"` + PolicyID string `json:"policy_id"` + ActionHash string `json:"action_hash"` + InputHash string `json:"input_hash"` + SanitizedSummary string `json:"sanitized_summary,omitempty"` + Decision Decision `json:"decision"` + Action ActionMeta `json:"action"` + MatchedRules []string `json:"matched_rules"` + Findings []ScanFinding `json:"findings"` + TransformPlan []TransformStep `json:"transform_plan,omitempty"` + Reason string `json:"reason,omitempty"` +} + +type APIError struct { + Code string `json:"code"` + Message string `json:"message"` + RequestID string `json:"request_id,omitempty"` + Details string `json:"details,omitempty"` +} + +type MatchCriteria struct { + ActionTypes []string `json:"action_types,omitempty"` + Tools []string `json:"tools,omitempty"` + ResourcePrefix []string `json:"resource_prefixes,omitempty"` + Commands []string `json:"commands,omitempty"` + Args []string `json:"args,omitempty"` + Adapters []string `json:"adapters,omitempty"` +} + +type Rule struct { + ID string `json:"id"` + Description string `json:"description"` + Priority int `json:"priority"` + Effect Decision `json:"effect"` + Match MatchCriteria `json:"match"` + EntityRequirements []string `json:"entity_requirements,omitempty"` + EntityTransforms []TransformStep `json:"entity_transforms,omitempty"` + RequireSensitiveOnly bool `json:"require_sensitive_only,omitempty"` +} + +type Policy struct { + PolicyID string `json:"policy_id"` + PolicyVersion string `json:"policy_version"` + Description string `json:"description,omitempty"` + UpdatedAt time.Time `json:"updated_at"` + Rules []Rule `json:"rules"` +} diff --git a/internal/policy/policy.go b/internal/policy/policy.go new file mode 100644 index 0000000..5e2c7ec --- /dev/null +++ b/internal/policy/policy.go @@ -0,0 +1,372 @@ +package policy + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/datafog/datafog-api/internal/adapters" + "github.com/datafog/datafog-api/internal/models" +) + +var RequiredDecisionInputs = map[models.Decision]struct{}{ + models.DecisionAllow: {}, + models.DecisionDeny: {}, + models.DecisionTransform: {}, + models.DecisionAllowWithRedaction: {}, +} + +func LoadPolicyFromFile(path string) (models.Policy, error) { + var policy models.Policy + + policyPath := strings.TrimSpace(path) + if strings.ContainsRune(policyPath, 0) { + return policy, fmt.Errorf("invalid policy path") + } + policyPath = filepath.Clean(policyPath) + if policyPath == "." { + return policy, fmt.Errorf("invalid policy path") + } + + content, err := os.ReadFile(policyPath) // #nosec G304 -- path comes from DATAFOG_POLICY_PATH and is validated by startup config. + if err != nil { + return policy, err + } + if err := json.Unmarshal(content, &policy); err != nil { + return policy, err + } + if err := ValidatePolicy(policy); err != nil { + return policy, err + } + return policy, nil +} + +func ValidatePolicy(policy models.Policy) error { + errors := make([]string, 0) + if strings.TrimSpace(policy.PolicyID) == "" { + errors = append(errors, "policy_id is required") + } + if strings.TrimSpace(policy.PolicyVersion) == "" { + errors = append(errors, "policy_version is required") + } + if len(policy.Rules) == 0 { + errors = append(errors, "policy must contain at least one rule") + } + + seenRuleIDs := map[string]struct{}{} + for _, rule := range policy.Rules { + ruleID := strings.TrimSpace(rule.ID) + if ruleID == "" { + errors = append(errors, "rule missing id") + continue + } + if _, ok := seenRuleIDs[ruleID]; ok { + errors = append(errors, fmt.Sprintf("duplicate rule id: %s", ruleID)) + } + seenRuleIDs[ruleID] = struct{}{} + if rule.Priority < 0 { + errors = append(errors, fmt.Sprintf("rule %s has negative priority: %d", ruleID, rule.Priority)) + } + if _, ok := RequiredDecisionInputs[rule.Effect]; !ok { + errors = append(errors, fmt.Sprintf("rule %s has unsupported effect: %s", ruleID, rule.Effect)) + } + for _, actionType := range rule.Match.ActionTypes { + if strings.TrimSpace(actionType) == "" { + errors = append(errors, fmt.Sprintf("rule %s has empty action_type condition", ruleID)) + } + } + for _, tool := range rule.Match.Tools { + if strings.TrimSpace(tool) == "" { + errors = append(errors, fmt.Sprintf("rule %s has empty tool condition", ruleID)) + } + } + for _, prefix := range rule.Match.ResourcePrefix { + if strings.TrimSpace(prefix) == "" { + errors = append(errors, fmt.Sprintf("rule %s has empty resource_prefix condition", ruleID)) + } + } + for _, command := range rule.Match.Commands { + if strings.TrimSpace(command) == "" { + errors = append(errors, fmt.Sprintf("rule %s has empty command condition", ruleID)) + } + } + for _, arg := range rule.Match.Args { + if strings.TrimSpace(arg) == "" { + errors = append(errors, fmt.Sprintf("rule %s has empty arg condition", ruleID)) + } + } + for _, adapter := range rule.Match.Adapters { + if strings.TrimSpace(adapter) == "" { + errors = append(errors, fmt.Sprintf("rule %s has empty adapter condition", ruleID)) + } + } + for _, requirement := range rule.EntityRequirements { + reqName := strings.ToLower(strings.TrimSpace(requirement)) + if reqName == "" { + errors = append(errors, fmt.Sprintf("rule %s has empty entity_requirement", ruleID)) + continue + } + if _, ok := defaultEntityTypes[reqName]; !ok { + errors = append(errors, fmt.Sprintf("rule %s references unsupported required entity type: %s", ruleID, requirement)) + } + } + for _, step := range rule.EntityTransforms { + if strings.TrimSpace(step.EntityType) == "" { + errors = append(errors, fmt.Sprintf("rule %s has entity transform without entity_type", ruleID)) + continue + } + entityType := strings.ToLower(strings.TrimSpace(step.EntityType)) + if _, ok := defaultEntityTypes[entityType]; !ok { + errors = append(errors, fmt.Sprintf("rule %s references unsupported transform entity type: %s", ruleID, step.EntityType)) + } + if _, ok := allowedModes[step.Mode]; !ok { + errors = append(errors, fmt.Sprintf("rule %s references unsupported transform mode: %s", ruleID, step.Mode)) + } + } + } + + if len(errors) == 0 { + return nil + } + return fmt.Errorf(strings.Join(errors, "; ")) +} + +var allowedModes = map[models.TransformMode]struct{}{ + models.TransformModeMask: {}, + models.TransformModeTokenize: {}, + models.TransformModeAnonymize: {}, + models.TransformModeRedact: {}, + models.TransformModeReplace: {}, + models.TransformModeHash: {}, +} + +type DecisionContext struct { + Action models.ActionMeta + Findings []models.ScanFinding +} + +type DecisionResult struct { + Decision models.Decision + MatchedRules []string + TransformPlan []models.TransformStep + Reason string +} + +var defaultEntityTransforms = []models.TransformStep{ + {EntityType: "email", Mode: models.TransformModeMask}, + {EntityType: "phone", Mode: models.TransformModeTokenize}, + {EntityType: "ssn", Mode: models.TransformModeAnonymize}, + {EntityType: "api_key", Mode: models.TransformModeRedact}, + {EntityType: "credit_card", Mode: models.TransformModeRedact}, + {EntityType: "ip_address", Mode: models.TransformModeMask}, + {EntityType: "date", Mode: models.TransformModeMask}, + {EntityType: "zip_code", Mode: models.TransformModeMask}, + {EntityType: "person", Mode: models.TransformModeRedact}, + {EntityType: "organization", Mode: models.TransformModeMask}, + {EntityType: "location", Mode: models.TransformModeMask}, +} + +var defaultEntityTypes = map[string]struct{}{ + "email": {}, + "phone": {}, + "ssn": {}, + "api_key": {}, + "credit_card": {}, + "ip_address": {}, + "date": {}, + "zip_code": {}, + "person": {}, + "organization": {}, + "location": {}, +} + +func Evaluate(policy models.Policy, ctx DecisionContext) DecisionResult { + if ctx.Action.Type == "" { + return DecisionResult{ + Decision: models.DecisionDeny, + Reason: "action.type is required", + } + } + + if len(policy.Rules) == 0 { + return DecisionResult{ + Decision: models.DecisionDeny, + Reason: "policy has no rules", + } + } + + rules := append([]models.Rule(nil), policy.Rules...) + sort.SliceStable(rules, func(i, j int) bool { + return rules[i].Priority > rules[j].Priority + }) + + hasFindings := map[string]struct{}{} + for _, f := range ctx.Findings { + hasFindings[strings.ToLower(f.EntityType)] = struct{}{} + } + + matchIDs := []string{} + transformPlan := []models.TransformStep{} + transformFound := false + transformWithRedaction := false + denyReason := "" + denyMatched := false + + matched := false + for _, rule := range rules { + if _, ok := RequiredDecisionInputs[rule.Effect]; !ok { + continue + } + if !matchAction(rule.Match, rule.RequireSensitiveOnly, ctx.Action) { + continue + } + if !hasRequiredEntities(rule.EntityRequirements, hasFindings) { + continue + } + matched = true + matchIDs = append(matchIDs, rule.ID) + + switch rule.Effect { + case models.DecisionDeny: + denyMatched = true + if denyReason == "" { + denyReason = rule.Description + } + case models.DecisionTransform: + transformFound = true + if len(rule.EntityTransforms) > 0 { + transformPlan = append(transformPlan, rule.EntityTransforms...) + } + case models.DecisionAllowWithRedaction: + transformWithRedaction = true + } + } + + if denyMatched { + return DecisionResult{ + Decision: models.DecisionDeny, + MatchedRules: matchIDs, + TransformPlan: nil, + Reason: denyReason, + } + } + if transformFound { + if len(transformPlan) == 0 { + transformPlan = defaultEntityTransforms + } + return DecisionResult{ + Decision: models.DecisionTransform, + MatchedRules: matchIDs, + TransformPlan: transformPlan, + } + } + if transformWithRedaction { + if len(transformPlan) == 0 { + transformPlan = defaultEntityTransforms + } + return DecisionResult{ + Decision: models.DecisionAllowWithRedaction, + MatchedRules: matchIDs, + TransformPlan: transformPlan, + } + } + if matched { + return DecisionResult{ + Decision: models.DecisionAllow, + MatchedRules: matchIDs, + } + } + + return DecisionResult{ + Decision: models.DecisionDeny, + Reason: "no matching rule", + } +} + +func matchAction(match models.MatchCriteria, requireSensitiveOnly bool, action models.ActionMeta) bool { + if !matchesField(match.ActionTypes, action.Type) { + return false + } + if !matchesField(match.Tools, action.Tool) { + return false + } + if !matchesField(match.Commands, action.Command) { + return false + } + if !matchesArgs(match.Args, action.Args) { + return false + } + if !adapters.MatchesAdapter(action.Tool, match.Adapters) { + return false + } + if requireSensitiveOnly && !action.Sensitive { + return false + } + if len(match.ResourcePrefix) > 0 && action.Resource == "" { + return false + } + for _, prefix := range match.ResourcePrefix { + if strings.HasPrefix(action.Resource, prefix) { + return true + } + } + if len(match.ResourcePrefix) > 0 { + return false + } + return true +} + +func matchesArgs(required []string, args []string) bool { + if len(required) == 0 { + return true + } + if len(args) == 0 { + return false + } + + for _, expected := range required { + matched := false + for _, value := range args { + if matchesField([]string{expected}, value) { + matched = true + break + } + } + if !matched { + return false + } + } + return true +} + +func matchesField(allowed []string, value string) bool { + if len(allowed) == 0 { + return true + } + for _, allow := range allowed { + if strings.EqualFold(allow, value) { + return true + } + } + return false +} + +func hasRequiredEntities(reqs []string, found map[string]struct{}) bool { + for _, req := range reqs { + reqName := strings.ToLower(strings.TrimSpace(req)) + if _, ok := defaultEntityTypes[reqName]; !ok { + return false + } + if _, ok := found[reqName]; !ok { + return false + } + } + return true +} + +func (res DecisionResult) String() string { + return fmt.Sprintf("%s decision=%s matched=%v reason=%s", res.Decision, res.Decision, res.MatchedRules, res.Reason) +} diff --git a/internal/policy/policy_test.go b/internal/policy/policy_test.go new file mode 100644 index 0000000..2ffa144 --- /dev/null +++ b/internal/policy/policy_test.go @@ -0,0 +1,426 @@ +package policy + +import ( + "os" + "strings" + "testing" + + "github.com/datafog/datafog-api/internal/models" +) + +func basePolicy() models.Policy { + return models.Policy{ + PolicyID: "mvp", + PolicyVersion: "v1", + Rules: []models.Rule{ + { + ID: "deny-api-key-shell", + Priority: 100, + Effect: models.DecisionDeny, + Match: models.MatchCriteria{ + ActionTypes: []string{"shell.exec"}, + }, + EntityRequirements: []string{"api_key"}, + }, + { + ID: "transform-sensitive", + Priority: 90, + Effect: models.DecisionTransform, + Match: models.MatchCriteria{ + ActionTypes: []string{"file.write", "http.request", "shell.exec"}, + }, + EntityRequirements: []string{"email"}, + }, + { + ID: "allow-safe", + Priority: 10, + Effect: models.DecisionAllow, + Match: models.MatchCriteria{ + ActionTypes: []string{"file.read", "shell.exec"}, + }, + }, + }, + } +} + +func TestEvaluateDenyOnAPIKeyForShell(t *testing.T) { + policy := basePolicy() + ctx := DecisionContext{ + Action: models.ActionMeta{Type: "shell.exec", Resource: "curl"}, + Findings: []models.ScanFinding{{EntityType: "api_key", Value: "ABC1234567890123", Start: 0, End: 16, Confidence: .9}}, + } + result := Evaluate(policy, ctx) + if result.Decision != models.DecisionDeny { + t.Fatalf("expected deny, got %s", result.Decision) + } + if result.MatchedRules[0] != "deny-api-key-shell" { + t.Fatalf("expected deny rule match, got %v", result.MatchedRules) + } +} + +func TestEvaluateMatchesCommandAndArgs(t *testing.T) { + policy := models.Policy{ + PolicyID: "mvp", + PolicyVersion: "v1", + Rules: []models.Rule{ + { + ID: "deny-rm-recursive", + Priority: 100, + Effect: models.DecisionDeny, + Match: models.MatchCriteria{ + ActionTypes: []string{"shell.exec"}, + Commands: []string{"rm"}, + Args: []string{"-rf"}, + }, + }, + { + ID: "allow-shell", + Priority: 10, + Effect: models.DecisionAllow, + Match: models.MatchCriteria{ + ActionTypes: []string{"shell.exec"}, + }, + }, + }, + } + result := Evaluate(policy, DecisionContext{ + Action: models.ActionMeta{ + Type: "shell.exec", + Command: "rm", + Args: []string{"-rf", "/tmp"}, + Resource: "rm", + }, + Findings: []models.ScanFinding{}, + }) + if result.Decision != models.DecisionDeny { + t.Fatalf("expected deny for rm -rf, got %s", result.Decision) + } + + result = Evaluate(policy, DecisionContext{ + Action: models.ActionMeta{ + Type: "shell.exec", + Command: "rm", + Args: []string{"-f", "/tmp"}, + }, + }) + if result.Decision != models.DecisionAllow { + t.Fatalf("expected allow for rm without recursive flag, got %s", result.Decision) + } +} + +func TestEvaluateRequireSensitiveOnly(t *testing.T) { + policy := models.Policy{ + PolicyID: "mvp", + PolicyVersion: "v1", + Rules: []models.Rule{ + { + ID: "transform-sensitive-shell", + Priority: 100, + Effect: models.DecisionTransform, + RequireSensitiveOnly: true, + Match: models.MatchCriteria{ + ActionTypes: []string{"file.write"}, + }, + EntityRequirements: []string{"email"}, + }, + { + ID: "allow-file-write", + Priority: 10, + Effect: models.DecisionAllow, + Match: models.MatchCriteria{ + ActionTypes: []string{"file.write"}, + }, + }, + }, + } + withSensitive := Evaluate(policy, DecisionContext{ + Action: models.ActionMeta{ + Type: "file.write", + Sensitive: true, + }, + Findings: []models.ScanFinding{ + {EntityType: "email", Value: "a@b.com", Start: 0, End: 7, Confidence: .98}, + }, + }) + if withSensitive.Decision != models.DecisionTransform { + t.Fatalf("expected transform when sensitive action matches require_sensitive_only rule, got %s", withSensitive.Decision) + } + + withoutSensitive := Evaluate(policy, DecisionContext{ + Action: models.ActionMeta{ + Type: "file.write", + }, + Findings: []models.ScanFinding{ + {EntityType: "email", Value: "a@b.com", Start: 0, End: 7, Confidence: .98}, + }, + }) + if withoutSensitive.Decision != models.DecisionAllow { + t.Fatalf("expected allow when action is not marked sensitive, got %s", withoutSensitive.Decision) + } +} + +func TestEvaluateTransformWhenSensitiveEntity(t *testing.T) { + policy := basePolicy() + ctx := DecisionContext{ + Action: models.ActionMeta{Type: "file.write", Resource: "notes.txt"}, + Findings: []models.ScanFinding{{EntityType: "email", Value: "a@b.com", Start: 0, End: 7, Confidence: .98}}, + } + result := Evaluate(policy, ctx) + if result.Decision != models.DecisionTransform { + t.Fatalf("expected transform, got %s", result.Decision) + } + if len(result.TransformPlan) == 0 { + t.Fatalf("expected transform plan") + } +} + +func TestEvaluateAllowWhenNoSensitiveEntity(t *testing.T) { + policy := basePolicy() + ctx := DecisionContext{ + Action: models.ActionMeta{Type: "file.read", Resource: "notes.txt"}, + Findings: []models.ScanFinding{}, + } + result := Evaluate(policy, ctx) + if result.Decision != models.DecisionAllow { + t.Fatalf("expected allow, got %s", result.Decision) + } +} + +func TestEvaluateDefaultDenyForUnknownAction(t *testing.T) { + policy := basePolicy() + result := Evaluate(policy, DecisionContext{Action: models.ActionMeta{Type: "unknown.action"}}) + if result.Decision != models.DecisionDeny { + t.Fatalf("expected deny for unknown action, got %s", result.Decision) + } +} + +func TestEvaluateDenylRuleAlwaysWins(t *testing.T) { + policy := models.Policy{ + PolicyID: "mvp", + PolicyVersion: "v1", + Rules: []models.Rule{ + { + ID: "transform-low-priority", + Priority: 90, + Effect: models.DecisionTransform, + Match: models.MatchCriteria{ + ActionTypes: []string{"file.write"}, + }, + EntityRequirements: []string{"email"}, + }, + { + ID: "deny-low-priority", + Priority: 10, + Effect: models.DecisionDeny, + Match: models.MatchCriteria{ + ActionTypes: []string{"file.write"}, + }, + EntityRequirements: []string{"api_key"}, + }, + }, + } + result := Evaluate(policy, DecisionContext{ + Action: models.ActionMeta{Type: "file.write", Resource: "notes.txt"}, + Findings: []models.ScanFinding{ + {EntityType: "email", Value: "a@b.com", Start: 0, End: 7, Confidence: .98}, + {EntityType: "api_key", Value: "ABCD1234EFGH5678", Start: 9, End: 25, Confidence: .98}, + }, + }) + if result.Decision != models.DecisionDeny { + t.Fatalf("expected deny to take precedence, got %s", result.Decision) + } + if len(result.MatchedRules) != 2 { + t.Fatalf("expected 2 matched rules, got %v", result.MatchedRules) + } +} + +func TestEvaluateTransformBeatsRedaction(t *testing.T) { + policy := models.Policy{ + PolicyID: "mvp", + PolicyVersion: "v1", + Rules: []models.Rule{ + { + ID: "redact-mid-priority", + Priority: 50, + Effect: models.DecisionAllowWithRedaction, + Match: models.MatchCriteria{ + ActionTypes: []string{"file.write"}, + }, + }, + { + ID: "transform-high-priority", + Priority: 40, + Effect: models.DecisionTransform, + Match: models.MatchCriteria{ + ActionTypes: []string{"file.write"}, + }, + EntityRequirements: []string{"email"}, + }, + }, + } + result := Evaluate(policy, DecisionContext{ + Action: models.ActionMeta{Type: "file.write", Resource: "notes.txt"}, + Findings: []models.ScanFinding{ + {EntityType: "email", Value: "a@b.com", Start: 0, End: 7, Confidence: .98}, + }, + }) + if result.Decision != models.DecisionTransform { + t.Fatalf("expected transform to beat allow_with_redaction, got %s", result.Decision) + } + if len(result.TransformPlan) == 0 { + t.Fatalf("expected transform plan for transform decision") + } +} + +func TestValidatePolicyRejectsUnknownEffect(t *testing.T) { + policy := basePolicy() + policy.Rules[0].Effect = models.Decision("unsupported") + if err := ValidatePolicy(policy); err == nil { + t.Fatal("expected validation error") + } else if !strings.Contains(err.Error(), "unsupported effect") { + t.Fatalf("unexpected validation error: %v", err) + } +} + +func TestValidatePolicyRejectsDuplicateRuleIDs(t *testing.T) { + policy := basePolicy() + policy.Rules[1].ID = policy.Rules[0].ID + if err := ValidatePolicy(policy); err == nil { + t.Fatal("expected duplicate rule id error") + } else if !strings.Contains(err.Error(), "duplicate rule id") { + t.Fatalf("unexpected validation error: %v", err) + } +} + +func TestValidatePolicyRejectsUnsupportedEntityRequirement(t *testing.T) { + policy := basePolicy() + policy.Rules[0].EntityRequirements = []string{"not_a_real_entity"} + if err := ValidatePolicy(policy); err == nil { + t.Fatal("expected unsupported entity requirement error") + } else if !strings.Contains(err.Error(), "unsupported required entity type") { + t.Fatalf("unexpected validation error: %v", err) + } +} + +func TestValidatePolicyRejectsInvalidTransformMode(t *testing.T) { + policy := basePolicy() + policy.Rules[1].EntityTransforms = []models.TransformStep{{EntityType: "email", Mode: "invalid"}} + if err := ValidatePolicy(policy); err == nil { + t.Fatal("expected invalid transform mode error") + } else if !strings.Contains(err.Error(), "unsupported transform mode") { + t.Fatalf("unexpected validation error: %v", err) + } +} + +func TestValidatePolicyRejectsUnsupportedTransformEntityType(t *testing.T) { + policy := basePolicy() + policy.Rules[1].EntityTransforms = []models.TransformStep{{EntityType: "not_real", Mode: models.TransformModeMask}} + if err := ValidatePolicy(policy); err == nil { + t.Fatal("expected unsupported transform entity type error") + } else if !strings.Contains(err.Error(), "unsupported transform entity type") { + t.Fatalf("unexpected validation error: %v", err) + } +} + +func TestValidatePolicyRejectsEmptyRuleID(t *testing.T) { + policy := basePolicy() + policy.Rules[0].ID = " " + if err := ValidatePolicy(policy); err == nil { + t.Fatal("expected missing rule id error") + } else if !strings.Contains(err.Error(), "rule missing id") { + t.Fatalf("unexpected validation error: %v", err) + } +} + +func TestValidatePolicyRejectsEmptyMatchEntries(t *testing.T) { + policy := basePolicy() + policy.Rules[0].Match.ActionTypes = []string{""} + policy.Rules[0].Match.ResourcePrefix = []string{" "} + policy.Rules[0].Match.Commands = []string{" "} + policy.Rules[0].Match.Args = []string{" "} + if err := ValidatePolicy(policy); err == nil { + t.Fatal("expected empty match criteria error") + } else if !strings.Contains(err.Error(), "empty action_type condition") && + !strings.Contains(err.Error(), "empty resource_prefix condition") && + !strings.Contains(err.Error(), "empty command condition") && + !strings.Contains(err.Error(), "empty arg condition") { + t.Fatalf("unexpected validation error: %v", err) + } +} + +type policyDecisionVector struct { + Name string `json:"name"` + Action models.ActionMeta `json:"action"` + Findings []models.ScanFinding `json:"findings"` + ExpectedDecision models.Decision `json:"expected_decision"` + ExpectedRules []string `json:"expected_matched_rules"` + ExpectedTransform bool `json:"expected_transform"` +} + +func TestEvaluateGoldenPolicyVectors(t *testing.T) { + vectors := []policyDecisionVector{ + { + Name: "allow read action", + Action: models.ActionMeta{Type: "file.read", Resource: "notes.txt"}, + ExpectedDecision: models.DecisionAllow, + ExpectedRules: []string{"allow-safe"}, + }, + { + Name: "transform file write with email", + Action: models.ActionMeta{Type: "file.write", Resource: "notes.txt"}, + Findings: []models.ScanFinding{{EntityType: "email", Value: "jane@x.com", Start: 0, End: 9, Confidence: 0.99}}, + ExpectedDecision: models.DecisionTransform, + ExpectedRules: []string{"transform-sensitive"}, + ExpectedTransform: true, + }, + { + Name: "deny shell with api key", + Action: models.ActionMeta{Type: "shell.exec", Resource: "curl"}, + Findings: []models.ScanFinding{{EntityType: "api_key", Value: "ABCD1234EFGH5678", Start: 0, End: 16, Confidence: 0.99}}, + ExpectedDecision: models.DecisionDeny, + ExpectedRules: []string{"deny-api-key-shell", "allow-safe"}, + }, + { + Name: "default deny for unmatched", + Action: models.ActionMeta{Type: "unknown.action"}, + ExpectedDecision: models.DecisionDeny, + }, + } + + policy := basePolicy() + + for _, vector := range vectors { + t.Run(vector.Name, func(t *testing.T) { + t.Parallel() + result := Evaluate(policy, DecisionContext{Action: vector.Action, Findings: vector.Findings}) + if result.Decision != vector.ExpectedDecision { + t.Fatalf("expected decision %q, got %q", vector.ExpectedDecision, result.Decision) + } + if len(vector.ExpectedRules) > 0 { + if len(result.MatchedRules) != len(vector.ExpectedRules) { + t.Fatalf("expected %d matched rules, got %v", len(vector.ExpectedRules), result.MatchedRules) + } + for idx := range vector.ExpectedRules { + if result.MatchedRules[idx] != vector.ExpectedRules[idx] { + t.Fatalf("expected rule %q at %d, got %v", vector.ExpectedRules[idx], idx, result.MatchedRules) + } + } + } + if vector.ExpectedTransform && len(result.TransformPlan) == 0 { + t.Fatalf("expected transform plan") + } + }) + } +} + +func TestLoadPolicyFromFileRejectsMissingMetadata(t *testing.T) { + policyPath := t.TempDir() + "/policy.json" + policy := `{"rules":[{"id":"allow-read","priority":1,"effect":"allow","match":{"action_types":["file.read"]}}]}` + if err := os.WriteFile(policyPath, []byte(policy), 0o644); err != nil { + t.Fatalf("seed policy file failed: %v", err) + } + + if _, err := LoadPolicyFromFile(policyPath); err == nil { + t.Fatal("expected policy load error") + } +} diff --git a/internal/receipts/store.go b/internal/receipts/store.go new file mode 100644 index 0000000..dbd7f00 --- /dev/null +++ b/internal/receipts/store.go @@ -0,0 +1,193 @@ +package receipts + +import ( + "bufio" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/datafog/datafog-api/internal/models" + "github.com/datafog/datafog-api/internal/policy" +) + +const maxReceiptLineBytes = 1024 * 1024 +const defaultReceiptFileMode = 0o600 +const defaultReceiptDirMode = 0o750 + +type ReceiptStore struct { + mu sync.RWMutex + filePath string + receipts map[string]models.Receipt + maxEntries int + entryCount int +} + +// MaxEntries sets the maximum number of receipts before rotation. +// 0 means no limit (default). +func MaxEntries(n int) func(*ReceiptStore) { + return func(s *ReceiptStore) { + s.maxEntries = n + } +} + +func NewReceiptStore(filePath string, opts ...func(*ReceiptStore)) (*ReceiptStore, error) { + if filePath == "" { + filePath = "datafog_receipts.jsonl" + } + filePath = strings.TrimSpace(filePath) + if strings.ContainsRune(filePath, 0) { + return nil, fmt.Errorf("invalid receipt path") + } + dir := filepath.Dir(filePath) + if dir != "." { + if err := os.MkdirAll(dir, defaultReceiptDirMode); err != nil { + return nil, err + } + } + + store := &ReceiptStore{ + filePath: filePath, + receipts: map[string]models.Receipt{}, + } + for _, opt := range opts { + opt(store) + } + f, err := os.OpenFile(filePath, os.O_CREATE|os.O_RDONLY, defaultReceiptFileMode) // #nosec G304 -- receipt path is validated from startup configuration. + if err != nil { + return nil, err + } + f.Close() + if err := store.loadExistingReceipts(); err != nil { + return nil, err + } + return store, nil +} + +func (s *ReceiptStore) NewReceipt(req models.DecideRequest, decision models.Decision, result policy.DecisionResult, policyMeta models.Policy) models.Receipt { + return models.Receipt{ + ReceiptID: newID(), + Timestamp: time.Now().UTC(), + RequestID: req.RequestID, + TraceID: req.TraceID, + TenantID: req.TenantID, + ActorID: req.ActorID, + SessionID: req.SessionID, + PolicyVersion: policyMeta.PolicyVersion, + PolicyID: policyMeta.PolicyID, + Decision: decision, + Action: req.Action, + MatchedRules: result.MatchedRules, + Findings: req.Findings, + TransformPlan: result.TransformPlan, + Reason: result.Reason, + } +} + +func (s *ReceiptStore) Save(receipt models.Receipt) (models.Receipt, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if receipt.ReceiptID == "" { + receipt.ReceiptID = newID() + } + + // Rotate if we've hit the max + if s.maxEntries > 0 && s.entryCount >= s.maxEntries { + if err := s.rotateLocked(); err != nil { + return models.Receipt{}, fmt.Errorf("receipt rotation failed: %w", err) + } + } + + data, err := json.Marshal(receipt) + if err != nil { + return models.Receipt{}, err + } + + f, err := os.OpenFile(s.filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, defaultReceiptFileMode) // #nosec G304 -- receipt path is validated from startup configuration. + if err != nil { + return models.Receipt{}, err + } + defer f.Close() + + if _, err := f.Write(appendWithLine(data)); err != nil { + return models.Receipt{}, err + } + if err := f.Sync(); err != nil { + return models.Receipt{}, err + } + + s.receipts[receipt.ReceiptID] = receipt + s.entryCount++ + return receipt, nil +} + +// rotateLocked archives the current receipts file and starts fresh. +// Must be called with s.mu held. +func (s *ReceiptStore) rotateLocked() error { + archivePath := s.filePath + "." + time.Now().UTC().Format("20060102T150405Z") + if err := os.Rename(s.filePath, archivePath); err != nil && !os.IsNotExist(err) { + return err + } + s.receipts = map[string]models.Receipt{} + s.entryCount = 0 + return nil +} + +// Count returns the number of receipts in memory. +func (s *ReceiptStore) Count() int { + s.mu.RLock() + defer s.mu.RUnlock() + return len(s.receipts) +} + +func (s *ReceiptStore) loadExistingReceipts() error { + f, err := os.OpenFile(s.filePath, os.O_RDONLY, defaultReceiptFileMode) // #nosec G304 -- receipt path is validated from startup configuration. + if err != nil { + return err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 64*1024), maxReceiptLineBytes) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + var receipt models.Receipt + if err := json.Unmarshal([]byte(line), &receipt); err != nil { + return fmt.Errorf("decode existing receipt: %w", err) + } + s.receipts[receipt.ReceiptID] = receipt + s.entryCount++ + } + if err := scanner.Err(); err != nil { + return err + } + return nil +} + +func appendWithLine(data []byte) []byte { + return append(append(make([]byte, 0, len(data)+1), data...), '\n') +} + +func (s *ReceiptStore) Get(id string) (models.Receipt, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + r, ok := s.receipts[id] + return r, ok +} + +func newID() string { + buf := make([]byte, 16) + if _, err := rand.Read(buf); err != nil { + return fmt.Sprintf("fallback-%d", time.Now().UnixNano()) + } + return hex.EncodeToString(buf) +} diff --git a/internal/receipts/store_test.go b/internal/receipts/store_test.go new file mode 100644 index 0000000..8cf85b2 --- /dev/null +++ b/internal/receipts/store_test.go @@ -0,0 +1,124 @@ +package receipts + +import ( + "encoding/json" + "os" + "strings" + "testing" + + "github.com/datafog/datafog-api/internal/models" + "github.com/datafog/datafog-api/internal/policy" +) + +func TestReceiptStoreSaveAndGet(t *testing.T) { + path := t.TempDir() + "/receipts.jsonl" + store, err := NewReceiptStore(path) + if err != nil { + t.Fatalf("new store failed: %v", err) + } + + req := models.DecideRequest{RequestID: "r1", Action: models.ActionMeta{Type: "file.read", Resource: "x"}} + result := policy.DecisionResult{Decision: models.DecisionAllow, MatchedRules: []string{"allow-1"}} + policyMeta := models.Policy{PolicyID: "m", PolicyVersion: "v1"} + receipt := store.NewReceipt(req, models.DecisionAllow, result, policyMeta) + saved, err := store.Save(receipt) + if err != nil { + t.Fatalf("save failed: %v", err) + } + if saved.ReceiptID == "" { + t.Fatalf("receipt id empty") + } + got, ok := store.Get(saved.ReceiptID) + if !ok { + t.Fatalf("receipt not found") + } + if got.Decision != models.DecisionAllow { + t.Fatalf("expected allow receipt") + } + + if _, err := os.Stat(path); err != nil { + t.Fatalf("receipt file missing: %v", err) + } +} + +func TestReceiptStoreLoadsExistingReceipts(t *testing.T) { + path := t.TempDir() + "/receipts.jsonl" + existing := models.Receipt{ + ReceiptID: "receipt-seeded", + PolicyID: "policy-1", + PolicyVersion: "v1", + RequestID: "r1", + Decision: models.DecisionDeny, + MatchedRules: []string{"seed"}, + } + data, err := json.Marshal(existing) + if err != nil { + t.Fatalf("marshal failed: %v", err) + } + if err := os.WriteFile(path, append(data, '\n'), 0o644); err != nil { + t.Fatalf("seed file write failed: %v", err) + } + + store, err := NewReceiptStore(path) + if err != nil { + t.Fatalf("new store failed: %v", err) + } + got, ok := store.Get("receipt-seeded") + if !ok { + t.Fatalf("expected to load existing receipt") + } + if got.Decision != models.DecisionDeny { + t.Fatalf("unexpected decision: %s", got.Decision) + } +} + +func TestReceiptStoreRejectsCorruptReceiptLine(t *testing.T) { + path := t.TempDir() + "/receipts.jsonl" + if err := os.WriteFile(path, []byte("{\n"), 0o644); err != nil { + t.Fatalf("seed file write failed: %v", err) + } + + if _, err := NewReceiptStore(path); err == nil { + t.Fatalf("expected receipt load failure on corrupt line") + } +} + +func TestReceiptStoreLoadsLargeReceiptLine(t *testing.T) { + path := t.TempDir() + "/receipts.jsonl" + existing := models.Receipt{ + ReceiptID: "receipt-large", + PolicyID: "policy-1", + Findings: []models.ScanFinding{ + { + EntityType: "email", + Value: strings.Repeat("x", 600*1024), + Start: 0, + End: 600 * 1024, + Confidence: 0.9, + }, + }, + PolicyVersion: "v1", + RequestID: "r1", + Decision: models.DecisionAllow, + MatchedRules: []string{"seed"}, + } + data, err := json.Marshal(existing) + if err != nil { + t.Fatalf("marshal failed: %v", err) + } + if err := os.WriteFile(path, append(data, '\n'), 0o644); err != nil { + t.Fatalf("seed file write failed: %v", err) + } + + store, err := NewReceiptStore(path) + if err != nil { + t.Fatalf("new store failed: %v", err) + } + got, ok := store.Get("receipt-large") + if !ok { + t.Fatalf("expected to load large receipt") + } + if got.ReceiptID != "receipt-large" { + t.Fatalf("expected loaded receipt id receipt-large, got %q", got.ReceiptID) + } +} diff --git a/internal/scan/detector.go b/internal/scan/detector.go new file mode 100644 index 0000000..7b6401c --- /dev/null +++ b/internal/scan/detector.go @@ -0,0 +1,171 @@ +package scan + +import ( + "regexp" + "sort" + "strconv" + "strings" + + "github.com/datafog/datafog-api/internal/models" +) + +// EntityPattern pairs a compiled regex with an optional validator. +// If Validate is non-nil it is called on every regex match; only matches +// that return true are kept. This lets us do cheap regex first, then +// expensive checks (Luhn, IP-range) only on candidates. +type EntityPattern struct { + Re *regexp.Regexp + Validate func(match string) bool +} + +// DefaultEntityPatterns maps entity type names to their detection patterns. +var DefaultEntityPatterns = map[string]EntityPattern{ + "email": { + Re: regexp.MustCompile(`(?i)\b[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}\b`), + }, + "phone": { + Re: regexp.MustCompile(`(?:\+?1[-.\s]?)?\(?\d{3}\)?[-.\s]?\d{3}[-.\s]?\d{4}`), + }, + "ssn": { + Re: regexp.MustCompile(`\b\d{3}-\d{2}-\d{4}\b`), + }, + "api_key": { + Re: regexp.MustCompile(`(?i)\b(?:apikey|api[_-]?key|token)[:=]\s*[a-zA-Z0-9]{16,64}\b`), + }, + "credit_card": { + Re: regexp.MustCompile(`\b(?:\d[ -]*?){13,19}\b`), + Validate: luhnValid, + }, + "ip_address": { + Re: regexp.MustCompile(`\b(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})\b`), + Validate: ipv4Valid, + }, + "date": { + Re: regexp.MustCompile( + `\b(?:` + + `\d{4}[-/]\d{1,2}[-/]\d{1,2}` + // YYYY-MM-DD or YYYY/MM/DD + `|` + + `\d{1,2}[-/]\d{1,2}[-/]\d{2,4}` + // MM/DD/YYYY, DD/MM/YYYY, MM-DD-YY + `|` + + `(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)[a-z]*\.?\s+\d{1,2},?\s+\d{4}` + // Month DD, YYYY + `)\b`, + ), + }, + "zip_code": { + Re: regexp.MustCompile(`\b\d{5}(?:-\d{4})?\b`), + }, +} + +var DefaultEntityConfidences = map[string]float64{ + "email": 0.995, + "phone": 0.88, + "ssn": 0.99, + "api_key": 0.94, + "credit_card": 0.95, // higher now with Luhn validation + "ip_address": 0.90, + "date": 0.85, + "zip_code": 0.80, +} + +func ScanText(text string, entityFilter []string) []models.ScanFinding { + requested := map[string]struct{}{} + if len(entityFilter) > 0 { + for _, name := range entityFilter { + requested[strings.ToLower(strings.TrimSpace(name))] = struct{}{} + } + } + + findings := make([]models.ScanFinding, 0) + + // Phase 1: Regex engine (fast, always available) + entityTypes := make([]string, 0, len(DefaultEntityPatterns)) + for entityType := range DefaultEntityPatterns { + entityTypes = append(entityTypes, entityType) + } + sort.Strings(entityTypes) + + for _, entityType := range entityTypes { + pattern := DefaultEntityPatterns[entityType] + if len(requested) > 0 { + if _, ok := requested[entityType]; !ok { + continue + } + } + + idxs := pattern.Re.FindAllStringIndex(text, -1) + for _, idx := range idxs { + if len(idx) != 2 || idx[0] < 0 || idx[1] < idx[0] { + continue + } + value := text[idx[0]:idx[1]] + if pattern.Validate != nil && !pattern.Validate(value) { + continue + } + findings = append(findings, models.ScanFinding{ + EntityType: entityType, + Value: value, + Start: idx[0], + End: idx[1], + Confidence: DefaultEntityConfidences[entityType], + }) + } + } + + // Phase 2: NER engine (heuristic, when enabled) + nerFindings := ScanNER(text, entityFilter) + findings = append(findings, nerFindings...) + + return findings +} + +// luhnValid implements the Luhn algorithm to validate credit card numbers. +// It strips spaces and dashes before checking. +func luhnValid(s string) bool { + // Strip spaces and dashes + var digits []int + for _, ch := range s { + if ch >= '0' && ch <= '9' { + digits = append(digits, int(ch-'0')) + } else if ch == ' ' || ch == '-' { + continue + } else { + return false + } + } + if len(digits) < 13 || len(digits) > 19 { + return false + } + + sum := 0 + double := false + for i := len(digits) - 1; i >= 0; i-- { + d := digits[i] + if double { + d *= 2 + if d > 9 { + d -= 9 + } + } + sum += d + double = !double + } + return sum%10 == 0 +} + +// ipv4Valid checks that each octet is 0-255. +func ipv4Valid(s string) bool { + parts := strings.Split(s, ".") + if len(parts) != 4 { + return false + } + for _, part := range parts { + n, err := strconv.Atoi(part) + if err != nil { + return false + } + if n < 0 || n > 255 { + return false + } + } + return true +} diff --git a/internal/scan/detector_test.go b/internal/scan/detector_test.go new file mode 100644 index 0000000..6147424 --- /dev/null +++ b/internal/scan/detector_test.go @@ -0,0 +1,307 @@ +package scan + +import ( + "encoding/json" + "reflect" + "testing" +) + +type scanGoldenCase struct { + Name string `json:"name"` + Text string `json:"text"` + Filter []string `json:"filter"` + Findings []FindingExpectation `json:"expected_findings"` +} + +type FindingExpectation struct { + EntityType string `json:"entity_type"` + Start int `json:"start"` + End int `json:"end"` + Value string `json:"value"` +} + +func TestScanTextFindsEmailAndPhone(t *testing.T) { + text := "contact jane.doe+team@example.com or call +1 415-555-0199" + findings := ScanText(text, nil) + + var emailSeen, phoneSeen bool + for _, f := range findings { + switch f.EntityType { + case "email": + emailSeen = true + if f.Value != "jane.doe+team@example.com" { + t.Fatalf("email value mismatch: %q", f.Value) + } + case "phone": + phoneSeen = true + if f.Start >= f.End { + t.Fatalf("invalid phone span: %v", f) + } + } + } + + if !emailSeen || !phoneSeen { + t.Fatalf("expected email and phone findings, got %#v", findings) + } +} + +func TestScanTextFiltersByType(t *testing.T) { + text := "api_key=ABCD1234EFGH5678 and 111-22-3333" + findings := ScanText(text, []string{"ssn"}) + + if len(findings) != 1 { + t.Fatalf("expected one finding, got %d", len(findings)) + } + if findings[0].EntityType != "ssn" { + t.Fatalf("expected ssn finding, got %q", findings[0].EntityType) + } + if findings[0].Start != 29 || findings[0].End != 40 { + t.Fatalf("expected ssn span [29,40], got [%d,%d]", findings[0].Start, findings[0].End) + } +} + +func TestScanTextGoldenCorpus(t *testing.T) { + data := []scanGoldenCase{ + { + Name: "default email and phone", + Text: "contact jane.doe@example.com on +1 415-555-0100", + Findings: []FindingExpectation{ + {EntityType: "email", Start: 8, End: 28, Value: "jane.doe@example.com"}, + {EntityType: "phone", Start: 32, End: 47, Value: "+1 415-555-0100"}, + }, + }, + { + Name: "filter phone", + Text: "identity number 111-22-3333 and phone 555-123-4567", + Filter: []string{"phone"}, + Findings: []FindingExpectation{{EntityType: "phone", Start: 38, End: 50, Value: "555-123-4567"}}, + }, + } + + for _, vector := range data { + t.Run(vector.Name, func(t *testing.T) { + got := ScanText(vector.Text, vector.Filter) + if len(got) != len(vector.Findings) { + t.Fatalf("expected %d findings, got %d: %+v", len(vector.Findings), len(got), got) + } + + for idx, exp := range vector.Findings { + if got[idx].EntityType != exp.EntityType { + t.Fatalf("expected %q at %d, got %q", exp.EntityType, idx, got[idx].EntityType) + } + if got[idx].Value != exp.Value || got[idx].Start != exp.Start || got[idx].End != exp.End { + t.Fatalf("finding mismatch at %d: got %+v expected type=%s start=%d end=%d value=%q", idx, got[idx], exp.EntityType, exp.Start, exp.End, exp.Value) + } + } + }) + } +} + +func TestScanTextCorpusIsDeterministicWhenReloadedFromJSON(t *testing.T) { + data := []scanGoldenCase{ + { + Name: "cc detection", + Text: "card 4111111111111111", + Findings: []FindingExpectation{{ + EntityType: "credit_card", + Start: 5, + End: 21, + Value: "4111111111111111", + }}, + }, + } + + raw, err := json.Marshal(data) + if err != nil { + t.Fatalf("marshal failed: %v", err) + } + var vectors []scanGoldenCase + if err := json.Unmarshal(raw, &vectors); err != nil { + t.Fatalf("unmarshal failed: %v", err) + } + + first := ScanText(vectors[0].Text, nil) + second := ScanText(vectors[0].Text, nil) + if !reflect.DeepEqual(first, second) { + t.Fatalf("expected deterministic results, got %+v and %+v", first, second) + } +} + +func TestScanTextNoPanicOnMalformedUTF8(t *testing.T) { + malformed := string([]byte("contact ")) + malformed += string([]byte{0xff, 0xfe}) + malformed += " jane@example.com" + + defer func() { + if recovered := recover(); recovered != nil { + t.Fatalf("ScanText panicked on malformed UTF-8: %v", recovered) + } + }() + + _ = ScanText(malformed, nil) +} + +// --- New entity type tests --- + +func TestScanTextDetectsIPAddress(t *testing.T) { + text := "server at 192.168.1.1 and gateway 10.0.0.1" + findings := ScanText(text, []string{"ip_address"}) + + if len(findings) != 2 { + t.Fatalf("expected 2 ip_address findings, got %d: %+v", len(findings), findings) + } + if findings[0].Value != "192.168.1.1" { + t.Fatalf("expected 192.168.1.1, got %q", findings[0].Value) + } + if findings[1].Value != "10.0.0.1" { + t.Fatalf("expected 10.0.0.1, got %q", findings[1].Value) + } +} + +func TestScanTextRejectsInvalidIPAddress(t *testing.T) { + text := "invalid ip 999.999.999.999 should not match" + findings := ScanText(text, []string{"ip_address"}) + + if len(findings) != 0 { + t.Fatalf("expected 0 findings for invalid IP, got %d: %+v", len(findings), findings) + } +} + +func TestScanTextDetectsDate(t *testing.T) { + tests := []struct { + name string + text string + value string + }{ + {"ISO format", "born on 1990-01-15 in city", "1990-01-15"}, + {"US slash", "due date 01/15/2025 payment", "01/15/2025"}, + {"US dash", "due date 01-15-2025 payment", "01-15-2025"}, + {"Month name", "born on January 15, 2025 in city", "January 15, 2025"}, + {"Month abbrev", "born on Jan 15, 2025 in city", "Jan 15, 2025"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + findings := ScanText(tt.text, []string{"date"}) + if len(findings) == 0 { + t.Fatalf("expected date finding for %q, got none", tt.text) + } + found := false + for _, f := range findings { + if f.Value == tt.value { + found = true + break + } + } + if !found { + t.Fatalf("expected value %q in findings %+v", tt.value, findings) + } + }) + } +} + +func TestScanTextDetectsZipCode(t *testing.T) { + text := "address in 90210 or full zip 10001-1234" + findings := ScanText(text, []string{"zip_code"}) + + if len(findings) != 2 { + t.Fatalf("expected 2 zip_code findings, got %d: %+v", len(findings), findings) + } + if findings[0].Value != "90210" { + t.Fatalf("expected 90210, got %q", findings[0].Value) + } + if findings[1].Value != "10001-1234" { + t.Fatalf("expected 10001-1234, got %q", findings[1].Value) + } +} + +func TestScanTextCreditCardLuhnValidation(t *testing.T) { + // Valid Visa test number (passes Luhn) + text := "card 4111111111111111 is valid" + findings := ScanText(text, []string{"credit_card"}) + + if len(findings) != 1 { + t.Fatalf("expected 1 credit_card finding, got %d: %+v", len(findings), findings) + } + if findings[0].Value != "4111111111111111" { + t.Fatalf("expected 4111111111111111, got %q", findings[0].Value) + } + + // Invalid number (fails Luhn) + textInvalid := "card 1234567890123456 is invalid" + findingsInvalid := ScanText(textInvalid, []string{"credit_card"}) + if len(findingsInvalid) != 0 { + t.Fatalf("expected 0 credit_card findings for invalid number, got %d: %+v", len(findingsInvalid), findingsInvalid) + } +} + +func TestLuhnValid(t *testing.T) { + tests := []struct { + name string + input string + valid bool + }{ + {"Visa test", "4111111111111111", true}, + {"Mastercard test", "5500000000000004", true}, + {"Amex test", "378282246310005", true}, + {"With spaces", "4111 1111 1111 1111", true}, + {"With dashes", "4111-1111-1111-1111", true}, + {"Invalid", "1234567890123456", false}, + {"Too short", "123", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := luhnValid(tt.input) + if got != tt.valid { + t.Fatalf("luhnValid(%q) = %v, want %v", tt.input, got, tt.valid) + } + }) + } +} + +func TestIPv4Valid(t *testing.T) { + tests := []struct { + name string + input string + valid bool + }{ + {"normal", "192.168.1.1", true}, + {"zeros", "0.0.0.0", true}, + {"max", "255.255.255.255", true}, + {"overflow", "256.1.1.1", false}, + {"overflow octet 4", "1.1.1.999", false}, + {"too few octets", "192.168.1", false}, + {"letters", "abc.def.ghi.jkl", false}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ipv4Valid(tt.input) + if got != tt.valid { + t.Fatalf("ipv4Valid(%q) = %v, want %v", tt.input, got, tt.valid) + } + }) + } +} + +func TestScanTextAllEntityTypes(t *testing.T) { + text := `Contact john@example.com or call 555-123-4567. +SSN: 123-45-6789. API key: api_key=Abc1234567890123456. +Card: 4111111111111111. Server: 192.168.1.100. +Born: 1990-01-15. Zip: 90210.` + + findings := ScanText(text, nil) + + found := map[string]bool{} + for _, f := range findings { + found[f.EntityType] = true + } + + expected := []string{"email", "phone", "ssn", "api_key", "credit_card", "ip_address", "date", "zip_code"} + for _, et := range expected { + if !found[et] { + t.Errorf("expected entity type %q to be detected, found types: %v", et, found) + } + } +} diff --git a/internal/scan/ner.go b/internal/scan/ner.go new file mode 100644 index 0000000..9e5113f --- /dev/null +++ b/internal/scan/ner.go @@ -0,0 +1,353 @@ +package scan + +import ( + "strings" + "unicode" + + "github.com/datafog/datafog-api/internal/models" +) + +// NER provides a lightweight, dictionary+heuristic named entity recognizer. +// It detects PERSON, ORGANIZATION, and LOCATION entities using: +// - Title-cased word sequences (2+ words starting with uppercase) +// - Contextual triggers ("Mr.", "Dr.", "Inc.", "Corp.", "in", "at") +// - Dictionaries of common names, organizations, and locations +// +// This is intentionally simple: no ML, no cgo, no external deps. +// Accuracy is traded for zero-dependency deployment. + +var nerConfidences = map[string]float64{ + "person": 0.70, + "organization": 0.65, + "location": 0.65, +} + +// personTriggers precede person names. +var personTriggers = map[string]bool{ + "mr": true, + "mr.": true, + "mrs": true, + "mrs.": true, + "ms": true, + "ms.": true, + "dr": true, + "dr.": true, + "prof": true, + "prof.": true, + "sir": true, + "madam": true, + "captain": true, + "capt": true, + "capt.": true, +} + +// orgSuffixes identify organization names. +var orgSuffixes = map[string]bool{ + "inc": true, + "inc.": true, + "corp": true, + "corp.": true, + "corporation": true, + "llc": true, + "llp": true, + "ltd": true, + "ltd.": true, + "co": true, + "co.": true, + "company": true, + "group": true, + "holdings": true, + "foundation": true, + "institute": true, + "university": true, + "association": true, + "technologies": true, + "systems": true, + "partners": true, + "labs": true, + "studios": true, +} + +// locationTriggers precede location names. +var locationTriggers = map[string]bool{ + "in": true, + "at": true, + "from": true, + "near": true, + "city": true, + "state": true, + "town": true, +} + +// commonFirstNames is a small set of frequently occurring first names. +var commonFirstNames = map[string]bool{ + "james": true, "john": true, "robert": true, "michael": true, + "william": true, "david": true, "richard": true, "joseph": true, + "thomas": true, "charles": true, "christopher": true, "daniel": true, + "matthew": true, "anthony": true, "mark": true, "donald": true, + "steven": true, "paul": true, "andrew": true, "joshua": true, + "mary": true, "patricia": true, "jennifer": true, "linda": true, + "elizabeth": true, "barbara": true, "susan": true, "jessica": true, + "sarah": true, "karen": true, "nancy": true, "lisa": true, + "margaret": true, "betty": true, "sandra": true, "ashley": true, + "emily": true, "donna": true, "michelle": true, "dorothy": true, + "alice": true, "jane": true, "alex": true, "sam": true, + "benjamin": true, "alexander": true, "peter": true, "george": true, + "edward": true, "henry": true, "jack": true, "oliver": true, + "emma": true, "sophia": true, "ava": true, "isabella": true, +} + +// wellKnownLocations covers major cities and countries. +var wellKnownLocations = map[string]bool{ + "new york": true, "los angeles": true, "chicago": true, + "houston": true, "phoenix": true, "philadelphia": true, + "san antonio": true, "san diego": true, "dallas": true, + "san jose": true, "san francisco": true, "seattle": true, + "denver": true, "boston": true, "nashville": true, + "washington": true, "atlanta": true, "miami": true, + "london": true, "paris": true, "tokyo": true, + "berlin": true, "sydney": true, "toronto": true, + "mumbai": true, "beijing": true, "shanghai": true, + "singapore": true, "dubai": true, "amsterdam": true, + "california": true, "texas": true, "florida": true, + "new jersey": true, "virginia": true, "massachusetts": true, + "united states": true, "united kingdom": true, "canada": true, + "australia": true, "germany": true, "france": true, + "japan": true, "china": true, "india": true, "brazil": true, +} + +// NEREnabled controls whether the NER engine runs. Can be toggled via env var. +var NEREnabled = true + +// ScanNER runs the heuristic NER engine over text and returns findings +// for person, organization, and location entities. +func ScanNER(text string, entityFilter []string) []models.ScanFinding { + if !NEREnabled { + return nil + } + + requested := map[string]struct{}{} + if len(entityFilter) > 0 { + for _, name := range entityFilter { + requested[strings.ToLower(strings.TrimSpace(name))] = struct{}{} + } + } + + wantPerson := len(requested) == 0 || hasKey(requested, "person") + wantOrg := len(requested) == 0 || hasKey(requested, "organization") + wantLoc := len(requested) == 0 || hasKey(requested, "location") + + findings := make([]models.ScanFinding, 0) + + tokens := tokenize(text) + + for i := 0; i < len(tokens); i++ { + tok := tokens[i] + + // Skip non-title-cased words (heuristic: NER entities start with uppercase) + if !isTitleCase(tok.text) { + continue + } + + // Gather consecutive title-cased words + span := []tokenInfo{tok} + for j := i + 1; j < len(tokens); j++ { + next := tokens[j] + // Allow small connecting words within multi-word names + if isConnector(next.text) && j+1 < len(tokens) && isTitleCase(tokens[j+1].text) { + span = append(span, next) + continue + } + if isTitleCase(next.text) { + span = append(span, next) + } else { + break + } + } + + fullText := buildSpanText(text, span) + start := span[0].start + end := span[len(span)-1].end + + // Check what the preceding word is for context + prevWord := "" + if i > 0 { + prevWord = strings.ToLower(tokens[i-1].text) + } + + // Detect if first word in span is likely sentence-initial (not a proper name) + sentenceInitial := i == 0 || isSentenceEnd(tokens[i-1].text) + + // Check the last word in the span for org suffixes + lastWord := strings.ToLower(span[len(span)-1].text) + + // Classification using heuristics + if wantOrg && len(span) >= 1 && orgSuffixes[lastWord] { + // For orgs, trim sentence-initial words that are unlikely part of the name. + // Walk forward from start to find the actual org name beginning. + orgSpan := span + if sentenceInitial && len(span) > 1 { + firstLower := strings.ToLower(span[0].text) + if !commonFirstNames[firstLower] && !orgSuffixes[firstLower] { + orgSpan = span[1:] + } + } + orgText := buildSpanText(text, orgSpan) + findings = append(findings, models.ScanFinding{ + EntityType: "organization", + Value: orgText, + Start: orgSpan[0].start, + End: orgSpan[len(orgSpan)-1].end, + Confidence: nerConfidences["organization"], + }) + i += len(span) - 1 + continue + } + + if wantPerson && personTriggers[prevWord] && len(span) >= 1 { + findings = append(findings, models.ScanFinding{ + EntityType: "person", + Value: fullText, + Start: start, + End: end, + Confidence: nerConfidences["person"] + 0.1, // higher confidence with trigger + }) + i += len(span) - 1 + continue + } + + if wantPerson && len(span) >= 2 { + firstLower := strings.ToLower(span[0].text) + if commonFirstNames[firstLower] { + findings = append(findings, models.ScanFinding{ + EntityType: "person", + Value: fullText, + Start: start, + End: end, + Confidence: nerConfidences["person"], + }) + i += len(span) - 1 + continue + } + } + + if wantLoc { + lowerFull := strings.ToLower(fullText) + if wellKnownLocations[lowerFull] { + findings = append(findings, models.ScanFinding{ + EntityType: "location", + Value: fullText, + Start: start, + End: end, + Confidence: nerConfidences["location"] + 0.15, // dictionary match + }) + i += len(span) - 1 + continue + } + } + + if wantLoc && locationTriggers[prevWord] && len(span) >= 1 { + findings = append(findings, models.ScanFinding{ + EntityType: "location", + Value: fullText, + Start: start, + End: end, + Confidence: nerConfidences["location"], + }) + i += len(span) - 1 + continue + } + } + + return findings +} + +type tokenInfo struct { + text string + start int + end int +} + +func tokenize(text string) []tokenInfo { + tokens := make([]tokenInfo, 0) + i := 0 + runes := []rune(text) + n := len(runes) + + for i < n { + // Skip whitespace + for i < n && unicode.IsSpace(runes[i]) { + i++ + } + if i >= n { + break + } + + start := i + // Collect word characters (letters, digits, apostrophes, periods for abbreviations) + for i < n && !unicode.IsSpace(runes[i]) { + i++ + } + + word := string(runes[start:i]) + // Strip trailing punctuation except periods (for abbreviations like "Mr.") + trimmed := strings.TrimRight(word, ",;:!?\"')") + if trimmed == "" { + continue + } + endPos := start + len([]rune(trimmed)) + tokens = append(tokens, tokenInfo{ + text: trimmed, + start: byteOffset(text, start), + end: byteOffset(text, endPos), + }) + } + + return tokens +} + +func byteOffset(text string, runeIdx int) int { + runes := []rune(text) + if runeIdx >= len(runes) { + return len(text) + } + return len(string(runes[:runeIdx])) +} + +func isTitleCase(s string) bool { + runes := []rune(s) + if len(runes) == 0 { + return false + } + return unicode.IsUpper(runes[0]) && len(runes) > 1 +} + +func isConnector(s string) bool { + lower := strings.ToLower(s) + return lower == "of" || lower == "the" || lower == "and" || lower == "de" || lower == "van" || lower == "von" +} + +func buildSpanText(text string, span []tokenInfo) string { + if len(span) == 0 { + return "" + } + start := span[0].start + end := span[len(span)-1].end + if start < 0 || end > len(text) || start >= end { + return "" + } + return text[start:end] +} + +func isSentenceEnd(s string) bool { + if len(s) == 0 { + return false + } + last := s[len(s)-1] + return last == '.' || last == '!' || last == '?' +} + +func hasKey(m map[string]struct{}, key string) bool { + _, ok := m[key] + return ok +} diff --git a/internal/scan/ner_test.go b/internal/scan/ner_test.go new file mode 100644 index 0000000..a48adfd --- /dev/null +++ b/internal/scan/ner_test.go @@ -0,0 +1,134 @@ +package scan + +import ( + "testing" +) + +func TestScanNERDetectsPerson(t *testing.T) { + tests := []struct { + name string + text string + expected string + }{ + {"titled trigger", "Contact Mr. John Smith for details", "John Smith"}, + {"Dr trigger", "Refer to Dr. Jane Williams immediately", "Jane Williams"}, + {"common first name", "Meeting with Sarah Johnson tomorrow", "Sarah Johnson"}, + {"common first name 2", "Email from Michael Chen about project", "Michael Chen"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + findings := ScanNER(tt.text, []string{"person"}) + found := false + for _, f := range findings { + if f.EntityType == "person" && f.Value == tt.expected { + found = true + break + } + } + if !found { + t.Fatalf("expected person %q in %q, got findings: %+v", tt.expected, tt.text, findings) + } + }) + } +} + +func TestScanNERDetectsOrganization(t *testing.T) { + tests := []struct { + name string + text string + expected string + }{ + {"inc suffix", "Filed by Acme Inc. yesterday", "Acme Inc."}, + {"corp suffix", "Work for DataFog Corp in tech", "DataFog Corp"}, + {"llc suffix", "Founded Bright Solutions LLC last year", "Bright Solutions LLC"}, + {"university suffix", "Studied at Stanford University for years", "Stanford University"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + findings := ScanNER(tt.text, []string{"organization"}) + found := false + for _, f := range findings { + if f.EntityType == "organization" && f.Value == tt.expected { + found = true + break + } + } + if !found { + t.Fatalf("expected organization %q in %q, got findings: %+v", tt.expected, tt.text, findings) + } + }) + } +} + +func TestScanNERDetectsLocation(t *testing.T) { + tests := []struct { + name string + text string + expected string + }{ + {"dictionary city", "Office in San Francisco downtown", "San Francisco"}, + {"dictionary city 2", "Moved to New York for work", "New York"}, + {"trigger word", "Lives in Portland with family", "Portland"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + findings := ScanNER(tt.text, []string{"location"}) + found := false + for _, f := range findings { + if f.EntityType == "location" && f.Value == tt.expected { + found = true + break + } + } + if !found { + t.Fatalf("expected location %q in %q, got findings: %+v", tt.expected, tt.text, findings) + } + }) + } +} + +func TestScanNERDisabledReturnsNothing(t *testing.T) { + NEREnabled = false + defer func() { NEREnabled = true }() + + findings := ScanNER("Contact Mr. John Smith at Acme Corp in New York", nil) + if len(findings) != 0 { + t.Fatalf("expected 0 findings when NER disabled, got %d: %+v", len(findings), findings) + } +} + +func TestScanNERFiltering(t *testing.T) { + text := "Mr. John Smith works at Acme Corp in New York" + + // Only request person + findings := ScanNER(text, []string{"person"}) + for _, f := range findings { + if f.EntityType != "person" { + t.Fatalf("expected only person findings, got %q", f.EntityType) + } + } +} + +func TestScanNERIntegration(t *testing.T) { + // Test that ScanText includes NER results alongside regex results + text := "Contact Mr. John Smith at john@example.com or 555-123-4567" + findings := ScanText(text, nil) + + foundTypes := map[string]bool{} + for _, f := range findings { + foundTypes[f.EntityType] = true + } + + if !foundTypes["person"] { + t.Error("expected person entity from NER") + } + if !foundTypes["email"] { + t.Error("expected email entity from regex") + } + if !foundTypes["phone"] { + t.Error("expected phone entity from regex") + } +} diff --git a/internal/server/demo.go b/internal/server/demo.go new file mode 100644 index 0000000..c4e3b27 --- /dev/null +++ b/internal/server/demo.go @@ -0,0 +1,307 @@ +package server + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/datafog/datafog-api/internal/models" + "github.com/datafog/datafog-api/internal/scan" + "github.com/datafog/datafog-api/internal/shim" +) + +// DemoHandler exposes endpoints that execute real commands and file +// operations through the shim gate. Must be explicitly enabled. +type DemoHandler struct { + gate *shim.Gate + sandboxDir string + server *Server + demoHTML []byte +} + +type demoExecRequest struct { + Command string `json:"command"` + Args []string `json:"args"` + Stdin string `json:"stdin,omitempty"` +} + +type demoExecResponse struct { + Decision models.DecideResponse `json:"decision"` + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + Error string `json:"error,omitempty"` + Blocked bool `json:"blocked"` + TimingMs int64 `json:"timing_ms"` + Findings []models.ScanFinding `json:"findings,omitempty"` +} + +type demoWriteRequest struct { + Filename string `json:"filename"` + Content string `json:"content"` +} + +type demoWriteResponse struct { + Decision models.DecideResponse `json:"decision"` + Written bool `json:"written"` + Path string `json:"path"` + Content string `json:"content"` + Error string `json:"error,omitempty"` + Blocked bool `json:"blocked"` + TimingMs int64 `json:"timing_ms"` + Findings []models.ScanFinding `json:"findings,omitempty"` +} + +type demoReadRequest struct { + Filename string `json:"filename"` +} + +type demoReadResponse struct { + Decision models.DecideResponse `json:"decision"` + Content string `json:"content"` + Error string `json:"error,omitempty"` + Blocked bool `json:"blocked"` + TimingMs int64 `json:"timing_ms"` + Findings []models.ScanFinding `json:"findings,omitempty"` +} + +// NewDemoHandler creates a demo handler backed by the given gate. +// It creates a sandbox directory for file operations. +func NewDemoHandler(gate *shim.Gate, srv *Server, demoHTMLPath string) (*DemoHandler, error) { + sandboxDir, err := os.MkdirTemp("", "datafog-demo-*") + if err != nil { + return nil, fmt.Errorf("create demo sandbox: %w", err) + } + var html []byte + if demoHTMLPath != "" { + html, err = os.ReadFile(demoHTMLPath) + if err != nil { + return nil, fmt.Errorf("read demo HTML: %w", err) + } + } + return &DemoHandler{ + gate: gate, + sandboxDir: sandboxDir, + server: srv, + demoHTML: html, + }, nil +} + +// Cleanup removes the sandbox directory. +func (d *DemoHandler) Cleanup() { + if d.sandboxDir != "" { + os.RemoveAll(d.sandboxDir) + } +} + +// Register adds the demo endpoints to the given mux. +func (d *DemoHandler) Register(mux *http.ServeMux) { + mux.HandleFunc("/demo", d.handleDemoPage) + mux.HandleFunc("/demo/exec", d.handleExec) + mux.HandleFunc("/demo/write-file", d.handleWriteFile) + mux.HandleFunc("/demo/read-file", d.handleReadFile) + mux.HandleFunc("/demo/seed", d.handleSeed) + mux.HandleFunc("/demo/sandbox", d.handleSandboxInfo) +} + +func (d *DemoHandler) handleDemoPage(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Write(d.demoHTML) +} + +func (d *DemoHandler) handleExec(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + d.server.respondError(w, http.StatusMethodNotAllowed, models.APIError{Code: "method_not_allowed", Message: "method must be POST"}) + return + } + + var req demoExecRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + d.server.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: err.Error()}) + return + } + if req.Command == "" { + d.server.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "command is required"}) + return + } + + // Scan the stdin/context for PII + textToScan := req.Stdin + if textToScan == "" { + textToScan = req.Command + " " + strings.Join(req.Args, " ") + } + findings := scan.ScanText(textToScan, nil) + + start := time.Now() + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + + decision, output, err := d.gate.ExecuteShell(ctx, req.Command, req.Args, textToScan, findings, len(findings) > 0) + elapsed := time.Since(start).Milliseconds() + + resp := demoExecResponse{ + Decision: decision, + TimingMs: elapsed, + Findings: findings, + } + + if err != nil { + resp.Blocked = true + resp.Error = err.Error() + } else { + resp.Stdout = string(output) + } + + d.server.respond(w, http.StatusOK, resp) +} + +func (d *DemoHandler) handleWriteFile(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + d.server.respondError(w, http.StatusMethodNotAllowed, models.APIError{Code: "method_not_allowed", Message: "method must be POST"}) + return + } + + var req demoWriteRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + d.server.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: err.Error()}) + return + } + if req.Filename == "" { + d.server.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "filename is required"}) + return + } + + // Sanitize filename to prevent directory traversal + cleanName := filepath.Base(req.Filename) + fullPath := filepath.Join(d.sandboxDir, cleanName) + + // Scan content for PII + findings := scan.ScanText(req.Content, nil) + + start := time.Now() + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + + decision, err := d.gate.WriteFile(ctx, fullPath, []byte(req.Content), 0o600, req.Content, findings, len(findings) > 0) + elapsed := time.Since(start).Milliseconds() + + resp := demoWriteResponse{ + Decision: decision, + Path: cleanName, + TimingMs: elapsed, + Findings: findings, + } + + if err != nil { + resp.Blocked = true + resp.Error = err.Error() + } else { + resp.Written = true + // Read back what was actually written (may be redacted) + if data, readErr := os.ReadFile(fullPath); readErr == nil { + resp.Content = string(data) + } + } + + d.server.respond(w, http.StatusOK, resp) +} + +func (d *DemoHandler) handleReadFile(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + d.server.respondError(w, http.StatusMethodNotAllowed, models.APIError{Code: "method_not_allowed", Message: "method must be POST"}) + return + } + + var req demoReadRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + d.server.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: err.Error()}) + return + } + if req.Filename == "" { + d.server.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "filename is required"}) + return + } + + cleanName := filepath.Base(req.Filename) + fullPath := filepath.Join(d.sandboxDir, cleanName) + + start := time.Now() + ctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + defer cancel() + + decision, output, err := d.gate.ReadFile(ctx, fullPath, "", nil, false) + elapsed := time.Since(start).Milliseconds() + + resp := demoReadResponse{ + Decision: decision, + TimingMs: elapsed, + } + + if err != nil { + resp.Blocked = true + resp.Error = err.Error() + } else { + resp.Content = string(output) + // Scan the output to report what was found + resp.Findings = scan.ScanText(string(output), nil) + } + + d.server.respond(w, http.StatusOK, resp) +} + +// handleSeed writes a file directly to the sandbox, bypassing the shim gate. +// This lets demo scenarios place raw PII on disk so that a subsequent +// gated read can demonstrate redaction on the way out. +func (d *DemoHandler) handleSeed(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + d.server.respondError(w, http.StatusMethodNotAllowed, models.APIError{Code: "method_not_allowed", Message: "method must be POST"}) + return + } + + var req demoWriteRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + d.server.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: err.Error()}) + return + } + if req.Filename == "" || req.Content == "" { + d.server.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "filename and content are required"}) + return + } + + cleanName := filepath.Base(req.Filename) + fullPath := filepath.Join(d.sandboxDir, cleanName) + + if err := os.WriteFile(fullPath, []byte(req.Content), 0o600); err != nil { + d.server.respondError(w, http.StatusInternalServerError, models.APIError{Code: "seed_error", Message: err.Error()}) + return + } + + d.server.respond(w, http.StatusOK, map[string]interface{}{ + "seeded": true, + "filename": cleanName, + }) +} + +func (d *DemoHandler) handleSandboxInfo(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + d.server.respondError(w, http.StatusMethodNotAllowed, models.APIError{Code: "method_not_allowed", Message: "method must be GET"}) + return + } + + entries, _ := os.ReadDir(d.sandboxDir) + files := make([]string, 0, len(entries)) + for _, entry := range entries { + if !entry.IsDir() { + files = append(files, entry.Name()) + } + } + + d.server.respond(w, http.StatusOK, map[string]interface{}{ + "sandbox_dir": d.sandboxDir, + "files": files, + }) +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000..5393f3e --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,919 @@ +package server + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "crypto/subtle" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "log" + "math" + "mime" + "net/http" + "strconv" + "strings" + "sync" + "time" + + "github.com/datafog/datafog-api/internal/models" + "github.com/datafog/datafog-api/internal/policy" + "github.com/datafog/datafog-api/internal/receipts" + "github.com/datafog/datafog-api/internal/scan" + "github.com/datafog/datafog-api/internal/shim" + "github.com/datafog/datafog-api/internal/transform" +) + +type Server struct { + policy models.Policy + store *receipts.ReceiptStore + eventReader shim.EventReader + apiToken string + rateLimiter *tokenBucket + startedAt time.Time + logger *log.Logger + mu sync.Mutex + statsMu sync.Mutex + decisions map[string]idempotentDecision + scans map[string]idempotentCachedResponse + transforms map[string]idempotentCachedResponse + anonymizes map[string]idempotentCachedResponse + totalCount int64 + errorCount int64 + statusHits map[int]int64 + pathHits map[string]int64 + methodHits map[string]int64 +} + +type requestIDContextKey struct{} + +type responseStatusWriter struct { + http.ResponseWriter + status int +} + +func (w *responseStatusWriter) WriteHeader(code int) { + w.status = code + w.ResponseWriter.WriteHeader(code) +} + +func (w *responseStatusWriter) Write(body []byte) (int, error) { + if w.status == 0 { + w.status = http.StatusOK + } + return w.ResponseWriter.Write(body) +} + +const ( + maxRequestBodyBytes int64 = 1024 * 1024 // 1 MiB +) + +type idempotentDecision struct { + requestHash string + response models.DecideResponse +} + +type idempotentCachedResponse struct { + requestHash string + body []byte + status int +} + +type metricsResponse struct { + TotalRequests int64 `json:"total_requests"` + ErrorRequests int64 `json:"error_requests"` + ByStatus map[string]int64 `json:"by_status"` + ByPath map[string]int64 `json:"by_path"` + ByMethod map[string]int64 `json:"by_method"` + StartedAt string `json:"started_at"` + UptimeSeconds float64 `json:"uptime_seconds"` +} + +func New(policyData models.Policy, store *receipts.ReceiptStore, logger *log.Logger, apiToken string, rateLimitRPS int) *Server { + if logger == nil { + logger = log.Default() + } + return &Server{ + policy: policyData, + store: store, + apiToken: apiToken, + rateLimiter: newTokenBucket(rateLimitRPS), + startedAt: time.Now().UTC(), + logger: logger, + decisions: map[string]idempotentDecision{}, + scans: map[string]idempotentCachedResponse{}, + transforms: map[string]idempotentCachedResponse{}, + anonymizes: map[string]idempotentCachedResponse{}, + statusHits: map[int]int64{}, + pathHits: map[string]int64{}, + methodHits: map[string]int64{}, + } +} + +func (s *Server) SetEventReader(reader shim.EventReader) { + s.eventReader = reader +} + +// HandlerWithDemo returns the HTTP handler with optional demo endpoints registered. +func (s *Server) HandlerWithDemo(demo *DemoHandler) http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/health", s.handleHealth) + mux.HandleFunc("/v1/policy/version", s.handlePolicyVersion) + mux.HandleFunc("/v1/scan", s.handleScan) + mux.HandleFunc("/v1/decide", s.handleDecide) + mux.HandleFunc("/v1/transform", s.handleTransform) + mux.HandleFunc("/v1/anonymize", s.handleAnonymize) + mux.HandleFunc("/v1/receipts/", s.handleReceipt) + mux.HandleFunc("/v1/events", s.handleEvents) + mux.HandleFunc("/metrics", s.handleMetrics) + if demo != nil { + demo.Register(mux) + } + return s.wrapMiddleware(mux) +} + +func (s *Server) Handler() http.Handler { + return s.HandlerWithDemo(nil) +} + +func (s *Server) wrapMiddleware(mux *http.ServeMux) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + origin := r.Header.Get("Origin") + if origin != "" { + w.Header().Set("Access-Control-Allow-Origin", origin) + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type, Authorization, X-API-Key, X-Request-ID") + w.Header().Set("Access-Control-Expose-Headers", "X-Request-ID") + } + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusNoContent) + return + } + + reqID := requestID(r) + if reqID == "" { + reqID = newRequestID() + } + r = r.WithContext(context.WithValue(r.Context(), requestIDContextKey{}, reqID)) + w.Header().Set("X-Request-ID", reqID) + + responseWriter := &responseStatusWriter{ResponseWriter: w} + startedAt := time.Now() + handler, pattern := mux.Handler(r) + defer func() { + if rec := recover(); rec != nil { + responseWriter.status = http.StatusInternalServerError + s.logger.Printf("request panic request_id=%s method=%s path=%s err=%v", reqID, r.Method, r.URL.Path, rec) + s.respondError(responseWriter, http.StatusInternalServerError, models.APIError{Code: "internal_error", Message: "internal server error", RequestID: reqID}) + } + if responseWriter.status == 0 { + responseWriter.status = http.StatusOK + } + if pattern == "" { + s.recordRequestMetrics(r.Method, "/_not_found", responseWriter.status) + } else { + s.recordRequestMetrics(r.Method, canonicalizedRoute(pattern, r.URL.Path), responseWriter.status) + } + s.logger.Printf("request complete request_id=%s method=%s path=%s status=%d latency_ms=%d", reqID, r.Method, r.URL.Path, responseWriter.status, time.Since(startedAt).Milliseconds()) + }() + + if !s.authorized(r) { + s.respondError(responseWriter, http.StatusUnauthorized, models.APIError{Code: "unauthorized", Message: "missing or invalid API token", RequestID: reqID}) + return + } + if !s.rateLimiter.allow() { + s.respondError(responseWriter, http.StatusTooManyRequests, models.APIError{Code: "rate_limited", Message: "request rate limit exceeded", RequestID: reqID}) + return + } + + if pattern == "" { + s.respondError(responseWriter, http.StatusNotFound, models.APIError{Code: "not_found", Message: "endpoint not found", RequestID: reqID}) + return + } + handler.ServeHTTP(responseWriter, r) + }) +} + +func (s *Server) authorized(r *http.Request) bool { + if s.apiToken == "" { + return true + } + + if token := authorizationToken(r.Header.Get("Authorization")); token != "" && constantTimeTokenEqual(token, s.apiToken) { + return true + } + + if token := strings.TrimSpace(r.Header.Get("X-API-Key")); token != "" && constantTimeTokenEqual(token, s.apiToken) { + return true + } + + return false +} + +type tokenBucket struct { + mu sync.Mutex + tokens float64 + rate float64 + capacity float64 + lastTime time.Time +} + +func newTokenBucket(rateLimit int) *tokenBucket { + if rateLimit <= 0 { + return nil + } + + rate := float64(rateLimit) + return &tokenBucket{ + tokens: rate, + rate: rate, + capacity: rate, + lastTime: time.Now(), + } +} + +func (tb *tokenBucket) allow() bool { + if tb == nil { + return true + } + + tb.mu.Lock() + defer tb.mu.Unlock() + + now := time.Now() + delta := now.Sub(tb.lastTime).Seconds() + if delta > 0 { + tb.tokens = math.Min(tb.capacity, tb.tokens+(delta*tb.rate)) + tb.lastTime = now + } + if tb.tokens < 1 { + return false + } + tb.tokens-- + return true +} + +func authorizationToken(value string) string { + parts := strings.Fields(strings.TrimSpace(value)) + if len(parts) != 2 || !strings.EqualFold(parts[0], "Bearer") { + return "" + } + return parts[1] +} + +func constantTimeTokenEqual(provided, expected string) bool { + return subtle.ConstantTimeCompare([]byte(provided), []byte(expected)) == 1 +} + +func canonicalizedRoute(pattern string, path string) string { + if strings.HasSuffix(pattern, "/") && strings.HasPrefix(path, "/v1/receipts/") { + return "/v1/receipts/{id}" + } + return pattern +} + +func (s *Server) recordRequestMetrics(method string, route string, status int) { + s.statsMu.Lock() + defer s.statsMu.Unlock() + s.totalCount++ + s.methodHits[method]++ + s.pathHits[route]++ + s.statusHits[status]++ + if status >= 400 { + s.errorCount++ + } +} + +func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + s.respondError(w, http.StatusMethodNotAllowed, models.APIError{Code: "method_not_allowed", Message: "method must be GET", RequestID: requestID(r)}) + return + } + res := models.HealthResponse{ + Status: "ok", + PolicyID: s.policy.PolicyID, + PolicyVersion: s.policy.PolicyVersion, + StartedAt: s.startedAt.Format(time.RFC3339), + } + s.respond(w, http.StatusOK, res) +} + +func (s *Server) handlePolicyVersion(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + s.respondError(w, http.StatusMethodNotAllowed, models.APIError{Code: "method_not_allowed", Message: "method must be GET", RequestID: requestID(r)}) + return + } + s.respond(w, http.StatusOK, map[string]string{ + "policy_id": s.policy.PolicyID, + "policy_version": s.policy.PolicyVersion, + }) +} + +func (s *Server) handleScan(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + s.respondError(w, http.StatusMethodNotAllowed, models.APIError{Code: "method_not_allowed", Message: "method must be POST", RequestID: requestID(r)}) + return + } + if !isJSONContentType(r.Header.Get("Content-Type")) { + s.respondError(w, http.StatusUnsupportedMediaType, models.APIError{Code: "unsupported_media_type", Message: "content-type must be application/json", RequestID: requestID(r)}) + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxRequestBodyBytes) + + var req models.ScanRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + s.respondErrorFromDecodeErr(w, r, err) + return + } + if req.Text == "" { + s.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "text is required", RequestID: requestID(r)}) + return + } + if req.IdempotencyKey != "" { + reqHash, err := hashScanRequest(req) + if err != nil { + s.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "unable to hash request payload", Details: err.Error(), RequestID: requestID(r)}) + return + } + s.mu.Lock() + existing, ok := s.scans[req.IdempotencyKey] + s.mu.Unlock() + if ok { + if existing.requestHash != reqHash { + s.respondError(w, http.StatusConflict, models.APIError{Code: "idempotency_conflict", Message: "different request payload for same idempotency_key", RequestID: requestID(r)}) + return + } + s.respondRaw(w, existing.status, existing.body) + return + } + } + + findings := scan.ScanText(req.Text, req.EntityTypes) + res := models.ScanResponse{ + RequestID: req.RequestID, + TraceID: req.TraceID, + Findings: findings, + PolicyVersion: s.policy.PolicyVersion, + PolicyID: s.policy.PolicyID, + } + if req.IdempotencyKey != "" { + body, err := json.Marshal(res) + if err != nil { + s.respondError(w, http.StatusInternalServerError, models.APIError{Code: "encode_error", Message: "unable to encode response", Details: err.Error(), RequestID: requestID(r)}) + return + } + hash, _ := hashScanRequest(req) + s.mu.Lock() + s.scans[req.IdempotencyKey] = idempotentCachedResponse{ + requestHash: hash, + body: body, + status: http.StatusOK, + } + s.mu.Unlock() + s.respondRaw(w, http.StatusOK, body) + return + } + s.respond(w, http.StatusOK, res) +} + +func (s *Server) handleDecide(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + s.respondError(w, http.StatusMethodNotAllowed, models.APIError{Code: "method_not_allowed", Message: "method must be POST", RequestID: requestID(r)}) + return + } + if !isJSONContentType(r.Header.Get("Content-Type")) { + s.respondError(w, http.StatusUnsupportedMediaType, models.APIError{Code: "unsupported_media_type", Message: "content-type must be application/json", RequestID: requestID(r)}) + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxRequestBodyBytes) + + var req models.DecideRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + s.respondErrorFromDecodeErr(w, r, err) + return + } + if req.Action.Type == "" { + s.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "action.type is required", RequestID: requestID(r)}) + return + } + if req.IdempotencyKey != "" { + reqHash, err := hashDecideRequest(req) + if err != nil { + s.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "unable to hash request payload", Details: err.Error(), RequestID: requestID(r)}) + return + } + s.mu.Lock() + existing, ok := s.decisions[req.IdempotencyKey] + s.mu.Unlock() + if ok { + if existing.requestHash != reqHash { + s.respondError(w, http.StatusConflict, models.APIError{Code: "idempotency_conflict", Message: "different request payload for same idempotency_key", RequestID: requestID(r)}) + return + } + s.respond(w, http.StatusOK, existing.response) + return + } + } + + findings := req.Findings + if len(findings) == 0 && req.Text != "" { + findings = scan.ScanText(req.Text, nil) + } + result := policy.Evaluate(s.policy, policy.DecisionContext{Action: req.Action, Findings: findings}) + actionHash, err := hashDecideAction(req.Action) + if err != nil { + s.respondError(w, http.StatusInternalServerError, models.APIError{Code: "hash_error", Message: "unable to hash action", Details: err.Error(), RequestID: requestID(r)}) + return + } + inputHash, err := hashDecideInput(req) + if err != nil { + s.respondError(w, http.StatusInternalServerError, models.APIError{Code: "hash_error", Message: "unable to hash request input", Details: err.Error(), RequestID: requestID(r)}) + return + } + receipt := s.store.NewReceipt(req, result.Decision, result, s.policy) + receipt.Findings = findings + receipt.ActionHash = actionHash + receipt.InputHash = inputHash + if len(result.TransformPlan) > 0 { + summary, err := json.Marshal(result.TransformPlan) + if err == nil { + receipt.SanitizedSummary = string(summary) + } else { + receipt.SanitizedSummary = `transform plan unavailable` + } + } + saved, err := s.store.Save(receipt) + if err != nil { + s.respondError(w, http.StatusInternalServerError, models.APIError{Code: "receipt_error", Message: "unable to persist receipt", Details: err.Error(), RequestID: requestID(r)}) + return + } + + res := models.DecideResponse{ + RequestID: req.RequestID, + TraceID: req.TraceID, + Decision: result.Decision, + ReceiptID: saved.ReceiptID, + PolicyVersion: s.policy.PolicyVersion, + PolicyID: s.policy.PolicyID, + MatchedRules: result.MatchedRules, + TransformPlan: result.TransformPlan, + Findings: findings, + Reason: result.Reason, + } + if req.IdempotencyKey != "" { + hash, _ := hashDecideRequest(req) + s.mu.Lock() + s.decisions[req.IdempotencyKey] = idempotentDecision{ + requestHash: hash, + response: res, + } + s.mu.Unlock() + } + s.respond(w, http.StatusOK, res) +} + +func (s *Server) handleTransform(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + s.respondError(w, http.StatusMethodNotAllowed, models.APIError{Code: "method_not_allowed", Message: "method must be POST", RequestID: requestID(r)}) + return + } + if !isJSONContentType(r.Header.Get("Content-Type")) { + s.respondError(w, http.StatusUnsupportedMediaType, models.APIError{Code: "unsupported_media_type", Message: "content-type must be application/json", RequestID: requestID(r)}) + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxRequestBodyBytes) + + var req models.TransformRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + s.respondErrorFromDecodeErr(w, r, err) + return + } + if req.Text == "" { + s.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "text is required", RequestID: requestID(r)}) + return + } + if req.Mode != "" { + req.Mode = models.TransformMode(strings.ToLower(strings.TrimSpace(string(req.Mode)))) + if !isAllowedTransformMode(req.Mode) { + s.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "unsupported transform mode", RequestID: requestID(r)}) + return + } + } + if len(req.EntityModes) > 0 { + canonicalModes := make(map[string]models.TransformMode, len(req.EntityModes)) + for entityType, mode := range req.EntityModes { + entityType = strings.TrimSpace(entityType) + if entityType == "" { + s.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "entity_modes keys must not be empty", RequestID: requestID(r)}) + return + } + canonicalMode := models.TransformMode(strings.ToLower(strings.TrimSpace(string(mode)))) + if !isAllowedTransformMode(canonicalMode) { + s.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "unsupported transform mode", RequestID: requestID(r)}) + return + } + canonicalModes[entityType] = canonicalMode + } + req.EntityModes = canonicalModes + } + if req.IdempotencyKey != "" { + reqHash, err := hashTransformRequest(req) + if err != nil { + s.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "unable to hash request payload", Details: err.Error(), RequestID: requestID(r)}) + return + } + s.mu.Lock() + existing, ok := s.transforms[req.IdempotencyKey] + s.mu.Unlock() + if ok { + if existing.requestHash != reqHash { + s.respondError(w, http.StatusConflict, models.APIError{Code: "idempotency_conflict", Message: "different request payload for same idempotency_key", RequestID: requestID(r)}) + return + } + s.respondRaw(w, existing.status, existing.body) + return + } + } + + findings := req.Findings + if len(findings) == 0 { + findings = scan.ScanText(req.Text, nil) + } + + entityModes := req.EntityModes + if len(entityModes) == 0 { + entityModes = map[string]models.TransformMode{} + if req.Mode != "" { + for _, f := range findings { + entityModes[f.EntityType] = req.Mode + } + } + } + plan := make([]models.TransformStep, 0, len(entityModes)) + for entityType, mode := range entityModes { + plan = append(plan, models.TransformStep{EntityType: entityType, Mode: mode}) + } + + output, stats := transform.ApplyTransforms(req.Text, findings, plan) + res := models.TransformResponse{ + RequestID: req.RequestID, + TraceID: req.TraceID, + Output: output, + PolicyID: s.policy.PolicyID, + PolicyVersion: s.policy.PolicyVersion, + Stats: stats, + } + if req.IdempotencyKey != "" { + body, err := json.Marshal(res) + if err != nil { + s.respondError(w, http.StatusInternalServerError, models.APIError{Code: "encode_error", Message: "unable to encode response", Details: err.Error(), RequestID: requestID(r)}) + return + } + hash, _ := hashTransformRequest(req) + s.mu.Lock() + s.transforms[req.IdempotencyKey] = idempotentCachedResponse{ + requestHash: hash, + body: body, + status: http.StatusOK, + } + s.mu.Unlock() + s.respondRaw(w, http.StatusOK, body) + return + } + s.respond(w, http.StatusOK, res) +} + +func (s *Server) handleAnonymize(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + s.respondError(w, http.StatusMethodNotAllowed, models.APIError{Code: "method_not_allowed", Message: "method must be POST", RequestID: requestID(r)}) + return + } + if !isJSONContentType(r.Header.Get("Content-Type")) { + s.respondError(w, http.StatusUnsupportedMediaType, models.APIError{Code: "unsupported_media_type", Message: "content-type must be application/json", RequestID: requestID(r)}) + return + } + r.Body = http.MaxBytesReader(w, r.Body, maxRequestBodyBytes) + + var req models.AnonymizeRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + s.respondErrorFromDecodeErr(w, r, err) + return + } + if req.Text == "" { + s.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "text is required", RequestID: requestID(r)}) + return + } + if req.IdempotencyKey != "" { + reqHash, err := hashAnonymizeRequest(req) + if err != nil { + s.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "unable to hash request payload", Details: err.Error(), RequestID: requestID(r)}) + return + } + s.mu.Lock() + existing, ok := s.anonymizes[req.IdempotencyKey] + s.mu.Unlock() + if ok { + if existing.requestHash != reqHash { + s.respondError(w, http.StatusConflict, models.APIError{Code: "idempotency_conflict", Message: "different request payload for same idempotency_key", RequestID: requestID(r)}) + return + } + s.respondRaw(w, existing.status, existing.body) + return + } + } + + findings := req.Findings + if len(findings) == 0 { + findings = scan.ScanText(req.Text, nil) + } + + plan := make([]models.TransformStep, 0) + seen := map[string]struct{}{} + for _, f := range findings { + if _, ok := seen[f.EntityType]; ok { + continue + } + seen[f.EntityType] = struct{}{} + plan = append(plan, models.TransformStep{EntityType: f.EntityType, Mode: models.TransformModeAnonymize}) + } + + output, stats := transform.ApplyTransforms(req.Text, findings, plan) + res := models.TransformResponse{ + RequestID: req.RequestID, + TraceID: req.TraceID, + Output: output, + PolicyID: s.policy.PolicyID, + PolicyVersion: s.policy.PolicyVersion, + Stats: stats, + } + if req.IdempotencyKey != "" { + body, err := json.Marshal(res) + if err != nil { + s.respondError(w, http.StatusInternalServerError, models.APIError{Code: "encode_error", Message: "unable to encode response", Details: err.Error(), RequestID: requestID(r)}) + return + } + hash, _ := hashAnonymizeRequest(req) + s.mu.Lock() + s.anonymizes[req.IdempotencyKey] = idempotentCachedResponse{ + requestHash: hash, + body: body, + status: http.StatusOK, + } + s.mu.Unlock() + s.respondRaw(w, http.StatusOK, body) + return + } + s.respond(w, http.StatusOK, res) +} + +func (s *Server) handleEvents(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + s.respondError(w, http.StatusMethodNotAllowed, models.APIError{Code: "method_not_allowed", Message: "method must be GET", RequestID: requestID(r)}) + return + } + if s.eventReader == nil { + s.respond(w, http.StatusOK, map[string]interface{}{"events": []shim.DecisionEvent{}, "total": 0}) + return + } + + q := shim.EventQuery{Limit: 100} + if after := r.URL.Query().Get("after"); after != "" { + if t, err := time.Parse(time.RFC3339, after); err == nil { + q.After = &t + } + } + if before := r.URL.Query().Get("before"); before != "" { + if t, err := time.Parse(time.RFC3339, before); err == nil { + q.Before = &t + } + } + if decision := r.URL.Query().Get("decision"); decision != "" { + q.Decision = decision + } + if adapter := r.URL.Query().Get("adapter"); adapter != "" { + q.Adapter = adapter + } + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + if n, err := strconv.Atoi(limitStr); err == nil && n > 0 && n <= 1000 { + q.Limit = n + } + } + + events, err := s.eventReader.Query(q) + if err != nil { + s.respondError(w, http.StatusInternalServerError, models.APIError{Code: "events_read_error", Message: err.Error(), RequestID: requestID(r)}) + return + } + if events == nil { + events = []shim.DecisionEvent{} + } + s.respond(w, http.StatusOK, map[string]interface{}{"events": events, "total": len(events)}) +} + +func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + s.respondError(w, http.StatusMethodNotAllowed, models.APIError{Code: "method_not_allowed", Message: "method must be GET", RequestID: requestID(r)}) + return + } + metrics := s.snapshotMetrics() + s.respond(w, http.StatusOK, metrics) +} + +func (s *Server) snapshotMetrics() metricsResponse { + s.statsMu.Lock() + defer s.statsMu.Unlock() + byStatus := map[string]int64{} + for status, count := range s.statusHits { + byStatus[strconv.Itoa(status)] = count + } + + byPath := map[string]int64{} + for path, count := range s.pathHits { + byPath[path] = count + } + + byMethod := map[string]int64{} + for method, count := range s.methodHits { + byMethod[method] = count + } + + return metricsResponse{ + TotalRequests: s.totalCount, + ErrorRequests: s.errorCount, + ByStatus: byStatus, + ByPath: byPath, + ByMethod: byMethod, + StartedAt: s.startedAt.Format(time.RFC3339), + UptimeSeconds: time.Since(s.startedAt).Seconds(), + } +} + +func (s *Server) handleReceipt(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + s.respondError(w, http.StatusMethodNotAllowed, models.APIError{Code: "method_not_allowed", Message: "method must be GET", RequestID: requestID(r)}) + return + } + + id := strings.TrimPrefix(r.URL.Path, "/v1/receipts/") + if id == "" || strings.Contains(id, "/") { + s.respondError(w, http.StatusNotFound, models.APIError{Code: "not_found", Message: "receipt id missing"}) + return + } + receipt, ok := s.store.Get(id) + if !ok { + s.respondError(w, http.StatusNotFound, models.APIError{Code: "not_found", Message: "receipt not found", RequestID: requestID(r)}) + return + } + s.respond(w, http.StatusOK, receipt) +} + +func (s *Server) respond(w http.ResponseWriter, status int, payload interface{}) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if err := json.NewEncoder(w).Encode(payload); err != nil { + s.logger.Printf("response encode failed: %v", err) + } +} + +func (s *Server) respondRaw(w http.ResponseWriter, status int, body []byte) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + if _, err := w.Write(body); err != nil { + s.logger.Printf("response write failed: %v", err) + } +} + +func isAllowedTransformMode(mode models.TransformMode) bool { + switch mode { + case models.TransformModeMask, models.TransformModeTokenize, models.TransformModeAnonymize, models.TransformModeRedact, models.TransformModeReplace, models.TransformModeHash: + return true + default: + return false + } +} + +func newRequestID() string { + id := make([]byte, 16) + if _, err := rand.Read(id); err != nil { + return fmt.Sprintf("rid-%d", time.Now().UnixNano()) + } + return hex.EncodeToString(id) +} + +func (s *Server) respondError(w http.ResponseWriter, status int, errResp models.APIError) { + if errResp.Code == "" { + errResp.Code = "error" + } + s.respond(w, status, map[string]models.APIError{"error": errResp}) +} + +func requestID(r *http.Request) string { + if rid, ok := r.Context().Value(requestIDContextKey{}).(string); ok && rid != "" { + return rid + } + if rid := r.Header.Get("x-request-id"); rid != "" { + return rid + } + return "" +} + +func hashDecideRequest(req models.DecideRequest) (string, error) { + req.IdempotencyKey = "" + req.RequestID = "" + req.TraceID = "" + req.SessionID = "" + req.ActorID = "" + req.TenantID = "" + body, err := json.Marshal(req) + if err != nil { + return "", err + } + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]), nil +} + +func hashDecideAction(action models.ActionMeta) (string, error) { + sum, err := hashPayload(action) + if err != nil { + return "", err + } + return sum, nil +} + +func hashDecideInput(req models.DecideRequest) (string, error) { + req.IdempotencyKey = "" + req.RequestID = "" + req.TraceID = "" + req.SessionID = "" + req.ActorID = "" + req.TenantID = "" + return hashPayload(req) +} + +func hashScanRequest(req models.ScanRequest) (string, error) { + req.IdempotencyKey = "" + req.RequestID = "" + req.TraceID = "" + body, err := json.Marshal(req) + if err != nil { + return "", err + } + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]), nil +} + +func hashPayload(value interface{}) (string, error) { + body, err := json.Marshal(value) + if err != nil { + return "", err + } + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]), nil +} + +func hashTransformRequest(req models.TransformRequest) (string, error) { + req.IdempotencyKey = "" + req.RequestID = "" + req.TraceID = "" + body, err := json.Marshal(req) + if err != nil { + return "", err + } + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]), nil +} + +func hashAnonymizeRequest(req models.AnonymizeRequest) (string, error) { + req.IdempotencyKey = "" + req.RequestID = "" + req.TraceID = "" + body, err := json.Marshal(req) + if err != nil { + return "", err + } + sum := sha256.Sum256(body) + return hex.EncodeToString(sum[:]), nil +} + +func isJSONContentType(value string) bool { + mediatype, _, err := mime.ParseMediaType(value) + if err != nil { + return false + } + return strings.EqualFold(strings.TrimSpace(mediatype), "application/json") +} + +func isRequestTooLarge(err error) bool { + var maxBytesErr *http.MaxBytesError + return errors.As(err, &maxBytesErr) +} + +func (s *Server) respondErrorFromDecodeErr(w http.ResponseWriter, r *http.Request, err error) { + if isRequestTooLarge(err) { + s.respondError(w, http.StatusRequestEntityTooLarge, models.APIError{Code: "request_too_large", Message: "request body exceeds limit", RequestID: requestID(r)}) + return + } + s.respondError(w, http.StatusBadRequest, models.APIError{Code: "invalid_request", Message: "invalid JSON body", Details: err.Error(), RequestID: requestID(r)}) +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go new file mode 100644 index 0000000..0609f61 --- /dev/null +++ b/internal/server/server_test.go @@ -0,0 +1,1003 @@ +package server + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/datafog/datafog-api/internal/models" + "github.com/datafog/datafog-api/internal/receipts" +) + +func testPolicy() models.Policy { + return models.Policy{ + PolicyID: "test", + PolicyVersion: "v1", + Rules: []models.Rule{ + {ID: "allow-read", Effect: models.DecisionAllow, Match: models.MatchCriteria{ActionTypes: []string{"file.read"}}, Priority: 10}, + {ID: "transform-write", Effect: models.DecisionTransform, Match: models.MatchCriteria{ActionTypes: []string{"file.write"}}, EntityRequirements: []string{"email"}, + EntityTransforms: []models.TransformStep{{EntityType: "email", Mode: models.TransformModeMask}}}, + {ID: "deny-shell", Effect: models.DecisionDeny, Match: models.MatchCriteria{ActionTypes: []string{"shell.exec"}}, EntityRequirements: []string{"api_key"}}, + }, + } +} + +func makeServer(t *testing.T) *http.Server { + return makeServerWithTokenAndRateLimit(t, "", 0) +} + +func makeServerWithToken(t *testing.T, apiToken string) *http.Server { + return makeServerWithTokenAndRateLimit(t, apiToken, 0) +} + +func makeServerWithTokenAndRateLimit(t *testing.T, apiToken string, rateLimitRPS int) *http.Server { + t.Helper() + store, err := receipts.NewReceiptStore(t.TempDir() + "/receipts.jsonl") + if err != nil { + t.Fatalf("new store: %v", err) + } + h := New(testPolicy(), store, nil, apiToken, rateLimitRPS) + return &http.Server{Handler: h.Handler()} +} + +func TestHealthEndpoint(t *testing.T) { + server := makeServer(t) + req := httptest.NewRequest(http.MethodGet, "/health", nil) + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + if ct := resp.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected Content-Type application/json, got %q", ct) + } + if resp.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.Code) + } + var got models.HealthResponse + if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + t.Fatalf("decode failed: %v", err) + } + if got.Status != "ok" { + t.Fatalf("expected status ok, got %q", got.Status) + } + if got.PolicyID != "test" { + t.Fatalf("expected policy id test, got %q", got.PolicyID) + } + if got.PolicyVersion != "v1" { + t.Fatalf("expected policy version v1, got %q", got.PolicyVersion) + } + if _, err := time.Parse(time.RFC3339, got.StartedAt); err != nil { + t.Fatalf("expected valid RFC3339 started_at, got %q", got.StartedAt) + } +} + +func TestTokenAuth(t *testing.T) { + server := makeServerWithToken(t, "token123") + + t.Run("passes_with_valid_bearer_token", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/health", nil) + req.Header.Set("Authorization", "Bearer token123") + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + if resp.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.Code) + } + }) + + t.Run("passes_with_valid_api_key_header", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/health", nil) + req.Header.Set("X-API-Key", "token123") + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + if resp.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.Code) + } + }) + + t.Run("missing_token_is_unauthorized", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/health", nil) + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + assertJSONError(t, resp, http.StatusUnauthorized, "unauthorized") + }) + + t.Run("invalid_token_is_unauthorized", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/health", nil) + req.Header.Set("X-API-Key", "bad-token") + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + assertJSONError(t, resp, http.StatusUnauthorized, "unauthorized") + }) +} + +func TestRateLimit(t *testing.T) { + server := makeServerWithTokenAndRateLimit(t, "", 2) + + req1 := httptest.NewRequest(http.MethodGet, "/health", nil) + resp1 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp1, req1) + if resp1.Code != http.StatusOK { + t.Fatalf("expected first request 200, got %d", resp1.Code) + } + + req2 := httptest.NewRequest(http.MethodGet, "/health", nil) + resp2 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp2, req2) + if resp2.Code != http.StatusOK { + t.Fatalf("expected second request 200, got %d", resp2.Code) + } + + req3 := httptest.NewRequest(http.MethodGet, "/health", nil) + resp3 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp3, req3) + assertJSONError(t, resp3, http.StatusTooManyRequests, "rate_limited") +} + +func TestPolicyVersionEndpoint(t *testing.T) { + server := makeServer(t) + req := httptest.NewRequest(http.MethodGet, "/v1/policy/version", nil) + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + if ct := resp.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected Content-Type application/json, got %q", ct) + } + if resp.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.Code) + } + + var got struct { + PolicyID string `json:"policy_id"` + PolicyVersion string `json:"policy_version"` + } + if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + t.Fatalf("decode failed: %v", err) + } + if got.PolicyID != "test" { + t.Fatalf("expected policy id test, got %q", got.PolicyID) + } + if got.PolicyVersion != "v1" { + t.Fatalf("expected policy version v1, got %q", got.PolicyVersion) + } +} + +func TestScanEndpoint(t *testing.T) { + server := makeServer(t) + body := bytes.NewBufferString(`{"text":"email jane@example.com"}`) + req := httptest.NewRequest(http.MethodPost, "/v1/scan", body) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + if ct := resp.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected Content-Type application/json, got %q", ct) + } + if resp.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.Code) + } + var scanned models.ScanResponse + if err := json.NewDecoder(resp.Body).Decode(&scanned); err != nil { + t.Fatalf("decode failed: %v", err) + } + if scanned.RequestID != "" { + t.Fatalf("expected empty request id, got %q", scanned.RequestID) + } + if scanned.PolicyID != "test" { + t.Fatalf("expected policy id test, got %q", scanned.PolicyID) + } + if scanned.PolicyVersion != "v1" { + t.Fatalf("expected policy version v1, got %q", scanned.PolicyVersion) + } + if len(scanned.Findings) != 1 { + t.Fatalf("expected one finding, got %d", len(scanned.Findings)) + } +} + +func TestTransformEndpoint(t *testing.T) { + server := makeServer(t) + body := bytes.NewBufferString(`{"text":"contact jane@example.com","mode":"mask"}`) + req := httptest.NewRequest(http.MethodPost, "/v1/transform", body) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + if ct := resp.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected Content-Type application/json, got %q", ct) + } + if resp.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.Code) + } + + var transformed models.TransformResponse + if err := json.NewDecoder(resp.Body).Decode(&transformed); err != nil { + t.Fatalf("decode failed: %v", err) + } + if transformed.PolicyID != "test" { + t.Fatalf("expected policy id test, got %q", transformed.PolicyID) + } + if transformed.PolicyVersion != "v1" { + t.Fatalf("expected policy version v1, got %q", transformed.PolicyVersion) + } + if transformed.Stats.EntitiesTransformed == 0 { + t.Fatalf("expected transformed entity count > 0") + } + if transformed.Stats.ModesApplied == "" { + t.Fatalf("expected modes applied") + } + if strings.Contains(transformed.Output, "jane@example.com") { + t.Fatalf("expected redacted output, got %q", transformed.Output) + } +} + +func TestAnonymizeEndpoint(t *testing.T) { + server := makeServer(t) + body := bytes.NewBufferString(`{"text":"contact jane@example.com","findings":[{"entity_type":"email","value":"jane@example.com","start":8,"end":23,"confidence":0.99}]}`) + req := httptest.NewRequest(http.MethodPost, "/v1/anonymize", body) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + if ct := resp.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected Content-Type application/json, got %q", ct) + } + if resp.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.Code) + } + + var anonymized models.TransformResponse + if err := json.NewDecoder(resp.Body).Decode(&anonymized); err != nil { + t.Fatalf("decode failed: %v", err) + } + if anonymized.PolicyID != "test" { + t.Fatalf("expected policy id test, got %q", anonymized.PolicyID) + } + if anonymized.Stats.EntitiesTransformed != 1 { + t.Fatalf("expected one transformed entity, got %d", anonymized.Stats.EntitiesTransformed) + } + if strings.Contains(anonymized.Output, "jane@example.com") { + t.Fatalf("expected anonymized output, got %q", anonymized.Output) + } +} + +func TestDecideAndReceiptFlow(t *testing.T) { + server := makeServer(t) + body := bytes.NewBufferString(`{"action":{"type":"file.write","resource":"notes.txt"},"text":"contact jane@example.com","request_id":"r1"}`) + req := httptest.NewRequest(http.MethodPost, "/v1/decide", body) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + if ct := resp.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected Content-Type application/json, got %q", ct) + } + if resp.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.Code) + } + var decided models.DecideResponse + if err := json.NewDecoder(resp.Body).Decode(&decided); err != nil { + t.Fatalf("decode failed: %v", err) + } + if decided.Decision != models.DecisionTransform { + t.Fatalf("expected transform, got %q", decided.Decision) + } + if decided.ReceiptID == "" { + t.Fatalf("expected receipt id") + } + + req2 := httptest.NewRequest(http.MethodGet, "/v1/receipts/"+decided.ReceiptID, nil) + resp2 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp2, req2) + if resp2.Code != http.StatusOK { + t.Fatalf("expected 200 receipt, got %d", resp2.Code) + } + var saved models.Receipt + if err := json.NewDecoder(resp2.Body).Decode(&saved); err != nil { + t.Fatalf("decode receipt failed: %v", err) + } + if saved.ReceiptID != decided.ReceiptID { + t.Fatalf("receipt id mismatch") + } + if saved.ActionHash == "" { + t.Fatalf("expected action hash") + } + if saved.InputHash == "" { + t.Fatalf("expected input hash") + } + if len(saved.ActionHash) != 64 { + t.Fatalf("expected action hash length 64, got %d", len(saved.ActionHash)) + } + if len(saved.InputHash) != 64 { + t.Fatalf("expected input hash length 64, got %d", len(saved.InputHash)) + } + if decided.Decision == models.DecisionTransform && saved.SanitizedSummary == "" { + t.Fatalf("expected sanitized summary for transform decision") + } +} + +func TestDecideTransformAndReceiptFlow(t *testing.T) { + server := makeServer(t) + decideBody := bytes.NewBufferString(`{"action":{"type":"file.write","resource":"notes.txt"},"text":"contact jane@example.com","request_id":"r1"}`) + decideReq := httptest.NewRequest(http.MethodPost, "/v1/decide", decideBody) + decideReq.Header.Set("Content-Type", "application/json") + decideResp := httptest.NewRecorder() + server.Handler.ServeHTTP(decideResp, decideReq) + if ct := decideResp.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected Content-Type application/json, got %q", ct) + } + if decideResp.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", decideResp.Code) + } + var decided models.DecideResponse + if err := json.NewDecoder(decideResp.Body).Decode(&decided); err != nil { + t.Fatalf("decode decide failed: %v", err) + } + if decided.Decision != models.DecisionTransform { + t.Fatalf("expected transform decision, got %q", decided.Decision) + } + + transformBody := bytes.NewBufferString(`{"text":"contact jane@example.com","mode":"mask","idempotency_key":"chain-1"}`) + transformReq := httptest.NewRequest(http.MethodPost, "/v1/transform", transformBody) + transformReq.Header.Set("Content-Type", "application/json") + transformResp := httptest.NewRecorder() + server.Handler.ServeHTTP(transformResp, transformReq) + if ct := transformResp.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected Content-Type application/json, got %q", ct) + } + if transformResp.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", transformResp.Code) + } + var transformed models.TransformResponse + if err := json.NewDecoder(transformResp.Body).Decode(&transformed); err != nil { + t.Fatalf("decode transformed failed: %v", err) + } + if strings.Contains(transformed.Output, "jane@example.com") { + t.Fatalf("expected masked output") + } + + receiptReq := httptest.NewRequest(http.MethodGet, "/v1/receipts/"+decided.ReceiptID, nil) + receiptResp := httptest.NewRecorder() + server.Handler.ServeHTTP(receiptResp, receiptReq) + if ct := receiptResp.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected Content-Type application/json, got %q", ct) + } + if receiptResp.Code != http.StatusOK { + t.Fatalf("expected 200 receipt, got %d", receiptResp.Code) + } + var receipt models.Receipt + if err := json.NewDecoder(receiptResp.Body).Decode(&receipt); err != nil { + t.Fatalf("decode receipt failed: %v", err) + } + if receipt.ReceiptID != decided.ReceiptID { + t.Fatalf("receipt id mismatch") + } + if receipt.Decision != decided.Decision { + t.Fatalf("expected receipt decision %q, got %q", decided.Decision, receipt.Decision) + } + if receipt.SanitizedSummary == "" { + t.Fatalf("expected sanitized summary") + } +} + +func TestHashDecideInputIgnoresRequestMetadata(t *testing.T) { + request1 := models.DecideRequest{ + RequestID: "r1", + TraceID: "trace-1", + TenantID: "tenant-1", + ActorID: "actor-1", + SessionID: "session-1", + Action: models.ActionMeta{ + Type: "file.write", + Resource: "notes.txt", + Args: []string{"--append"}, + }, + Text: "contact jane@example.com", + Findings: []models.ScanFinding{ + {EntityType: "email", Value: "jane@example.com", Start: 8, End: 23, Confidence: 0.99}, + }, + IdempotencyKey: "id1", + } + request2 := models.DecideRequest{ + RequestID: "r2", + TraceID: "trace-2", + TenantID: "tenant-2", + ActorID: "actor-2", + SessionID: "session-2", + Action: request1.Action, + Text: request1.Text, + Findings: request1.Findings, + } + if request1.Action.Type == "" || request2.Action.Type == "" { + t.Fatalf("setup failure") + } + + got1, err := hashDecideInput(request1) + if err != nil { + t.Fatalf("hashDecideInput failed: %v", err) + } + got2, err := hashDecideInput(request2) + if err != nil { + t.Fatalf("hashDecideInput failed: %v", err) + } + if got1 != got2 { + t.Fatalf("expected request metadata to be excluded, got %q and %q", got1, got2) + } + + request2.IdempotencyKey = "different-key" + request2.Action = models.ActionMeta{ + Type: "file.read", + Resource: "notes.txt", + Args: []string{"--append"}, + } + got3, err := hashDecideInput(request2) + if err != nil { + t.Fatalf("hashDecideInput failed: %v", err) + } + if got3 == got1 { + t.Fatalf("expected different input payloads to produce different input hash") + } +} + +func TestHashDecideActionStableForEquivalentAction(t *testing.T) { + action := models.ActionMeta{ + Type: "file.write", + Tool: "shell", + Resource: "notes.txt", + Args: []string{"--append", "--force"}, + Sensitive: true, + } + got1, err := hashDecideAction(action) + if err != nil { + t.Fatalf("hashDecideAction failed: %v", err) + } + got2, err := hashDecideAction(action) + if err != nil { + t.Fatalf("hashDecideAction failed: %v", err) + } + if got1 != got2 { + t.Fatalf("expected stable hashing, got %q and %q", got1, got2) + } + if len(got1) != 64 { + t.Fatalf("expected hash length 64, got %d", len(got1)) + } +} + +func TestDecideIdempotentReplay(t *testing.T) { + server := makeServer(t) + body1 := bytes.NewBufferString(`{"action":{"type":"file.write","resource":"notes.txt"},"text":"contact jane@example.com","request_id":"r1","idempotency_key":"idem-1"}`) + req1 := httptest.NewRequest(http.MethodPost, "/v1/decide", body1) + req1.Header.Set("Content-Type", "application/json") + resp1 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp1, req1) + if resp1.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp1.Code) + } + var first models.DecideResponse + if err := json.NewDecoder(resp1.Body).Decode(&first); err != nil { + t.Fatalf("decode failed: %v", err) + } + + body2 := bytes.NewBufferString(`{"action":{"type":"file.write","resource":"notes.txt"},"text":"contact jane@example.com","request_id":"r2","idempotency_key":"idem-1"}`) + req2 := httptest.NewRequest(http.MethodPost, "/v1/decide", body2) + req2.Header.Set("Content-Type", "application/json") + resp2 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp2, req2) + if resp2.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp2.Code) + } + var second models.DecideResponse + if err := json.NewDecoder(resp2.Body).Decode(&second); err != nil { + t.Fatalf("decode failed: %v", err) + } + + if first.ReceiptID == "" || second.ReceiptID == "" { + t.Fatalf("expected receipt ids") + } + if first.ReceiptID != second.ReceiptID { + t.Fatalf("expected same receipt for idempotent requests, got %q and %q", first.ReceiptID, second.ReceiptID) + } + if first.Decision != models.DecisionTransform || second.Decision != models.DecisionTransform { + t.Fatalf("expected transform decisions, got %q and %q", first.Decision, second.Decision) + } +} + +func TestDecideIdempotencyConflict(t *testing.T) { + server := makeServer(t) + body1 := bytes.NewBufferString(`{"action":{"type":"file.write","resource":"notes.txt"},"text":"contact jane@example.com","idempotency_key":"idem-conflict"}`) + req1 := httptest.NewRequest(http.MethodPost, "/v1/decide", body1) + req1.Header.Set("Content-Type", "application/json") + resp1 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp1, req1) + if resp1.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp1.Code) + } + + body2 := bytes.NewBufferString(`{"action":{"type":"file.write","resource":"notes.txt"},"text":"contact different@example.com","idempotency_key":"idem-conflict"}`) + req2 := httptest.NewRequest(http.MethodPost, "/v1/decide", body2) + req2.Header.Set("Content-Type", "application/json") + resp2 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp2, req2) + assertJSONError(t, resp2, http.StatusConflict, "idempotency_conflict") +} + +func TestScanIdempotentReplay(t *testing.T) { + server := makeServer(t) + body1 := bytes.NewBufferString(`{"text":"contact jane@example.com","idempotency_key":"scan-idem-1","request_id":"r1"}`) + req1 := httptest.NewRequest(http.MethodPost, "/v1/scan", body1) + req1.Header.Set("Content-Type", "application/json") + resp1 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp1, req1) + if resp1.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp1.Code) + } + var first models.ScanResponse + if err := json.NewDecoder(resp1.Body).Decode(&first); err != nil { + t.Fatalf("decode failed: %v", err) + } + + body2 := bytes.NewBufferString(`{"text":"contact jane@example.com","idempotency_key":"scan-idem-1","request_id":"r2"}`) + req2 := httptest.NewRequest(http.MethodPost, "/v1/scan", body2) + req2.Header.Set("Content-Type", "application/json") + resp2 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp2, req2) + if resp2.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp2.Code) + } + var second models.ScanResponse + if err := json.NewDecoder(resp2.Body).Decode(&second); err != nil { + t.Fatalf("decode failed: %v", err) + } + + if len(first.Findings) != len(second.Findings) || first.Findings[0].EntityType != second.Findings[0].EntityType { + t.Fatalf("expected identical findings") + } +} + +func TestScanIdempotencyConflict(t *testing.T) { + server := makeServer(t) + body1 := bytes.NewBufferString(`{"text":"contact jane@example.com","idempotency_key":"scan-idem-conflict"}`) + req1 := httptest.NewRequest(http.MethodPost, "/v1/scan", body1) + req1.Header.Set("Content-Type", "application/json") + resp1 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp1, req1) + if resp1.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp1.Code) + } + + body2 := bytes.NewBufferString(`{"text":"different text with no pii","idempotency_key":"scan-idem-conflict"}`) + req2 := httptest.NewRequest(http.MethodPost, "/v1/scan", body2) + req2.Header.Set("Content-Type", "application/json") + resp2 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp2, req2) + assertJSONError(t, resp2, http.StatusConflict, "idempotency_conflict") +} + +func TestTransformIdempotentReplay(t *testing.T) { + server := makeServer(t) + body1 := bytes.NewBufferString(`{"text":"contact jane@example.com","mode":"mask","idempotency_key":"transform-idem-1","request_id":"r1"}`) + req1 := httptest.NewRequest(http.MethodPost, "/v1/transform", body1) + req1.Header.Set("Content-Type", "application/json") + resp1 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp1, req1) + if resp1.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp1.Code) + } + var first models.TransformResponse + if err := json.NewDecoder(resp1.Body).Decode(&first); err != nil { + t.Fatalf("decode failed: %v", err) + } + + body2 := bytes.NewBufferString(`{"text":"contact jane@example.com","mode":"mask","idempotency_key":"transform-idem-1","request_id":"r2"}`) + req2 := httptest.NewRequest(http.MethodPost, "/v1/transform", body2) + req2.Header.Set("Content-Type", "application/json") + resp2 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp2, req2) + if resp2.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp2.Code) + } + var second models.TransformResponse + if err := json.NewDecoder(resp2.Body).Decode(&second); err != nil { + t.Fatalf("decode failed: %v", err) + } + + if first.Stats.EntitiesTransformed != second.Stats.EntitiesTransformed { + t.Fatalf("expected identical transformed entity count") + } + if first.Stats.ModesApplied != second.Stats.ModesApplied { + t.Fatalf("expected identical modes applied") + } +} + +func TestTransformIdempotencyConflict(t *testing.T) { + server := makeServer(t) + body1 := bytes.NewBufferString(`{"text":"contact jane@example.com","mode":"mask","idempotency_key":"transform-idem-conflict"}`) + req1 := httptest.NewRequest(http.MethodPost, "/v1/transform", body1) + req1.Header.Set("Content-Type", "application/json") + resp1 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp1, req1) + if resp1.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp1.Code) + } + + body2 := bytes.NewBufferString(`{"text":"different text","mode":"mask","idempotency_key":"transform-idem-conflict"}`) + req2 := httptest.NewRequest(http.MethodPost, "/v1/transform", body2) + req2.Header.Set("Content-Type", "application/json") + resp2 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp2, req2) + assertJSONError(t, resp2, http.StatusConflict, "idempotency_conflict") +} + +func TestAnonymizeIdempotentReplay(t *testing.T) { + server := makeServer(t) + body1 := bytes.NewBufferString(`{"text":"contact jane@example.com","idempotency_key":"anon-idem-1","request_id":"r1"}`) + req1 := httptest.NewRequest(http.MethodPost, "/v1/anonymize", body1) + req1.Header.Set("Content-Type", "application/json") + resp1 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp1, req1) + if resp1.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp1.Code) + } + var first models.TransformResponse + if err := json.NewDecoder(resp1.Body).Decode(&first); err != nil { + t.Fatalf("decode failed: %v", err) + } + + body2 := bytes.NewBufferString(`{"text":"contact jane@example.com","idempotency_key":"anon-idem-1","request_id":"r2"}`) + req2 := httptest.NewRequest(http.MethodPost, "/v1/anonymize", body2) + req2.Header.Set("Content-Type", "application/json") + resp2 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp2, req2) + if resp2.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp2.Code) + } + var second models.TransformResponse + if err := json.NewDecoder(resp2.Body).Decode(&second); err != nil { + t.Fatalf("decode failed: %v", err) + } + + if first.Stats.EntitiesTransformed != second.Stats.EntitiesTransformed { + t.Fatalf("expected identical transformed entity count") + } + if first.Output != second.Output { + t.Fatalf("expected identical anonymized output") + } +} + +func TestAnonymizeIdempotencyConflict(t *testing.T) { + server := makeServer(t) + body1 := bytes.NewBufferString(`{"text":"contact jane@example.com","idempotency_key":"anon-idem-conflict"}`) + req1 := httptest.NewRequest(http.MethodPost, "/v1/anonymize", body1) + req1.Header.Set("Content-Type", "application/json") + resp1 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp1, req1) + if resp1.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp1.Code) + } + + body2 := bytes.NewBufferString(`{"text":"another contact john@example.com","idempotency_key":"anon-idem-conflict"}`) + req2 := httptest.NewRequest(http.MethodPost, "/v1/anonymize", body2) + req2.Header.Set("Content-Type", "application/json") + resp2 := httptest.NewRecorder() + server.Handler.ServeHTTP(resp2, req2) + assertJSONError(t, resp2, http.StatusConflict, "idempotency_conflict") +} + +func TestValidateMethodAndBadInputs(t *testing.T) { + server := makeServer(t) + t.Run("method_not_allowed", func(t *testing.T) { + tests := []struct { + name string + method string + path string + wantStatus int + }{ + {name: "health", method: http.MethodPost, path: "/health", wantStatus: http.StatusMethodNotAllowed}, + {name: "policy_version", method: http.MethodPost, path: "/v1/policy/version", wantStatus: http.StatusMethodNotAllowed}, + {name: "scan", method: http.MethodGet, path: "/v1/scan", wantStatus: http.StatusMethodNotAllowed}, + {name: "decide", method: http.MethodGet, path: "/v1/decide", wantStatus: http.StatusMethodNotAllowed}, + {name: "transform", method: http.MethodGet, path: "/v1/transform", wantStatus: http.StatusMethodNotAllowed}, + {name: "anonymize", method: http.MethodGet, path: "/v1/anonymize", wantStatus: http.StatusMethodNotAllowed}, + {name: "receipts", method: http.MethodPost, path: "/v1/receipts/abc", wantStatus: http.StatusMethodNotAllowed}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + req := httptest.NewRequest(tc.method, tc.path, nil) + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + assertJSONError(t, resp, tc.wantStatus, "method_not_allowed") + }) + } + }) + + t.Run("bad_request_payloads", func(t *testing.T) { + scanReq := httptest.NewRequest(http.MethodPost, "/v1/scan", bytes.NewBufferString(`{"text":""}`)) + scanReq.Header.Set("Content-Type", "application/json") + scanResp := httptest.NewRecorder() + server.Handler.ServeHTTP(scanResp, scanReq) + assertJSONError(t, scanResp, http.StatusBadRequest, "invalid_request") + + decideReq := httptest.NewRequest(http.MethodPost, "/v1/decide", bytes.NewBufferString(`{"action":{"type":""},"text":"jane@example.com"}`)) + decideReq.Header.Set("Content-Type", "application/json") + decideResp := httptest.NewRecorder() + server.Handler.ServeHTTP(decideResp, decideReq) + assertJSONError(t, decideResp, http.StatusBadRequest, "invalid_request") + + transformReq := httptest.NewRequest(http.MethodPost, "/v1/transform", bytes.NewBufferString(`{"text":""}`)) + transformReq.Header.Set("Content-Type", "application/json") + transformResp := httptest.NewRecorder() + server.Handler.ServeHTTP(transformResp, transformReq) + assertJSONError(t, transformResp, http.StatusBadRequest, "invalid_request") + + anonymizeReq := httptest.NewRequest(http.MethodPost, "/v1/anonymize", bytes.NewBufferString(`{"text":""}`)) + anonymizeReq.Header.Set("Content-Type", "application/json") + anonymizeResp := httptest.NewRecorder() + server.Handler.ServeHTTP(anonymizeResp, anonymizeReq) + assertJSONError(t, anonymizeResp, http.StatusBadRequest, "invalid_request") + + invalidModeReq := httptest.NewRequest(http.MethodPost, "/v1/transform", bytes.NewBufferString(`{"text":"jane@example.com","mode":"unsupported-mode"}`)) + invalidModeReq.Header.Set("Content-Type", "application/json") + invalidModeResp := httptest.NewRecorder() + server.Handler.ServeHTTP(invalidModeResp, invalidModeReq) + assertJSONError(t, invalidModeResp, http.StatusBadRequest, "invalid_request") + + invalidEntityModeReq := httptest.NewRequest(http.MethodPost, "/v1/transform", bytes.NewBufferString(`{"text":"jane@example.com","entity_modes":{"email":"unsupported-mode"}}`)) + invalidEntityModeReq.Header.Set("Content-Type", "application/json") + invalidEntityModeResp := httptest.NewRecorder() + server.Handler.ServeHTTP(invalidEntityModeResp, invalidEntityModeReq) + assertJSONError(t, invalidEntityModeResp, http.StatusBadRequest, "invalid_request") + }) + + t.Run("invalid_content_type", func(t *testing.T) { + scanReq := httptest.NewRequest(http.MethodPost, "/v1/scan", bytes.NewBufferString(`{"text":"x"}`)) + scanResp := httptest.NewRecorder() + server.Handler.ServeHTTP(scanResp, scanReq) + assertJSONError(t, scanResp, http.StatusUnsupportedMediaType, "unsupported_media_type") + + decideReq := httptest.NewRequest(http.MethodPost, "/v1/decide", bytes.NewBufferString(`{"action":{"type":"file.read"},"text":"x"}`)) + decideReq.Header.Set("Content-Type", "text/plain") + decideResp := httptest.NewRecorder() + server.Handler.ServeHTTP(decideResp, decideReq) + assertJSONError(t, decideResp, http.StatusUnsupportedMediaType, "unsupported_media_type") + + transformReq := httptest.NewRequest(http.MethodPost, "/v1/transform", bytes.NewBufferString(`{"text":"x"}`)) + transformReq.Header.Set("Content-Type", "text/plain; charset=utf-8") + transformResp := httptest.NewRecorder() + server.Handler.ServeHTTP(transformResp, transformReq) + assertJSONError(t, transformResp, http.StatusUnsupportedMediaType, "unsupported_media_type") + + validCharsetReq := httptest.NewRequest(http.MethodPost, "/v1/anonymize", bytes.NewBufferString(`{"text":"jane@example.com"}`)) + validCharsetReq.Header.Set("Content-Type", "application/json; charset=utf-8") + validCharsetResp := httptest.NewRecorder() + server.Handler.ServeHTTP(validCharsetResp, validCharsetReq) + if validCharsetResp.Code != http.StatusOK { + t.Fatalf("expected 200 for valid json content type with charset, got %d", validCharsetResp.Code) + } + }) + + t.Run("request_too_large", func(t *testing.T) { + payload := `{"text":"` + strings.Repeat("x", int(maxRequestBodyBytes)+1) + `"}` + req := httptest.NewRequest(http.MethodPost, "/v1/scan", bytes.NewBufferString(payload)) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + assertJSONError(t, resp, http.StatusRequestEntityTooLarge, "request_too_large") + + decideReq := httptest.NewRequest(http.MethodPost, "/v1/decide", bytes.NewBufferString(`{"action":{"type":"file.read"},"text":"`+strings.Repeat("x", int(maxRequestBodyBytes)+1)+`"}`)) + decideReq.Header.Set("Content-Type", "application/json") + decideResp := httptest.NewRecorder() + server.Handler.ServeHTTP(decideResp, decideReq) + assertJSONError(t, decideResp, http.StatusRequestEntityTooLarge, "request_too_large") + + transformReq := httptest.NewRequest(http.MethodPost, "/v1/transform", bytes.NewBufferString(payload)) + transformReq.Header.Set("Content-Type", "application/json") + transformResp := httptest.NewRecorder() + server.Handler.ServeHTTP(transformResp, transformReq) + assertJSONError(t, transformResp, http.StatusRequestEntityTooLarge, "request_too_large") + + anonymizeReq := httptest.NewRequest(http.MethodPost, "/v1/anonymize", bytes.NewBufferString(payload)) + anonymizeReq.Header.Set("Content-Type", "application/json") + anonymizeResp := httptest.NewRecorder() + server.Handler.ServeHTTP(anonymizeResp, anonymizeReq) + assertJSONError(t, anonymizeResp, http.StatusRequestEntityTooLarge, "request_too_large") + }) + + t.Run("missing_receipt", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/v1/receipts/does-not-exist", nil) + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + assertJSONError(t, resp, http.StatusNotFound, "not_found") + + req = httptest.NewRequest(http.MethodGet, "/v1/receipts/", nil) + resp = httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + assertJSONError(t, resp, http.StatusNotFound, "not_found") + }) + + t.Run("not_found_routes", func(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/v1/does-not-exist", nil) + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + assertJSONError(t, resp, http.StatusNotFound, "not_found") + + req = httptest.NewRequest(http.MethodGet, "/completely/missing", nil) + resp = httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + assertJSONError(t, resp, http.StatusNotFound, "not_found") + }) +} + +func TestMetricsEndpoint(t *testing.T) { + server := makeServer(t) + + healthReq := httptest.NewRequest(http.MethodGet, "/health", nil) + healthResp := httptest.NewRecorder() + server.Handler.ServeHTTP(healthResp, healthReq) + if healthResp.Code != http.StatusOK { + t.Fatalf("expected 200 for health, got %d", healthResp.Code) + } + + scanReq := httptest.NewRequest(http.MethodPost, "/v1/scan", strings.NewReader(`{"text":"contact jane@example.com"}`)) + scanReq.Header.Set("Content-Type", "application/json") + scanResp := httptest.NewRecorder() + server.Handler.ServeHTTP(scanResp, scanReq) + if scanResp.Code != http.StatusOK { + t.Fatalf("expected 200 for scan, got %d", scanResp.Code) + } + + notFoundReq := httptest.NewRequest(http.MethodGet, "/v1/does-not-exist", nil) + notFoundResp := httptest.NewRecorder() + server.Handler.ServeHTTP(notFoundResp, notFoundReq) + if notFoundResp.Code != http.StatusNotFound { + t.Fatalf("expected 404 for unknown route, got %d", notFoundResp.Code) + } + + metricsReq := httptest.NewRequest(http.MethodGet, "/metrics", nil) + metricsResp := httptest.NewRecorder() + server.Handler.ServeHTTP(metricsResp, metricsReq) + if metricsResp.Code != http.StatusOK { + t.Fatalf("expected 200 for metrics, got %d", metricsResp.Code) + } + assertJSONContentType(t, metricsResp) + + var got metricsResponse + if err := json.NewDecoder(metricsResp.Body).Decode(&got); err != nil { + t.Fatalf("decode metrics failed: %v", err) + } + if got.TotalRequests != 3 { + t.Fatalf("expected 3 total requests, got %d", got.TotalRequests) + } + if got.ErrorRequests != 1 { + t.Fatalf("expected 1 error request, got %d", got.ErrorRequests) + } + if got.ByMethod["GET"] != 2 { + t.Fatalf("expected 2 GET requests, got %d", got.ByMethod["GET"]) + } + if got.ByMethod["POST"] != 1 { + t.Fatalf("expected 1 POST request, got %d", got.ByMethod["POST"]) + } + if got.ByStatus["200"] != 2 { + t.Fatalf("expected 2 status 200 requests, got %d", got.ByStatus["200"]) + } + if got.ByStatus["404"] != 1 { + t.Fatalf("expected 1 status 404 request, got %d", got.ByStatus["404"]) + } + if got.ByPath["/health"] != 1 { + t.Fatalf("expected /health to be tracked once, got %d", got.ByPath["/health"]) + } + if got.ByPath["/v1/scan"] != 1 { + t.Fatalf("expected /v1/scan to be tracked once, got %d", got.ByPath["/v1/scan"]) + } + if got.ByPath["/_not_found"] != 1 { + t.Fatalf("expected /_not_found to be tracked once, got %d", got.ByPath["/_not_found"]) + } + if _, err := time.Parse(time.RFC3339, got.StartedAt); err != nil { + t.Fatalf("expected started_at to be RFC3339, got %q", got.StartedAt) + } + if got.UptimeSeconds < 0 { + t.Fatalf("expected non-negative uptime, got %f", got.UptimeSeconds) + } +} + +func TestMetricsMethodNotAllowed(t *testing.T) { + server := makeServer(t) + req := httptest.NewRequest(http.MethodPost, "/metrics", nil) + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + assertJSONError(t, resp, http.StatusMethodNotAllowed, "method_not_allowed") +} + +func TestInvalidJSONHandling(t *testing.T) { + server := makeServer(t) + + req := httptest.NewRequest(http.MethodPost, "/v1/scan", bytes.NewBufferString(`{`)) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + assertJSONError(t, resp, http.StatusBadRequest, "invalid_request") + + req = httptest.NewRequest(http.MethodPost, "/v1/decide", bytes.NewBufferString(`{`)) + req.Header.Set("Content-Type", "application/json") + resp = httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + assertJSONError(t, resp, http.StatusBadRequest, "invalid_request") + + req = httptest.NewRequest(http.MethodPost, "/v1/transform", bytes.NewBufferString(`{`)) + req.Header.Set("Content-Type", "application/json") + resp = httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + assertJSONError(t, resp, http.StatusBadRequest, "invalid_request") + + req = httptest.NewRequest(http.MethodPost, "/v1/anonymize", bytes.NewBufferString(`{`)) + req.Header.Set("Content-Type", "application/json") + resp = httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + assertJSONError(t, resp, http.StatusBadRequest, "invalid_request") +} + +func TestDenyDecision(t *testing.T) { + server := makeServer(t) + body := bytes.NewBufferString(`{"action":{"type":"shell.exec","resource":"curl"},"text":"api_key=ABCD1234EFGH5678"}`) + req := httptest.NewRequest(http.MethodPost, "/v1/decide", body) + req.Header.Set("Content-Type", "application/json") + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + assertJSONContentType(t, resp) + if resp.Code != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.Code) + } + var decided models.DecideResponse + if err := json.NewDecoder(resp.Body).Decode(&decided); err != nil { + t.Fatalf("decode failed: %v", err) + } + if decided.Decision != models.DecisionDeny { + t.Fatalf("expected deny, got %q", decided.Decision) + } +} + +func assertJSONError(t *testing.T, resp *httptest.ResponseRecorder, status int, code string) models.APIError { + t.Helper() + if resp.Code != status { + t.Fatalf("expected %d, got %d", status, resp.Code) + } + assertJSONContentType(t, resp) + var got struct { + Error models.APIError `json:"error"` + } + if err := json.NewDecoder(resp.Body).Decode(&got); err != nil { + t.Fatalf("decode error body failed: %v", err) + } + if got.Error.Code != code { + t.Fatalf("expected error code %q, got %q", code, got.Error.Code) + } + return got.Error +} + +func TestErrorIncludesRequestIDHeader(t *testing.T) { + server := makeServer(t) + req := httptest.NewRequest(http.MethodPost, "/v1/scan", bytes.NewBufferString(`{"text":""}`)) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("x-request-id", "req-123") + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + err := assertJSONError(t, resp, http.StatusBadRequest, "invalid_request") + if err.RequestID != "req-123" { + t.Fatalf("expected request id header to be echoed, got %q", err.RequestID) + } + if got := resp.Header().Get("X-Request-ID"); got != "req-123" { + t.Fatalf("expected response request id header to be echoed, got %q", got) + } +} + +func TestRequestIDGeneratedWhenMissing(t *testing.T) { + server := makeServer(t) + req := httptest.NewRequest(http.MethodGet, "/health", nil) + resp := httptest.NewRecorder() + server.Handler.ServeHTTP(resp, req) + if got := resp.Header().Get("X-Request-ID"); got == "" { + t.Fatalf("expected generated request id header") + } +} + +func assertJSONContentType(t *testing.T, resp *httptest.ResponseRecorder) { + t.Helper() + if ct := resp.Header().Get("Content-Type"); ct != "application/json" { + t.Fatalf("expected Content-Type application/json, got %q", ct) + } +} diff --git a/internal/shim/enforcer.go b/internal/shim/enforcer.go new file mode 100644 index 0000000..17de779 --- /dev/null +++ b/internal/shim/enforcer.go @@ -0,0 +1,331 @@ +package shim + +import ( + "context" + "fmt" + "io/fs" + "os" + "os/exec" + "time" + + "github.com/datafog/datafog-api/internal/models" + "github.com/datafog/datafog-api/internal/scan" + "github.com/datafog/datafog-api/internal/transform" +) + +type CommandRunner interface { + Run(ctx context.Context, command string, args ...string) ([]byte, error) +} + +type FileReader interface { + ReadFile(path string) ([]byte, error) +} + +type FileWriter interface { + WriteFile(path string, data []byte, perm fs.FileMode) error +} + +type EnforcementMode string + +const ( + ModeEnforced EnforcementMode = "enforced" + ModeObserve EnforcementMode = "observe" +) + +type GateOption func(*Gate) + +func WithMode(mode EnforcementMode) GateOption { + return func(g *Gate) { + g.Mode = mode + } +} + +func WithEventSink(sink DecisionEventSink) GateOption { + return func(g *Gate) { + if sink == nil { + g.EventSink = noopEventSink{} + return + } + g.EventSink = sink + } +} + +type Gate struct { + Client DecisionClient + Runner CommandRunner + Reader FileReader + Writer FileWriter + Mode EnforcementMode + EventSink DecisionEventSink +} + +func NewGate(client DecisionClient, opts ...GateOption) *Gate { + g := &Gate{ + Client: client, + Runner: &osCommandRunner{}, + Reader: &osFileReader{}, + Writer: &osFileWriter{}, + Mode: ModeEnforced, + EventSink: noopEventSink{}, + } + for _, opt := range opts { + if opt != nil { + opt(g) + } + } + g.normalizeMode() + return g +} + +type osCommandRunner struct{} + +func (r *osCommandRunner) Run(ctx context.Context, command string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, command, args...) // #nosec G204 -- command execution is an explicit policy-gated feature. + return cmd.CombinedOutput() +} + +type osFileReader struct{} + +func (r *osFileReader) ReadFile(path string) ([]byte, error) { + return os.ReadFile(path) +} + +type osFileWriter struct{} + +func (r *osFileWriter) WriteFile(path string, data []byte, perm fs.FileMode) error { + return os.WriteFile(path, data, perm) +} + +type PolicyDecisionError struct { + Response models.DecideResponse +} + +func (e *PolicyDecisionError) Error() string { + return fmt.Sprintf("policy denied action decision=%s rules=%v reason=%v", e.Response.Decision, e.Response.MatchedRules, e.Response.Reason) +} + +func (r *Gate) normalizeMode() { + switch r.Mode { + case "", ModeEnforced, ModeObserve: + return + default: + r.Mode = ModeEnforced + } +} + +func (r *Gate) Check(ctx context.Context, req models.DecideRequest) (models.DecideResponse, error) { + if r.Client == nil { + return models.DecideResponse{}, fmt.Errorf("policy decision client is not configured") + } + return r.Client.Decide(ctx, req) +} + +func (r *Gate) permitDecision(decision models.Decision) bool { + switch decision { + case models.DecisionAllow, models.DecisionAllowWithRedaction: + return true + default: + return false + } +} + +func (r *Gate) shouldAllow(decision models.Decision) bool { + if r.Mode == "" { + r.normalizeMode() + } + if r.permitDecision(decision) { + return true + } + return r.Mode == ModeObserve +} + +func (r *Gate) executeRequest(ctx context.Context, req models.DecideRequest, run func(context.Context) ([]byte, error)) (models.DecideResponse, []byte, error) { + result, err := r.Check(ctx, req) + if err != nil { + if r.Mode == ModeEnforced { + r.recordDecisionEvent(req, result, false, err) + return result, nil, err + } + fallback := models.DecideResponse{ + Decision: models.DecisionAllow, + Reason: err.Error(), + RequestID: req.RequestID, + TraceID: req.TraceID, + } + r.recordDecisionEvent(req, fallback, true, err) + output, runErr := run(ctx) + return fallback, output, runErr + } + if !r.shouldAllow(result.Decision) { + r.recordDecisionEvent(req, result, false, nil) + return result, nil, &PolicyDecisionError{Response: result} + } + r.recordDecisionEvent(req, result, true, nil) + output, runErr := run(ctx) + return result, output, runErr +} + +func (r *Gate) readRequest(action models.ActionMeta, text string, findings []models.ScanFinding) models.DecideRequest { + return models.DecideRequest{ + Action: action, + Text: text, + Findings: findings, + RequestID: "", + TraceID: "", + } +} + +func (r *Gate) ExecuteShell(ctx context.Context, command string, args []string, text string, findings []models.ScanFinding, sensitive bool) (models.DecideResponse, []byte, error) { + if r.Runner == nil { + return models.DecideResponse{}, nil, fmt.Errorf("command runner is not configured") + } + + action := models.ActionMeta{ + Type: "shell.exec", + Tool: "shell", + Resource: command, + Command: command, + Args: append([]string(nil), args...), + Sensitive: sensitive, + } + return r.executeRequest(ctx, r.readRequest(action, text, findings), func(ctx context.Context) ([]byte, error) { + return r.Runner.Run(ctx, command, args...) + }) +} + +func (r *Gate) ReadFile(ctx context.Context, path string, text string, findings []models.ScanFinding, sensitive bool) (models.DecideResponse, []byte, error) { + if r.Reader == nil { + return models.DecideResponse{}, nil, fmt.Errorf("file reader is not configured") + } + action := models.ActionMeta{ + Type: "file.read", + Tool: "fs", + Resource: path, + Sensitive: sensitive, + } + result, output, err := r.executeRequest(ctx, r.readRequest(action, text, findings), func(ctx context.Context) ([]byte, error) { + return r.Reader.ReadFile(path) + }) + if err != nil { + return result, output, err + } + + // Apply redaction to read output when decision is allow_with_redaction + if result.Decision == models.DecisionAllowWithRedaction && len(result.TransformPlan) > 0 && output != nil { + output = r.applyRedaction(output, result.TransformPlan, nil) + } + + return result, output, nil +} + +func (r *Gate) WriteFile(ctx context.Context, path string, data []byte, perm fs.FileMode, text string, findings []models.ScanFinding, sensitive bool) (models.DecideResponse, error) { + if r.Writer == nil { + return models.DecideResponse{}, fmt.Errorf("file writer is not configured") + } + action := models.ActionMeta{ + Type: "file.write", + Tool: "fs", + Resource: path, + Sensitive: sensitive, + } + req := r.readRequest(action, text, findings) + result, err := r.Check(ctx, req) + if err != nil { + if r.Mode == ModeEnforced { + r.recordDecisionEvent(req, result, false, err) + return result, err + } + fallback := models.DecideResponse{ + Decision: models.DecisionAllow, + Reason: err.Error(), + RequestID: req.RequestID, + TraceID: req.TraceID, + } + r.recordDecisionEvent(req, fallback, true, err) + return fallback, r.Writer.WriteFile(path, data, perm) + } + if !r.shouldAllow(result.Decision) { + r.recordDecisionEvent(req, result, false, nil) + return result, &PolicyDecisionError{Response: result} + } + + // Apply transform plan on allow_with_redaction + writeData := data + if result.Decision == models.DecisionAllowWithRedaction && len(result.TransformPlan) > 0 { + writeData = r.applyRedaction(data, result.TransformPlan, findings) + } + + r.recordDecisionEvent(req, result, true, nil) + return result, r.Writer.WriteFile(path, writeData, perm) +} + +func (r *Gate) ExecuteCommand(ctx context.Context, adapterName string, target string, args []string, text string, findings []models.ScanFinding, sensitive bool) (models.DecideResponse, []byte, error) { + if r.Runner == nil { + return models.DecideResponse{}, nil, fmt.Errorf("command runner is not configured") + } + if adapterName == "" { + return models.DecideResponse{}, nil, fmt.Errorf("adapter name is required") + } + if target == "" { + return models.DecideResponse{}, nil, fmt.Errorf("target binary is required") + } + + action := models.ActionMeta{ + Type: "command.exec", + Tool: adapterName, + Resource: target, + Sensitive: sensitive, + } + if len(args) > 0 { + action.Command = args[0] + action.Args = append([]string(nil), args...) + } + return r.executeRequest(ctx, r.readRequest(action, text, findings), func(ctx context.Context) ([]byte, error) { + return r.Runner.Run(ctx, target, args...) + }) +} + +func (r *Gate) recordDecisionEvent(req models.DecideRequest, decision models.DecideResponse, allowed bool, checkErr error) { + if r.EventSink == nil { + r.EventSink = noopEventSink{} + } + r.EventSink.Record(DecisionEvent{ + Timestamp: time.Now().UTC(), + Mode: string(r.Mode), + ActionType: req.Action.Type, + Tool: req.Action.Tool, + Resource: req.Action.Resource, + Command: req.Action.Command, + Args: append([]string(nil), req.Action.Args...), + Sensitive: req.Action.Sensitive, + Decision: string(decision.Decision), + Allowed: allowed, + ReceiptID: decision.ReceiptID, + Matched: append([]string(nil), decision.MatchedRules...), + Reason: decision.Reason, + CheckError: errorString(checkErr), + RequestID: req.RequestID, + TraceID: req.TraceID, + }) +} + +// applyRedaction scans the content for PII and applies the transform plan. +// If findings are provided, they are used directly; otherwise the content is scanned. +func (r *Gate) applyRedaction(data []byte, plan []models.TransformStep, findings []models.ScanFinding) []byte { + text := string(data) + if len(findings) == 0 { + findings = scan.ScanText(text, nil) + } + if len(findings) == 0 { + return data + } + output, _ := transform.ApplyTransforms(text, findings, plan) + return []byte(output) +} + +func errorString(err error) string { + if err == nil { + return "" + } + return err.Error() +} diff --git a/internal/shim/enforcer_test.go b/internal/shim/enforcer_test.go new file mode 100644 index 0000000..1df77ae --- /dev/null +++ b/internal/shim/enforcer_test.go @@ -0,0 +1,371 @@ +package shim + +import ( + "context" + "errors" + "io/fs" + "strings" + "testing" + + "github.com/datafog/datafog-api/internal/models" +) + +type fakeDecisionClient struct { + response models.DecideResponse + err error + calls int + lastReq models.DecideRequest +} + +func (c *fakeDecisionClient) Decide(ctx context.Context, req models.DecideRequest) (models.DecideResponse, error) { + c.calls++ + c.lastReq = req + if c.err != nil { + return models.DecideResponse{}, c.err + } + return c.response, nil +} + +type fakeCommandRunner struct { + called bool + cmd string + args []string + out []byte + err error +} + +func (r *fakeCommandRunner) Run(ctx context.Context, command string, args ...string) ([]byte, error) { + r.called = true + r.cmd = command + r.args = append([]string{}, args...) + return r.out, r.err +} + +type fakeFileReader struct { + called bool + path string + data []byte + err error +} + +func (r *fakeFileReader) ReadFile(path string) ([]byte, error) { + r.called = true + r.path = path + return r.data, r.err +} + +type fakeFileWriter struct { + called bool + path string + data []byte + perm fs.FileMode + err error +} + +func (r *fakeFileWriter) WriteFile(path string, data []byte, perm fs.FileMode) error { + r.called = true + r.path = path + r.data = append([]byte{}, data...) + r.perm = perm + return r.err +} + +type fakeEventRecorder struct { + events []DecisionEvent +} + +func (r *fakeEventRecorder) Record(event DecisionEvent) { + r.events = append(r.events, event) +} + +func TestShellExecutionAllowed(t *testing.T) { + decision := models.DecideResponse{ + Decision: models.DecisionAllow, + ReceiptID: "r1", + MatchedRules: []string{"allow-shell"}, + } + decider := &fakeDecisionClient{response: decision} + runner := &fakeCommandRunner{out: []byte("ok\n")} + interceptor := &Gate{ + Client: decider, + Runner: runner, + } + + res, out, err := interceptor.ExecuteShell(context.Background(), "ls", []string{"-la"}, "file ls -la", []models.ScanFinding{}, true) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if string(out) != "ok\n" { + t.Fatalf("expected command output, got %q", string(out)) + } + if !runner.called { + t.Fatalf("expected command runner to be called") + } + if decider.calls != 1 { + t.Fatalf("expected one policy call, got %d", decider.calls) + } + if decider.lastReq.Action.Type != "shell.exec" { + t.Fatalf("expected shell action type, got %q", decider.lastReq.Action.Type) + } + if decider.lastReq.Action.Command != "ls" { + t.Fatalf("expected command ls, got %q", decider.lastReq.Action.Command) + } + if decider.lastReq.Action.Args[0] != "-la" { + t.Fatalf("expected first arg -la, got %q", decider.lastReq.Action.Args[0]) + } + if runner.cmd != "ls" || runner.args[0] != "-la" { + t.Fatalf("expected shell invocation, got %q %v", runner.cmd, runner.args) + } + if res.ReceiptID != "r1" { + t.Fatalf("expected receipt id, got %q", res.ReceiptID) + } + if !decider.lastReq.Action.Sensitive { + t.Fatalf("expected sensitive field to be preserved") + } +} + +func TestShellExecutionDenied(t *testing.T) { + decider := &fakeDecisionClient{ + response: models.DecideResponse{ + Decision: models.DecisionDeny, + Reason: "blocked command", + MatchedRules: []string{"deny-shell"}, + }, + } + runner := &fakeCommandRunner{out: []byte("ok")} + interceptor := &Gate{ + Client: decider, + Runner: runner, + } + + _, _, err := interceptor.ExecuteShell(context.Background(), "rm", []string{"-rf", "/tmp"}, "", nil, false) + if err == nil { + t.Fatalf("expected denied action error") + } + var denied *PolicyDecisionError + if !errors.As(err, &denied) { + t.Fatalf("expected PolicyDecisionError, got %T", err) + } + if runner.called { + t.Fatalf("expected command runner to be skipped on deny") + } + if len(denied.Response.MatchedRules) != 1 || denied.Response.MatchedRules[0] != "deny-shell" { + t.Fatalf("expected denied rule reason, got %+v", denied.Response.MatchedRules) + } +} + +func TestShellExecutionAllowsInObserveMode(t *testing.T) { + decider := &fakeDecisionClient{ + response: models.DecideResponse{ + Decision: models.DecisionDeny, + ReceiptID: "r2", + MatchedRules: []string{"deny-shell"}, + }, + } + runner := &fakeCommandRunner{out: []byte("ok\n")} + recorder := &fakeEventRecorder{} + interceptor := NewGate(decider, WithMode(ModeObserve), WithEventSink(recorder)) + interceptor.Runner = runner + + res, out, err := interceptor.ExecuteShell(context.Background(), "rm", []string{"-rf", "/tmp"}, "", []models.ScanFinding{}, false) + if err != nil { + t.Fatalf("expected no error in observe mode, got %v", err) + } + if string(out) != "ok\n" { + t.Fatalf("expected command output, got %q", string(out)) + } + if res.Decision != models.DecisionDeny { + t.Fatalf("expected deny decision for observability, got %q", res.Decision) + } + if !runner.called { + t.Fatalf("expected runner to execute in observe mode") + } + if len(recorder.events) != 1 { + t.Fatalf("expected one event, got %d", len(recorder.events)) + } + if recorder.events[0].Mode != string(ModeObserve) { + t.Fatalf("expected observe event mode, got %q", recorder.events[0].Mode) + } +} + +func TestShellExecutionPolicyErrorPassesInObserveMode(t *testing.T) { + decider := &fakeDecisionClient{ + err: errors.New("policy unavailable"), + } + runner := &fakeCommandRunner{out: []byte("ok\n")} + recorder := &fakeEventRecorder{} + interceptor := NewGate(decider, WithMode(ModeObserve), WithEventSink(recorder)) + interceptor.Runner = runner + + _, out, err := interceptor.ExecuteShell(context.Background(), "ls", nil, "", nil, false) + if err != nil { + t.Fatalf("expected no error when API is unreachable in observe mode, got %v", err) + } + if string(out) != "ok\n" { + t.Fatalf("expected command output, got %q", string(out)) + } + if len(recorder.events) != 1 || recorder.events[0].CheckError == "" { + t.Fatalf("expected policy error in event, got %#v", recorder.events) + } +} + +func TestCommandAdapterExecution(t *testing.T) { + decider := &fakeDecisionClient{ + response: models.DecideResponse{ + Decision: models.DecisionAllow, + }, + } + runner := &fakeCommandRunner{out: []byte("run\n")} + interceptor := &Gate{ + Client: decider, + Runner: runner, + } + + res, out, err := interceptor.ExecuteCommand(context.Background(), "git", "/usr/bin/git", []string{"status"}, "", nil, false) + if err != nil { + t.Fatalf("expected allow, got %v", err) + } + if string(out) != "run\n" { + t.Fatalf("expected command output, got %q", string(out)) + } + if !runner.called { + t.Fatalf("expected command runner") + } + if runner.cmd != "/usr/bin/git" { + t.Fatalf("expected target binary, got %q", runner.cmd) + } + if decider.lastReq.Action.Type != "command.exec" { + t.Fatalf("expected command.exec action, got %q", decider.lastReq.Action.Type) + } + if decider.lastReq.Action.Tool != "git" { + t.Fatalf("expected tool git, got %q", decider.lastReq.Action.Tool) + } + if decider.lastReq.Action.Command != "status" { + t.Fatalf("expected command to be first arg, got %q", decider.lastReq.Action.Command) + } + if res.ReceiptID != "" { + t.Fatalf("did not expect receipt id in mocked response") + } +} + +func TestReadFileAllowed(t *testing.T) { + decider := &fakeDecisionClient{ + response: models.DecideResponse{ + Decision: models.DecisionAllow, + }, + } + reader := &fakeFileReader{data: []byte("payload")} + interceptor := &Gate{ + Client: decider, + Reader: reader, + } + + _, data, err := interceptor.ReadFile(context.Background(), "/tmp/a.txt", "", []models.ScanFinding{{EntityType: "email", Value: "x@y.z", Start: 0, End: 5, Confidence: 0.9}}, false) + if err != nil { + t.Fatalf("expected allow, got %v", err) + } + if string(data) != "payload" { + t.Fatalf("expected payload, got %q", string(data)) + } + if reader.path != "/tmp/a.txt" { + t.Fatalf("expected reader path, got %q", reader.path) + } + if decider.lastReq.Action.Type != "file.read" { + t.Fatalf("expected file.read action, got %q", decider.lastReq.Action.Type) + } +} + +func TestWriteFileAllowed(t *testing.T) { + decider := &fakeDecisionClient{ + response: models.DecideResponse{ + Decision: models.DecisionAllow, + }, + } + writer := &fakeFileWriter{} + interceptor := &Gate{ + Client: decider, + Writer: writer, + } + + _, err := interceptor.WriteFile(context.Background(), "/tmp/a.txt", []byte("payload"), 0o600, "note", []models.ScanFinding{}, true) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if writer.path != "/tmp/a.txt" { + t.Fatalf("expected write path %q, got %q", "/tmp/a.txt", writer.path) + } + if string(writer.data) != "payload" { + t.Fatalf("expected payload write, got %q", string(writer.data)) + } + if writer.perm != 0o600 { + t.Fatalf("expected perm 600, got %v", writer.perm) + } +} + +func TestWriteFileRedactsOnAllowWithRedaction(t *testing.T) { + decider := &fakeDecisionClient{ + response: models.DecideResponse{ + Decision: models.DecisionAllowWithRedaction, + TransformPlan: []models.TransformStep{ + {EntityType: "email", Mode: models.TransformModeRedact}, + }, + }, + } + writer := &fakeFileWriter{} + interceptor := &Gate{ + Client: decider, + Writer: writer, + } + + data := []byte("contact alice@example.com for info") + findings := []models.ScanFinding{ + {EntityType: "email", Value: "alice@example.com", Start: 8, End: 25, Confidence: 0.99}, + } + + res, err := interceptor.WriteFile(context.Background(), "/tmp/out.txt", data, 0o600, string(data), findings, true) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Decision != models.DecisionAllowWithRedaction { + t.Fatalf("expected allow_with_redaction, got %q", res.Decision) + } + // The written data should have the email redacted + if strings.Contains(string(writer.data), "alice@example.com") { + t.Fatalf("expected email to be redacted in written data, got %q", string(writer.data)) + } + if !strings.Contains(string(writer.data), "[REDACTED]") { + t.Fatalf("expected [REDACTED] in written data, got %q", string(writer.data)) + } +} + +func TestReadFileRedactsOnAllowWithRedaction(t *testing.T) { + fileContent := "user email is bob@example.com" + decider := &fakeDecisionClient{ + response: models.DecideResponse{ + Decision: models.DecisionAllowWithRedaction, + TransformPlan: []models.TransformStep{ + {EntityType: "email", Mode: models.TransformModeRedact}, + }, + }, + } + reader := &fakeFileReader{data: []byte(fileContent)} + interceptor := &Gate{ + Client: decider, + Reader: reader, + } + + res, output, err := interceptor.ReadFile(context.Background(), "/tmp/data.txt", "", nil, false) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if res.Decision != models.DecisionAllowWithRedaction { + t.Fatalf("expected allow_with_redaction, got %q", res.Decision) + } + // The output should have the email redacted + if strings.Contains(string(output), "bob@example.com") { + t.Fatalf("expected email to be redacted in output, got %q", string(output)) + } + if !strings.Contains(string(output), "[REDACTED]") { + t.Fatalf("expected [REDACTED] in output, got %q", string(output)) + } +} diff --git a/internal/shim/events.go b/internal/shim/events.go new file mode 100644 index 0000000..3d921d3 --- /dev/null +++ b/internal/shim/events.go @@ -0,0 +1,142 @@ +package shim + +import ( + "bufio" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "time" +) + +type DecisionEvent struct { + Timestamp time.Time `json:"timestamp"` + Mode string `json:"mode"` + ActionType string `json:"action_type"` + Tool string `json:"tool"` + Resource string `json:"resource"` + Command string `json:"command"` + Args []string `json:"args"` + Sensitive bool `json:"sensitive"` + Decision string `json:"decision"` + Allowed bool `json:"allowed"` + ReceiptID string `json:"receipt_id,omitempty"` + Matched []string `json:"matched_rules,omitempty"` + Reason string `json:"reason,omitempty"` + CheckError string `json:"check_error,omitempty"` + RequestID string `json:"request_id,omitempty"` + TraceID string `json:"trace_id,omitempty"` +} + +type DecisionEventSink interface { + Record(event DecisionEvent) +} + +// EventQuery allows filtering events by time range, decision type, and adapter. +type EventQuery struct { + After *time.Time + Before *time.Time + Decision string + Adapter string + Limit int +} + +// EventReader reads stored events with optional filtering. +type EventReader interface { + Query(q EventQuery) ([]DecisionEvent, error) +} + +type noopEventSink struct{} + +func (s noopEventSink) Record(_ DecisionEvent) {} + +type NDJSONDecisionEventSink struct { + path string + mu sync.Mutex +} + +func NewNDJSONDecisionEventSink(path string) *NDJSONDecisionEventSink { + return &NDJSONDecisionEventSink{path: path} +} + +func (s *NDJSONDecisionEventSink) Record(event DecisionEvent) { + if s == nil || s.path == "" { + return + } + + if err := os.MkdirAll(filepath.Dir(s.path), 0o750); err != nil { + return + } + + payload, err := json.Marshal(event) + if err != nil { + return + } + + s.mu.Lock() + defer s.mu.Unlock() + + file, err := os.OpenFile(s.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600) + if err != nil { + return + } + defer file.Close() + + _, _ = fmt.Fprintln(file, string(payload)) +} + +// Query reads events from the NDJSON file and applies filters. +func (s *NDJSONDecisionEventSink) Query(q EventQuery) ([]DecisionEvent, error) { + if s == nil || s.path == "" { + return nil, nil + } + + s.mu.Lock() + defer s.mu.Unlock() + + f, err := os.Open(s.path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + defer f.Close() + + var events []DecisionEvent + scanner := bufio.NewScanner(f) + scanner.Buffer(make([]byte, 64*1024), 1024*1024) + + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if line == "" { + continue + } + var event DecisionEvent + if err := json.Unmarshal([]byte(line), &event); err != nil { + continue + } + + if q.After != nil && event.Timestamp.Before(*q.After) { + continue + } + if q.Before != nil && event.Timestamp.After(*q.Before) { + continue + } + if q.Decision != "" && !strings.EqualFold(event.Decision, q.Decision) { + continue + } + if q.Adapter != "" && !strings.EqualFold(event.Tool, q.Adapter) { + continue + } + + events = append(events, event) + if q.Limit > 0 && len(events) >= q.Limit { + break + } + } + + return events, scanner.Err() +} diff --git a/internal/shim/http.go b/internal/shim/http.go new file mode 100644 index 0000000..3e2525c --- /dev/null +++ b/internal/shim/http.go @@ -0,0 +1,118 @@ +package shim + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "time" + + "github.com/datafog/datafog-api/internal/models" +) + +type DecisionClient interface { + Decide(ctx context.Context, req models.DecideRequest) (models.DecideResponse, error) +} + +type APIError struct { + StatusCode int + Code string + Message string + Details string +} + +func (e APIError) Error() string { + return fmt.Sprintf("datafog policy API error: status=%d code=%s message=%s", e.StatusCode, e.Code, e.Message) +} + +type HTTPDecisionClient struct { + DecisionEndpoint string + APIKey string + HTTPClient *http.Client +} + +func NewHTTPDecisionClient(baseURL, apiKey string) *HTTPDecisionClient { + return &HTTPDecisionClient{ + DecisionEndpoint: buildDecideEndpoint(baseURL), + APIKey: strings.TrimSpace(apiKey), + HTTPClient: &http.Client{ + Timeout: 5 * time.Second, + }, + } +} + +func (c *HTTPDecisionClient) Decide(ctx context.Context, req models.DecideRequest) (models.DecideResponse, error) { + var out models.DecideResponse + if c.HTTPClient == nil { + return out, fmt.Errorf("http client is not configured") + } + if strings.TrimSpace(c.DecisionEndpoint) == "" { + return out, fmt.Errorf("decision endpoint is required") + } + + body, err := json.Marshal(req) + if err != nil { + return out, fmt.Errorf("marshal decide request: %w", err) + } + + httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, c.DecisionEndpoint, bytes.NewReader(body)) + if err != nil { + return out, fmt.Errorf("create decide request: %w", err) + } + httpReq.Header.Set("Content-Type", "application/json") + if c.APIKey != "" { + httpReq.Header.Set("X-API-Key", c.APIKey) + } + + resp, err := c.HTTPClient.Do(httpReq) + if err != nil { + return out, fmt.Errorf("call decide API: %w", err) + } + defer resp.Body.Close() + + respBody, err := io.ReadAll(resp.Body) + if err != nil { + return out, fmt.Errorf("read decide response: %w", err) + } + if resp.StatusCode != http.StatusOK { + apiErr := APIError{ + StatusCode: resp.StatusCode, + } + var parsed struct { + Error models.APIError `json:"error"` + } + if err := json.Unmarshal(respBody, &parsed); err == nil { + apiErr.Code = parsed.Error.Code + apiErr.Message = parsed.Error.Message + apiErr.Details = parsed.Error.Details + } else { + apiErr.Message = strings.TrimSpace(string(respBody)) + if apiErr.Message == "" { + apiErr.Message = "policy service error" + } + } + return out, apiErr + } + + if err := json.Unmarshal(respBody, &out); err != nil { + return out, fmt.Errorf("unmarshal decide response: %w", err) + } + return out, nil +} + +func buildDecideEndpoint(baseURL string) string { + baseURL = strings.TrimSpace(baseURL) + if baseURL == "" { + baseURL = "http://localhost:8080" + } + if strings.HasSuffix(baseURL, "/v1/decide") { + return baseURL + } + if strings.HasSuffix(baseURL, "/v1") { + return baseURL + "/decide" + } + return strings.TrimRight(baseURL, "/") + "/v1/decide" +} diff --git a/internal/shim/http_test.go b/internal/shim/http_test.go new file mode 100644 index 0000000..49ae300 --- /dev/null +++ b/internal/shim/http_test.go @@ -0,0 +1,88 @@ +package shim + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + "github.com/datafog/datafog-api/internal/models" +) + +func TestBuildDecideEndpoint(t *testing.T) { + tests := []struct { + name string + base string + expected string + }{ + { + name: "root path", + base: "http://localhost:8080", + expected: "http://localhost:8080/v1/decide", + }, + { + name: "v1 path", + base: "http://localhost:8080/v1", + expected: "http://localhost:8080/v1/decide", + }, + { + name: "full decide path", + base: "http://localhost:8080/v1/decide", + expected: "http://localhost:8080/v1/decide", + }, + } + for _, tc := range tests { + got := buildDecideEndpoint(tc.base) + if got != tc.expected { + t.Fatalf("expected %s, got %s", tc.expected, got) + } + } +} + +func TestHTTPDecisionClientPostsToDecide(t *testing.T) { + var gotAction models.ActionMeta + var gotToken string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + t.Fatalf("expected POST, got %s", r.Method) + } + if r.URL.Path != "/v1/decide" { + t.Fatalf("expected path /v1/decide, got %s", r.URL.Path) + } + gotToken = r.Header.Get("X-API-Key") + var req models.DecideRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + t.Fatalf("decode request failed: %v", err) + } + gotAction = req.Action + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(models.DecideResponse{ + Decision: models.DecisionAllow, + ReceiptID: "r1", + MatchedRules: []string{"allow-test"}, + }) + })) + defer server.Close() + + client := NewHTTPDecisionClient(server.URL, "token-1") + got, err := client.Decide(context.Background(), models.DecideRequest{ + Action: models.ActionMeta{ + Type: "file.read", + Resource: "notes.txt", + Sensitive: true, + }, + }) + if err != nil { + t.Fatalf("decide failed: %v", err) + } + if got.Decision != models.DecisionAllow { + t.Fatalf("expected allow, got %s", got.Decision) + } + if gotToken != "token-1" { + t.Fatalf("expected api token header, got %q", gotToken) + } + if gotAction.Type != "file.read" { + t.Fatalf("expected action type file.read, got %q", gotAction.Type) + } +} diff --git a/internal/transform/transform.go b/internal/transform/transform.go new file mode 100644 index 0000000..a54b0d4 --- /dev/null +++ b/internal/transform/transform.go @@ -0,0 +1,108 @@ +package transform + +import ( + "crypto/sha256" + "encoding/hex" + "fmt" + "sort" + "strings" + + "github.com/datafog/datafog-api/internal/models" +) + +func ApplyTransforms(input string, findings []models.ScanFinding, steps []models.TransformStep) (string, models.TransformStats) { + if len(findings) == 0 { + return input, models.TransformStats{} + } + + modeMap := map[string]models.TransformMode{} + for _, step := range steps { + modeMap[step.EntityType] = step.Mode + } + + filtered := make([]models.ScanFinding, 0, len(findings)) + for _, finding := range findings { + if _, ok := modeMap[finding.EntityType]; !ok { + continue + } + filtered = append(filtered, finding) + } + + if len(filtered) == 0 { + return input, models.TransformStats{} + } + + sort.SliceStable(filtered, func(i, j int) bool { + if filtered[i].Start == filtered[j].Start { + return filtered[i].End > filtered[j].End + } + return filtered[i].Start > filtered[j].Start + }) + + appliedModes := map[string]bool{} + bytes := []byte(input) + count := 0 + for _, finding := range filtered { + if finding.Start < 0 || finding.End > len(bytes) || finding.Start >= finding.End { + continue + } + mode := modeMap[finding.EntityType] + replacement := ReplacementForModeWithType(mode, finding.EntityType, finding.Value) + before := bytes[:finding.Start] + after := bytes[finding.End:] + bytes = append(before, append([]byte(replacement), after...)...) + appliedModes[fmt.Sprintf("%s:%s", finding.EntityType, modeMap[finding.EntityType])] = true + count++ + } + + parts := make([]string, 0, len(appliedModes)) + for k := range appliedModes { + parts = append(parts, k) + } + sort.Strings(parts) + + return string(bytes), models.TransformStats{ + EntitiesTransformed: count, + ModesApplied: strings.Join(parts, ","), + } +} + +func replacementForMode(mode models.TransformMode, value string) string { + switch mode { + case models.TransformModeTokenize: + return "TOK-" + deterministicPrefix(value) + case models.TransformModeAnonymize: + return "anon-" + deterministicPrefix(value) + case models.TransformModeRedact: + return "[REDACTED]" + case models.TransformModeMask: + return strings.Repeat("*", len(value)) + case models.TransformModeReplace: + return "[REPLACED]" + case models.TransformModeHash: + return deterministicHash(value) + default: + return "[FILTERED]" + } +} + +// ReplacementForModeWithType returns a pseudonymized replacement +// with entity-type context (e.g., "[PERSON_A1B2C3]"). +func ReplacementForModeWithType(mode models.TransformMode, entityType string, value string) string { + if mode == models.TransformModeReplace { + prefix := strings.ToUpper(entityType) + return "[" + prefix + "_" + deterministicPrefix(value) + "]" + } + return replacementForMode(mode, value) +} + +func deterministicPrefix(value string) string { + hash := sha256.Sum256([]byte(value)) + encoded := hex.EncodeToString(hash[:]) + return encoded[:8] +} + +func deterministicHash(value string) string { + hash := sha256.Sum256([]byte(value)) + return hex.EncodeToString(hash[:]) +} diff --git a/internal/transform/transform_test.go b/internal/transform/transform_test.go new file mode 100644 index 0000000..df3b572 --- /dev/null +++ b/internal/transform/transform_test.go @@ -0,0 +1,134 @@ +package transform + +import ( + "strings" + "testing" + + "github.com/datafog/datafog-api/internal/models" +) + +func TestApplyTransformsMasksAndTokenizes(t *testing.T) { + input := "email=alice@example.com token=1234123412341234" + findings := []models.ScanFinding{ + {EntityType: "email", Start: 6, End: 22, Value: "alice@example.com"}, + {EntityType: "credit_card", Start: 29, End: 45, Value: "1234123412341234"}, + } + steps := []models.TransformStep{ + {EntityType: "email", Mode: models.TransformModeMask}, + {EntityType: "credit_card", Mode: models.TransformModeTokenize}, + } + out, stats := ApplyTransforms(input, findings, steps) + if out == input { + t.Fatalf("expected transformed output") + } + if stats.EntitiesTransformed != 2 { + t.Fatalf("expected 2 transformed entities, got %d", stats.EntitiesTransformed) + } +} + +func TestTransformIgnoresEntitiesWithoutPlan(t *testing.T) { + input := "hello 555-000-1111" + findings := []models.ScanFinding{{EntityType: "phone", Start: 6, End: 17, Value: "555-000-1111"}} + out, stats := ApplyTransforms(input, findings, []models.TransformStep{{EntityType: "email", Mode: models.TransformModeMask}}) + if out != input { + t.Fatalf("expected unchanged output, got %q", out) + } + if stats.EntitiesTransformed != 0 { + t.Fatalf("expected no transformations") + } +} + +func TestTransformModeReplace(t *testing.T) { + input := "email alice@example.com" + findings := []models.ScanFinding{ + {EntityType: "email", Start: 6, End: 22, Value: "alice@example.com"}, + } + steps := []models.TransformStep{ + {EntityType: "email", Mode: models.TransformModeReplace}, + } + out, stats := ApplyTransforms(input, findings, steps) + if !strings.Contains(out, "[EMAIL_") { + t.Fatalf("expected pseudonymized replacement with [EMAIL_...], got %q", out) + } + if stats.EntitiesTransformed != 1 { + t.Fatalf("expected 1 transformed entity, got %d", stats.EntitiesTransformed) + } + + // Deterministic: same input produces same output + out2, _ := ApplyTransforms(input, findings, steps) + if out != out2 { + t.Fatalf("expected deterministic output, got %q and %q", out, out2) + } +} + +func TestTransformModeHash(t *testing.T) { + input := "ssn 123-45-6789" + findings := []models.ScanFinding{ + {EntityType: "ssn", Start: 4, End: 15, Value: "123-45-6789"}, + } + steps := []models.TransformStep{ + {EntityType: "ssn", Mode: models.TransformModeHash}, + } + out, stats := ApplyTransforms(input, findings, steps) + if strings.Contains(out, "123-45-6789") { + t.Fatalf("expected SSN to be replaced with hash, got %q", out) + } + // SHA256 hash is 64 hex chars + replaced := strings.TrimPrefix(out, "ssn ") + if len(replaced) != 64 { + t.Fatalf("expected 64-char SHA256 hash, got %d chars: %q", len(replaced), replaced) + } + if stats.EntitiesTransformed != 1 { + t.Fatalf("expected 1 transformed entity, got %d", stats.EntitiesTransformed) + } +} + +func TestTransformModeRedact(t *testing.T) { + input := "key api_key=Secret12345678901234" + findings := []models.ScanFinding{ + {EntityType: "api_key", Start: 4, End: 31, Value: "api_key=Secret12345678901234"}, + } + steps := []models.TransformStep{ + {EntityType: "api_key", Mode: models.TransformModeRedact}, + } + out, _ := ApplyTransforms(input, findings, steps) + if !strings.Contains(out, "[REDACTED]") { + t.Fatalf("expected [REDACTED] in output, got %q", out) + } +} + +func TestAllSixModes(t *testing.T) { + modes := []struct { + mode models.TransformMode + contains string + }{ + {models.TransformModeMask, "****"}, + {models.TransformModeTokenize, "TOK-"}, + {models.TransformModeAnonymize, "anon-"}, + {models.TransformModeRedact, "[REDACTED]"}, + {models.TransformModeReplace, "[EMAIL_"}, + {models.TransformModeHash, ""}, // just check it's 64 hex chars + } + + for _, tt := range modes { + t.Run(string(tt.mode), func(t *testing.T) { + input := "test alice@example.com end" + findings := []models.ScanFinding{ + {EntityType: "email", Start: 5, End: 22, Value: "alice@example.com"}, + } + steps := []models.TransformStep{ + {EntityType: "email", Mode: tt.mode}, + } + out, stats := ApplyTransforms(input, findings, steps) + if stats.EntitiesTransformed != 1 { + t.Fatalf("expected 1 entity transformed") + } + if tt.contains != "" && !strings.Contains(out, tt.contains) { + t.Fatalf("expected output to contain %q, got %q", tt.contains, out) + } + if strings.Contains(out, "alice@example.com") { + t.Fatalf("original value should not be present in output") + } + }) + } +} diff --git a/scripts/ci/he-docs-config.json b/scripts/ci/he-docs-config.json new file mode 100644 index 0000000..e6f5356 --- /dev/null +++ b/scripts/ci/he-docs-config.json @@ -0,0 +1,123 @@ +{ + "required_docs": [ + "AGENTS.md", + "docs/PLANS.md", + "docs/DOMAIN_DOCS.md" + ], + "expected_runbooks": [ + "docs/runbooks/update-agents-md.md", + "docs/runbooks/update-domain-docs.md", + "docs/runbooks/code-review.md", + "docs/runbooks/review-findings.md", + "docs/runbooks/address-review-findings.md", + "docs/runbooks/validate-current-state.md", + "docs/runbooks/reproduce-bug.md", + "docs/runbooks/pull-request.md", + "docs/runbooks/respond-to-feedback.md", + "docs/runbooks/verify-release.md", + "docs/runbooks/record-evidence.md", + "docs/runbooks/ci-failures.md", + "docs/runbooks/merge-change.md" + ], + "domain_docs": [ + "docs/DESIGN.md", + "docs/DATA.md", + "docs/FRONTEND.md", + "docs/PRODUCT_SENSE.md", + "docs/RELIABILITY.md", + "docs/SECURITY.md", + "docs/OBSERVABILITY.md", + "docs/design-docs/core-beliefs.md" + ], + "required_headings": { + "docs/SECURITY.md": [ + "## Threat Model", + "## Auth Model", + "## Data Sensitivity", + "## Compliance", + "## Controls" + ], + "docs/RELIABILITY.md": [ + "## Reliability Goals", + "## Failure Modes", + "## Monitoring", + "## Operational Guardrails" + ], + "docs/FRONTEND.md": [ + "## Stack", + "## Conventions", + "## Component Architecture", + "## Performance", + "## Accessibility" + ], + "docs/DESIGN.md": [ + "## Design Principles", + "## Visual Direction", + "## Interaction Standards" + ], + "docs/PRODUCT_SENSE.md": [ + "## Target Users", + "## Key Outcomes", + "## Decision Heuristics", + "## Quality Criteria" + ], + "docs/DATA.md": [ + "## Data Model", + "## Migrations", + "## Backfills And Data Fixes", + "## Integrity And Consistency", + "## Sensitive Data Notes" + ], + "docs/OBSERVABILITY.md": [ + "## Logging Strategy", + "## Metrics", + "## Traces", + "## Health Checks", + "## Agent Access" + ] + }, + "artifact_placeholder_patterns": [ + "", + "" + ], + "lint_completed_plans": true, + "required_spec_frontmatter_keys": [ + "slug", + "status", + "date", + "owner", + "plan_mode", + "spike_recommended", + "priority" + ], + "required_plan_frontmatter_keys": [ + "slug", + "status", + "phase", + "plan_mode", + "priority", + "owner" + ], + "required_spike_frontmatter_keys": [ + "slug", + "status", + "date", + "owner", + "timebox" + ], + "drift_rules": [ + { + "regex": "(^auth/|/auth/|^middleware/|/middleware/|(^|/)security/|(^|/)permissions/)", + "doc": "docs/SECURITY.md" + }, + { + "regex": "(^infra/|^ops/|^deploy/|^terraform/|^k8s/|^helm/|(^|/)monitoring/|(^|/)alerts/)", + "doc": "docs/RELIABILITY.md" + }, + { + "regex": "(^package\\\\.json$|^pnpm-lock\\\\.yaml$|^yarn\\\\.lock$|^bun\\\\.lockb$|^tsconfig\\\\.json$|^vite\\\\.config\\\\.|^next\\\\.config\\\\.)", + "doc": "docs/FRONTEND.md" + } + ] +} diff --git a/scripts/ci/he-docs-drift.sh b/scripts/ci/he-docs-drift.sh new file mode 100755 index 0000000..0c1965c --- /dev/null +++ b/scripts/ci/he-docs-drift.sh @@ -0,0 +1,112 @@ +#!/bin/bash +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +DEFAULT_CONFIG_PATH="scripts/ci/he-docs-config.json" + +config_path="${HARNESS_DOCS_CONFIG:-$DEFAULT_CONFIG_PATH}" +config_file="${REPO_ROOT}/${config_path}" + +if [[ ! -f "$config_file" ]]; then + echo "Error: he-docs-drift missing/invalid config: Missing config '${config_path}'. Fix: create it (bootstrap should do this) or set HARNESS_DOCS_CONFIG." >&2 + exit 2 +fi + +cfg="$(cat "$config_file")" +if ! echo "$cfg" | jq -e 'type == "object"' >/dev/null 2>&1; then + echo "Error: he-docs-drift missing/invalid config: Config must be a JSON object." >&2 + exit 2 +fi + +base_ref="${GITHUB_BASE_REF:-}" +head_ref="${GITHUB_HEAD_REF:-}" + +if [[ -n "$base_ref" ]]; then + diff_range="origin/${base_ref}...HEAD" +else + if git -C "$REPO_ROOT" rev-parse -q --verify HEAD~1 >/dev/null 2>&1; then + diff_range="HEAD~1...HEAD" + else + diff_range="" + fi +fi + +echo "he-docs-drift: starting" >&2 +echo "Repro: bash scripts/ci/he-docs-drift.sh" >&2 +if [[ -n "$base_ref" ]]; then + echo "PR context: base_ref='${base_ref}' head_ref='${head_ref}' diff='${diff_range}'" >&2 +else + echo "Local context: diff='${diff_range}'" >&2 +fi + +if [[ -n "$diff_range" ]]; then + changed="$(git -C "$REPO_ROOT" diff --name-only "$diff_range" 2>/dev/null || true)" +else + changed="$(git -C "$REPO_ROOT" diff-tree --no-commit-id --name-only -r HEAD 2>/dev/null || true)" +fi + +# Trim empty lines +changed="$(echo "$changed" | sed '/^[[:space:]]*$/d')" + +if [[ -z "$changed" ]]; then + echo "he-docs-drift: no changes detected" + exit 0 +fi + +# Build list of changed docs (files starting with docs/) +changed_docs="$(echo "$changed" | grep '^docs/' || true)" + +# Extract drift_rules array; default to empty array if missing or wrong type +drift_rules="$(echo "$cfg" | jq -c '.drift_rules // [] | if type == "array" then . else [] end')" +rule_count="$(echo "$drift_rules" | jq 'length')" + +missing=0 + +for ((i = 0; i < rule_count; i++)); do + rule="$(echo "$drift_rules" | jq -c ".[$i]")" + + # Skip non-object entries + if ! echo "$rule" | jq -e 'type == "object"' >/dev/null 2>&1; then + continue + fi + + regex="$(echo "$rule" | jq -r '.regex // empty')" + doc="$(echo "$rule" | jq -r '.doc // empty')" + + if [[ -z "$regex" || -z "$doc" ]]; then + continue + fi + + # Validate regex by testing it + if ! echo "" | grep -qE "$regex" 2>/dev/null && [[ $? -eq 2 ]]; then + echo "Error: invalid drift rule regex: ${regex}" >&2 + missing=1 + continue + fi + + # Find changed files matching the regex + matching="$(echo "$changed" | grep -E "$regex" || true)" + + if [[ -z "$matching" ]]; then + continue + fi + + # Check if the required doc is in the changed docs list + if ! echo "$changed_docs" | grep -qxF "$doc" 2>/dev/null; then + sample="$(echo "$matching" | head -n 10 | sed 's/^/- /')" + echo "::error file=${doc},title=Docs drift gate::Missing required doc update '${doc}' when files match /${regex}/ (see job logs for matching files)." + echo "Missing doc update: '${doc}' should change when files match /${regex}/." >&2 + echo "Matching files (up to 10):" >&2 + echo "$sample" >&2 + echo "Fix: update '${doc}' in this PR, or edit drift_rules in '${DEFAULT_CONFIG_PATH}' (or HARNESS_DOCS_CONFIG) if this mapping is wrong." >&2 + missing=1 + fi +done + +if [[ "$missing" -ne 0 ]]; then + echo "Error: docs drift gate failed (see missing doc updates above)" >&2 + exit 1 +fi + +echo "he-docs-drift: OK" +exit 0 diff --git a/scripts/ci/he-docs-lint.sh b/scripts/ci/he-docs-lint.sh new file mode 100755 index 0000000..114449b --- /dev/null +++ b/scripts/ci/he-docs-lint.sh @@ -0,0 +1,234 @@ +#!/bin/bash +set -euo pipefail + +# ── Constants ──────────────────────────────────────────────────────────────── +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +DEFAULT_CONFIG_PATH="scripts/ci/he-docs-config.json" + +# ── Globals ────────────────────────────────────────────────────────────────── +ERRORS=0 +WARNINGS=0 + +# ── Helpers ────────────────────────────────────────────────────────────────── + +_env_flag() { + local name="$1" + local default="${2:-0}" + local val="${!name:-$default}" + [[ "$val" == "1" ]] +} + +_load_config() { + local config_path="${HARNESS_DOCS_CONFIG:-$DEFAULT_CONFIG_PATH}" + local path="$REPO_ROOT/$config_path" + if [[ ! -f "$path" ]]; then + echo "Error: he-docs-lint missing/invalid config: Missing config '$config_path'. Fix: create it (bootstrap should do this) or set HARNESS_DOCS_CONFIG." >&2 + return 1 + fi + # Validate it is a JSON object + if ! jq -e 'type == "object"' "$path" >/dev/null 2>&1; then + echo "Error: he-docs-lint missing/invalid config: Config must be a JSON object." >&2 + return 1 + fi + cat "$path" +} + +_gh_annotate() { + local level="$1" file="$2" title="$3" msg="$4" + if [[ -n "$file" ]]; then + echo "::${level} file=${file},title=${title}::${msg}" + else + echo "::${level} title=${title}::${msg}" + fi +} + +_emit() { + local level="$1" file="$2" title="$3" msg="$4" + _gh_annotate "$level" "$file" "$title" "$msg" + local upper + upper="$(echo "$level" | tr '[:lower:]' '[:upper:]')" + echo "${upper}: ${msg}" >&2 + if [[ "$level" == "error" ]]; then + ERRORS=$((ERRORS + 1)) + else + WARNINGS=$((WARNINGS + 1)) + fi +} + +_has_exact_line() { + local path="$1" needle="$2" + grep -Fxq "$needle" "$path" 2>/dev/null +} + +# ── Checks ─────────────────────────────────────────────────────────────────── + +_check_required_docs() { + local cfg="$1" + local count + count="$(echo "$cfg" | jq -r '.required_docs | if type == "array" then length else 0 end')" + if [[ "$count" -eq 0 ]]; then + return + fi + local i doc + for ((i = 0; i < count; i++)); do + doc="$(echo "$cfg" | jq -r ".required_docs[$i]")" + if [[ "$doc" == "null" ]] || [[ -z "$doc" ]]; then + continue + fi + if [[ ! -e "$REPO_ROOT/$doc" ]]; then + _emit "error" "$doc" "Required doc missing" \ + "Missing required doc: '$doc'. Fix: create it (run he-bootstrap if this repo is not bootstrapped) or adjust required_docs in config." + fi + done +} + +_check_domain_doc_headings() { + local cfg="$1" + local is_obj + is_obj="$(echo "$cfg" | jq -r '.required_headings | type')" + if [[ "$is_obj" != "object" ]]; then + return + fi + + local docs + docs="$(echo "$cfg" | jq -r '.required_headings | keys[]')" + if [[ -z "$docs" ]]; then + return + fi + + local doc + while IFS= read -r doc; do + [[ -z "$doc" ]] && continue + local path="$REPO_ROOT/$doc" + if [[ ! -f "$path" ]]; then + continue # on-demand domain docs + fi + + local headings_count + headings_count="$(echo "$cfg" | jq -r --arg d "$doc" '.required_headings[$d] | if type == "array" then length else 0 end')" + if [[ "$headings_count" -eq 0 ]]; then + _emit "error" "$doc" "Missing config headings" \ + "No required headings configured for '$doc'. Fix: add required_headings['$doc'] in config or remove the entry." + continue + fi + + local missing=() + local j heading + for ((j = 0; j < headings_count; j++)); do + heading="$(echo "$cfg" | jq -r --arg d "$doc" ".required_headings[\$d][$j]")" + if [[ "$heading" == "null" ]] || [[ -z "$heading" ]]; then + continue + fi + if ! _has_exact_line "$path" "$heading"; then + missing+=("$heading") + fi + done + + if [[ ${#missing[@]} -gt 0 ]]; then + local joined + joined="$(printf "%s; " "${missing[@]}")" + joined="${joined%; }" # trim trailing "; " + _emit "error" "$doc" "Missing headings" \ + "Missing required headings in '$doc': ${joined}. Fix: add them." + fi + done <<< "$docs" +} + +_check_seed_markers() { + local cfg="$1" + local fail_level="warning" + if _env_flag "HARNESS_FAIL_ON_SEED_MARKERS" "0"; then + fail_level="error" + fi + + local count + count="$(echo "$cfg" | jq -r '.domain_docs | if type == "array" then length else 0 end')" + if [[ "$count" -eq 0 ]]; then + return + fi + + local i doc path + for ((i = 0; i < count; i++)); do + doc="$(echo "$cfg" | jq -r ".domain_docs[$i]")" + if [[ "$doc" == "null" ]] || [[ -z "$doc" ]]; then + continue + fi + path="$REPO_ROOT/$doc" + if [[ ! -f "$path" ]]; then + continue + fi + if grep -q ' blocks once this repo has real domain context." + fi + done +} + +_check_generated_last_updated() { + local gen_dir="$REPO_ROOT/docs/generated" + if [[ ! -d "$gen_dir" ]]; then + return + fi + + local fail_level="warning" + if _env_flag "HARNESS_FAIL_ON_GENERATED_PLACEHOLDERS" "0"; then + fail_level="error" + fi + + local path rel + for path in "$gen_dir"/*.md; do + [[ -e "$path" ]] || continue # handle no-match glob + rel="${path#"$REPO_ROOT/"}" + # Skip known non-generated docs + if [[ "$rel" == "docs/generated/README.md" ]] || [[ "$rel" == "docs/generated/memory.md" ]]; then + continue + fi + local text + text="$(cat "$path")" + + # Check for missing last_updated line (BSD/GNU portable; avoid grep -P) + if ! echo "$text" | grep -Eq '^[[:space:]]*-[[:space:]]*last_updated:[[:space:]]*'; then + _emit "error" "$rel" "Missing last_updated" \ + "Generated doc '$rel' must include a 'last_updated' line. Fix: add e.g. '- last_updated: 2026-02-15 12:34'." + fi + + # Check for placeholder last_updated value + if echo "$text" | grep -Eq 'last_updated:[[:space:]]*&2 + exit 1 + fi + + echo "he-docs-lint: OK ($WARNINGS warning(s))" + exit 0 +} + +main "$@" diff --git a/scripts/ci/he-plans-lint.sh b/scripts/ci/he-plans-lint.sh new file mode 100755 index 0000000..0da6b89 --- /dev/null +++ b/scripts/ci/he-plans-lint.sh @@ -0,0 +1,354 @@ +#!/bin/bash +set -euo pipefail + +# ── Repo root (two levels above this script) ────────────────────────── +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +DEFAULT_CONFIG_PATH="scripts/ci/he-docs-config.json" + +# ── Default required headings ───────────────────────────────────────── +DEFAULT_REQUIRED_HEADINGS=( + "## Purpose / Big Picture" + "## Progress" + "## Surprises & Discoveries" + "## Decision Log" + "## Outcomes & Retrospective" + "## Context and Orientation" + "## Milestones" + "## Plan of Work" + "## Concrete Steps" + "## Validation and Acceptance" + "## Idempotence and Recovery" + "## Artifacts and Notes" + "## Interfaces and Dependencies" + "## Pull Request" + "## Review Findings" + "## Verify/Release Decision" + "## Revision Notes" +) + +# ── Counters ────────────────────────────────────────────────────────── +ERRORS=0 +WARNINGS=0 + +# ── Emit a finding (GitHub annotation + stderr) ────────────────────── +emit() { + local level="$1" file="$2" title="$3" msg="$4" + if [[ -n "$file" ]]; then + echo "::${level} file=${file},title=${title}::${msg}" + else + echo "::${level} title=${title}::${msg}" + fi + echo "${level^^}: ${msg}" >&2 + if [[ "$level" == "error" ]]; then + (( ERRORS++ )) || true + else + (( WARNINGS++ )) || true + fi +} + +# ── Load config ─────────────────────────────────────────────────────── +load_config() { + local config_rel="${HARNESS_DOCS_CONFIG:-$DEFAULT_CONFIG_PATH}" + local config_path="$REPO_ROOT/$config_rel" + if [[ ! -f "$config_path" ]]; then + echo "Error: he-plans-lint missing/invalid config: Missing config '${config_rel}'. Fix: create it (bootstrap should do this) or set HARNESS_DOCS_CONFIG." >&2 + exit 2 + fi + # Validate it is a JSON object + if ! jq -e 'type == "object"' "$config_path" >/dev/null 2>&1; then + echo "Error: he-plans-lint missing/invalid config: Config must be a JSON object." >&2 + exit 2 + fi + CONFIG_PATH="$config_path" +} + +# ── Config helpers ──────────────────────────────────────────────────── +cfg_get() { + # $1 = jq expression, returns raw output + jq -r "$1" "$CONFIG_PATH" +} + +cfg_get_array() { + # $1 = jq path to array, outputs one element per line + jq -r "$1 // [] | if type == \"array\" then .[] else empty end" "$CONFIG_PATH" 2>/dev/null +} + +# ── Extract YAML frontmatter (between first --- and second ---) ────── +# Sets FRONTMATTER variable. Returns 1 if no frontmatter found. +extract_frontmatter() { + local file="$1" + FRONTMATTER="" + local first_line + first_line="$(head -1 "$file")" + if [[ "$(echo "$first_line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" != "---" ]]; then + return 1 + fi + # Find closing --- (skip line 1, find next ---) + local end_line + end_line="$(awk 'NR > 1 && /^[[:space:]]*---[[:space:]]*$/ { print NR; exit }' "$file")" + if [[ -z "$end_line" ]]; then + return 1 + fi + # Extract lines between line 2 and end_line-1 + FRONTMATTER="$(sed -n "2,$((end_line - 1))p" "$file")" + return 0 +} + +# ── Parse frontmatter key-value pairs ──────────────────────────────── +# Reads FRONTMATTER, outputs "key=value" lines +frontmatter_keys=() +frontmatter_vals=() + +parse_frontmatter_kv() { + frontmatter_keys=() + frontmatter_vals=() + while IFS= read -r raw; do + local line + line="$(echo "$raw" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + # Skip empty / comment lines + [[ -z "$line" || "$line" == \#* ]] && continue + # Must contain a colon + [[ "$line" != *:* ]] && continue + local key val + key="$(echo "$line" | cut -d: -f1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + val="$(echo "$line" | cut -d: -f2- | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + frontmatter_keys+=("$key") + frontmatter_vals+=("$val") + done <<< "$FRONTMATTER" +} + +# ── Lookup a frontmatter value by key ──────────────────────────────── +fm_get() { + local needle="$1" + for i in "${!frontmatter_keys[@]}"; do + if [[ "${frontmatter_keys[$i]}" == "$needle" ]]; then + echo "${frontmatter_vals[$i]}" + return 0 + fi + done + return 1 +} + +fm_has_key() { + local needle="$1" + for k in "${frontmatter_keys[@]}"; do + [[ "$k" == "$needle" ]] && return 0 + done + return 1 +} + +# ── Extract section lines (between heading and next ## heading) ────── +# Outputs section body lines to stdout +section_lines() { + local file="$1" heading="$2" + awk -v h="$heading" ' + BEGIN { found=0 } + $0 == h { found=1; next } + found && /^## / { exit } + found { print } + ' "$file" +} + +# ── Check: exact heading line exists in file ───────────────────────── +has_exact_line() { + local file="$1" needle="$2" + grep -qxF "$needle" "$file" +} + +# ── Check: Progress section ────────────────────────────────────────── +check_progress() { + local file_rel="$1" file_abs="$2" + local body + body="$(section_lines "$file_abs" "## Progress")" + + # Check non-empty (has at least one non-blank line) + if ! echo "$body" | grep -q '[^[:space:]]'; then + emit "error" "$file_rel" "Missing Progress content" \ + "Plan '${file_rel}' has an empty ## Progress section." + return + fi + + # Check timestamped checkbox pattern + if ! echo "$body" | grep -qE '^- \[[ xX]\] \([0-9]{4}-[0-9]{2}-[0-9]{2}[^)]*\) P[0-9]+'; then + emit "error" "$file_rel" "Progress format" \ + "Plan '${file_rel}' must include timestamped progress checkboxes with IDs (e.g. '- [ ] (2026-02-15T12:00:00Z) P1 ...')." + fi +} + +# ── Check: checklists only in Progress ─────────────────────────────── +check_checklists_only_in_progress() { + local file_rel="$1" file_abs="$2" + local bad=0 + local in_progress=0 + while IFS= read -r line; do + if [[ "$line" == "## Progress" ]]; then + in_progress=1 + continue + fi + if [[ "$line" == "## "* ]]; then + in_progress=0 + fi + if [[ $in_progress -eq 0 ]] && echo "$line" | grep -qE '^- \[[ xX]\]'; then + bad=1 + break + fi + done < "$file_abs" + + if [[ $bad -eq 1 ]]; then + emit "error" "$file_rel" "Checklist scope" \ + "Plan '${file_rel}' contains checklist items outside ## Progress." + fi +} + +# ── Check: Decision Log ────────────────────────────────────────────── +check_decision_log() { + local file_rel="$1" file_abs="$2" + local body + body="$(section_lines "$file_abs" "## Decision Log")" + + if ! echo "$body" | grep -q '[^[:space:]]'; then + emit "error" "$file_rel" "Missing Decision Log content" \ + "Plan '${file_rel}' has an empty ## Decision Log section." + return + fi + + if ! echo "$body" | grep -q '^- Decision:'; then + emit "error" "$file_rel" "Decision format" \ + "Plan '${file_rel}' should record decisions using '- Decision:' entries." + fi +} + +# ── Check: Revision Notes ──────────────────────────────────────────── +check_revision_notes() { + local file_rel="$1" file_abs="$2" + local body + body="$(section_lines "$file_abs" "## Revision Notes")" + + if ! echo "$body" | grep -q '[^[:space:]]'; then + emit "error" "$file_rel" "Missing Revision Notes content" \ + "Plan '${file_rel}' has an empty ## Revision Notes section." + return + fi + + if ! echo "$body" | grep -q '^- '; then + emit "error" "$file_rel" "Revision Notes format" \ + "Plan '${file_rel}' should include at least one bullet in ## Revision Notes." + fi +} + +# ── Check: placeholder tokens ──────────────────────────────────────── +check_placeholders() { + local file_rel="$1" file_abs="$2" + local fail_ph=0 + [[ "${HARNESS_FAIL_ON_ARTIFACT_PLACEHOLDERS:-0}" == "1" ]] && fail_ph=1 + + local text + text="$(cat "$file_abs")" + + while IFS= read -r pattern; do + [[ -z "$pattern" ]] && continue + if echo "$text" | grep -qF "$pattern"; then + local level="warning" + local msg="Plan '${file_rel}' contains placeholder token '${pattern}'." + if [[ $fail_ph -eq 1 ]]; then + level="error" + else + msg="${msg} (Set HARNESS_FAIL_ON_ARTIFACT_PLACEHOLDERS=1 to enforce.)" + fi + emit "$level" "$file_rel" "Placeholder token" "$msg" + break + fi + done < <(cfg_get_array '.artifact_placeholder_patterns') +} + +# ── Check a single plan file ───────────────────────────────────────── +check_plan() { + local file_abs="$1" + local file_rel="${file_abs#"$REPO_ROOT/"}" + + # Frontmatter + if ! extract_frontmatter "$file_abs"; then + emit "error" "$file_rel" "Missing YAML frontmatter" \ + "Plan '${file_rel}' must start with YAML frontmatter delimited by '---' lines." + return + fi + + parse_frontmatter_kv + + # Required frontmatter keys + while IFS= read -r key; do + [[ -z "$key" ]] && continue + if ! fm_has_key "$key"; then + emit "error" "$file_rel" "Missing frontmatter key" \ + "Plan '${file_rel}' missing YAML frontmatter key '${key}:'." + fi + done < <(cfg_get_array '.required_plan_frontmatter_keys') + + # plan_mode validation + local plan_mode + plan_mode="$(fm_get "plan_mode" 2>/dev/null || true)" + plan_mode="$(echo "$plan_mode" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [[ -n "$plan_mode" && "$plan_mode" != "trivial" && "$plan_mode" != "lightweight" && "$plan_mode" != "execution" ]]; then + emit "error" "$file_rel" "Invalid plan_mode" \ + "Plan '${file_rel}' has invalid plan_mode '${plan_mode}' (must be 'trivial', 'lightweight', or 'execution')." + fi + + # Required headings + for h in "${DEFAULT_REQUIRED_HEADINGS[@]}"; do + if ! has_exact_line "$file_abs" "$h"; then + emit "error" "$file_rel" "Missing heading" \ + "Plan '${file_rel}' missing required heading line '${h}'." + fi + done + + # Section-level checks + check_progress "$file_rel" "$file_abs" + check_checklists_only_in_progress "$file_rel" "$file_abs" + check_decision_log "$file_rel" "$file_abs" + check_revision_notes "$file_rel" "$file_abs" + check_placeholders "$file_rel" "$file_abs" +} + +# ══════════════════════════════════════════════════════════════════════ +# Main +# ══════════════════════════════════════════════════════════════════════ +load_config + +echo "he-plans-lint: starting" +echo "Repro: bash scripts/ci/he-plans-lint.sh" + +plans_active="$REPO_ROOT/docs/plans/active" +plans_completed="$REPO_ROOT/docs/plans/completed" + +files=() +if [[ -d "$plans_active" ]]; then + while IFS= read -r -d '' f; do + files+=("$f") + done < <(find "$plans_active" -maxdepth 1 -name '*.md' -print0 | sort -z) +fi + +lint_completed="$(cfg_get '.lint_completed_plans // true')" +if [[ "$lint_completed" != "false" && -d "$plans_completed" ]]; then + while IFS= read -r -d '' f; do + files+=("$f") + done < <(find "$plans_completed" -maxdepth 1 -name '*.md' -print0 | sort -z) +fi + +if [[ ${#files[@]} -eq 0 ]]; then + echo "he-plans-lint: OK (no plan files)" + exit 0 +fi + +for f in "${files[@]}"; do + check_plan "$f" +done + +if [[ $ERRORS -gt 0 ]]; then + echo "he-plans-lint: FAIL (${ERRORS} error(s), ${WARNINGS} warning(s))" >&2 + exit 1 +fi + +echo "he-plans-lint: OK (${WARNINGS} warning(s))" +exit 0 diff --git a/scripts/ci/he-runbooks-lint.sh b/scripts/ci/he-runbooks-lint.sh new file mode 100755 index 0000000..0d9a0d9 --- /dev/null +++ b/scripts/ci/he-runbooks-lint.sh @@ -0,0 +1,445 @@ +#!/bin/bash +set -euo pipefail + +# --------------------------------------------------------------------------- +# he-runbooks-lint.sh -- Lint runbook frontmatter & content +# +# Exit codes: 0=OK, 1=FAIL, 2=config error +# --------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +DEFAULT_CONFIG_PATH="scripts/ci/he-docs-config.json" + +ERRORS=0 +WARNINGS=0 + +# ── env helpers ──────────────────────────────────────────────────────────── + +env_flag() { + local name="$1" + local default="${2:-0}" + local val="${!name:-$default}" + [[ "$val" == "1" ]] +} + +# ── emit / annotate ─────────────────────────────────────────────────────── + +gh_annotate() { + local level="$1" file="$2" title="$3" msg="$4" + if [[ -n "$file" ]]; then + echo "::${level} file=${file},title=${title}::${msg}" + else + echo "::${level} title=${title}::${msg}" + fi +} + +emit() { + local level="$1" file="$2" title="$3" msg="$4" + gh_annotate "$level" "$file" "$title" "$msg" + local upper + upper="$(echo "$level" | tr '[:lower:]' '[:upper:]')" + echo "${upper}: ${msg}" >&2 +} + +emit_and_count() { + local level="$1" file="$2" title="$3" msg="$4" + if [[ "$level" == "error" ]]; then + (( ERRORS++ )) || true + else + (( WARNINGS++ )) || true + fi + emit "$level" "$file" "$title" "$msg" +} + +# ── config ──────────────────────────────────────────────────────────────── + +load_config() { + local config_rel="${HARNESS_DOCS_CONFIG:-$DEFAULT_CONFIG_PATH}" + local config_path="$REPO_ROOT/$config_rel" + if [[ ! -f "$config_path" ]]; then + echo "Error: he-runbooks-lint missing/invalid config: Missing config '${config_rel}'. Fix: create it (bootstrap should do this) or set HARNESS_DOCS_CONFIG." >&2 + return 1 + fi + # Validate it is a JSON object + if ! jq -e 'type == "object"' "$config_path" >/dev/null 2>&1; then + echo "Error: he-runbooks-lint missing/invalid config: Config must be a JSON object." >&2 + return 1 + fi + CONFIG_PATH="$config_path" +} + +# ── frontmatter extraction ──────────────────────────────────────────────── + +# Reads file, outputs the frontmatter block (lines between first --- and +# second ---) to stdout. Returns 1 if no frontmatter found. +extract_frontmatter() { + local file="$1" + local in_fm=0 + local first_line=1 + local block="" + + while IFS= read -r line || [[ -n "$line" ]]; do + local trimmed + trimmed="$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if (( first_line )); then + first_line=0 + if [[ "$trimmed" == "---" ]]; then + in_fm=1 + continue + else + return 1 + fi + fi + if (( in_fm )); then + if [[ "$trimmed" == "---" ]]; then + printf '%s' "$block" + return 0 + fi + if [[ -n "$block" ]]; then + block="${block}"$'\n'"${line}" + else + block="${line}" + fi + fi + done < "$file" + + # Reached EOF without closing --- + return 1 +} + +# ── frontmatter parsing ────────────────────────────────────────────────── + +# Sets global variables: FM_TITLE, FM_USE_WHEN, FM_CALLED_FROM (newline- +# separated list), FM_KEYS (newline-separated list), FM_HAS_CALLED_FROM. +parse_frontmatter() { + local block="$1" + + FM_TITLE="" + FM_USE_WHEN="" + FM_CALLED_FROM="" + FM_KEYS="" + FM_HAS_CALLED_FROM=0 + + local lines=() + while IFS= read -r line; do + lines+=("$line") + done <<< "$block" + + local i=0 + local count=${#lines[@]} + + while (( i < count )); do + local raw="${lines[$i]}" + local trimmed + trimmed="$(echo "$raw" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + + # skip blanks and comments + if [[ -z "$trimmed" || "$trimmed" == \#* ]]; then + (( i++ )) || true + continue + fi + + # must contain a colon to be a key + if [[ "$trimmed" != *:* ]]; then + (( i++ )) || true + continue + fi + + local key val + key="$(echo "$trimmed" | sed 's/:.*//' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + val="$(echo "$trimmed" | sed 's/^[^:]*://' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + + if [[ -n "$key" ]]; then + if [[ -n "$FM_KEYS" ]]; then + FM_KEYS="${FM_KEYS}"$'\n'"${key}" + else + FM_KEYS="$key" + fi + fi + + if [[ "$key" == "title" ]]; then + FM_TITLE="$(echo "$val" | sed "s/^[[:space:]]*//;s/[[:space:]]*$//;s/^[\"']//;s/[\"']$//")" + (( i++ )) || true + continue + fi + + if [[ "$key" == "use_when" ]]; then + FM_USE_WHEN="$(echo "$val" | sed "s/^[[:space:]]*//;s/[[:space:]]*$//;s/^[\"']//;s/[\"']$//")" + (( i++ )) || true + continue + fi + + if [[ "$key" == "called_from" ]]; then + FM_HAS_CALLED_FROM=1 + # Inline array form: [a, b, c] + if [[ "$val" == \[* ]]; then + parse_called_from_inline "$val" + (( i++ )) || true + continue + fi + # YAML list form + local items="" + (( i++ )) || true + while (( i < count )); do + local sub="${lines[$i]}" + local sub_trimmed + sub_trimmed="$(echo "$sub" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [[ -z "$sub_trimmed" ]]; then + (( i++ )) || true + continue + fi + # If it looks like a new key (has colon, doesn't start with -) + if [[ "$sub_trimmed" == *:* && "$sub_trimmed" != -* ]]; then + break + fi + if [[ "$sub_trimmed" == -* ]]; then + local item + item="$(echo "$sub_trimmed" | sed 's/^-[[:space:]]*//' | sed "s/^[[:space:]]*//;s/[[:space:]]*$//;s/^[\"']//;s/[\"']$//")" + if [[ -n "$item" ]]; then + if [[ -n "$items" ]]; then + items="${items}"$'\n'"${item}" + else + items="$item" + fi + fi + fi + (( i++ )) || true + done + FM_CALLED_FROM="$items" + continue + fi + + (( i++ )) || true + done +} + +# Parses inline [a, b, c] into FM_CALLED_FROM (newline-separated). +parse_called_from_inline() { + local val="$1" + FM_CALLED_FROM="" + # Strip outer brackets + local inner + inner="$(echo "$val" | sed 's/^[[:space:]]*\[//;s/\][[:space:]]*$//')" + inner="$(echo "$inner" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [[ -z "$inner" ]]; then + return + fi + local IFS=',' + local parts + read -ra parts <<< "$inner" + for p in "${parts[@]}"; do + local trimmed + trimmed="$(echo "$p" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [[ -n "$trimmed" ]]; then + if [[ -n "$FM_CALLED_FROM" ]]; then + FM_CALLED_FROM="${FM_CALLED_FROM}"$'\n'"${trimmed}" + else + FM_CALLED_FROM="$trimmed" + fi + fi + done +} + +# ── suspicious gate-waiver check ───────────────────────────────────────── + +# Checks file text for patterns that suggest waiving skill gates. +# Sets SUSPICIOUS_MATCH to the matched snippet, or empty string. +check_suspicious_gate_waiver() { + local file="$1" + + SUSPICIOUS_MATCH="" + + local patterns=( + '\b(skip|waive|override|ignore)\b.{0,80}\b(gate|review|verify|verify-release|security|data|tests?)\b' + '\b(disable|turn off)\b.{0,80}\b(tests?|checks?|ci)\b' + '\b(force merge|merge anyway|ignore failing)\b' + ) + + for pat in "${patterns[@]}"; do + local match="" + # Use grep -ioP for PCRE; fall back to grep -ioE + match="$(grep -ioP "$pat" "$file" 2>/dev/null | head -1)" || true + if [[ -z "$match" ]]; then + match="$(grep -ioE "$pat" "$file" 2>/dev/null | head -1)" || true + fi + if [[ -z "$match" ]]; then + continue + fi + + # Find byte offset to check prefix for negation + local byte_offset="" + byte_offset="$(grep -iobP "$pat" "$file" 2>/dev/null | head -1 | cut -d: -f1)" || true + if [[ -z "$byte_offset" ]]; then + byte_offset="$(grep -iobE "$pat" "$file" 2>/dev/null | head -1 | cut -d: -f1)" || true + fi + + if [[ -n "$byte_offset" ]] && (( byte_offset > 0 )); then + local prefix_start=$(( byte_offset > 40 ? byte_offset - 40 : 0 )) + local prefix_len=$(( byte_offset - prefix_start )) + local prefix + prefix="$(dd if="$file" bs=1 skip="$prefix_start" count="$prefix_len" 2>/dev/null | tr '[:upper:]' '[:lower:]')" + # Check negation prefixes + local negated=0 + for neg in "do not" "don't" "must not" "never" "cannot" "can't" "should not"; do + if [[ "$prefix" == *"$neg"* ]]; then + negated=1 + break + fi + done + if (( negated )); then + continue + fi + fi + + # Clean up the snippet + local snippet + snippet="$(echo "$match" | tr '\n' ' ' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + SUSPICIOUS_MATCH="$snippet" + return 0 + done + + return 1 +} + +# ── lint a single runbook ───────────────────────────────────────────────── + +lint_runbook() { + local path="$1" + local fail_missing_called_from="$2" + local fail_extra_keys="$3" + + local rel="${path#"$REPO_ROOT/"}" + local strict=0 + env_flag "HARNESS_STRICT_RUNBOOKS" "0" && strict=1 || true + + # --- frontmatter presence --- + local block="" + if ! block="$(extract_frontmatter "$path")"; then + local level="warning" + (( strict )) && level="error" + emit_and_count "$level" "$rel" "Runbook frontmatter" \ + "Runbook '${rel}' must start with YAML frontmatter ('---')." + return + fi + + # --- parse --- + parse_frontmatter "$block" + + # --- required fields --- + if [[ -z "$FM_TITLE" ]]; then + local level="warning" + (( strict )) && level="error" + emit_and_count "$level" "$rel" "Runbook frontmatter" \ + "Runbook '${rel}' frontmatter must include a 'title:' field." + fi + + if [[ -z "$FM_USE_WHEN" ]]; then + local level="warning" + (( strict )) && level="error" + emit_and_count "$level" "$rel" "Runbook frontmatter" \ + "Runbook '${rel}' frontmatter must include a 'use_when:' field." + fi + + # --- called_from --- + if (( ! FM_HAS_CALLED_FROM )) || [[ -z "$FM_CALLED_FROM" ]]; then + local level="warning" + if (( strict )) || [[ "$fail_missing_called_from" == "1" ]]; then + level="error" + fi + emit_and_count "$level" "$rel" "Runbook frontmatter" \ + "Runbook '${rel}' frontmatter should include non-empty 'called_from:' (list of skills/steps where this runbook is applied)." + fi + + # --- extra keys --- + local extras="" + if [[ -n "$FM_KEYS" ]]; then + while IFS= read -r k; do + if [[ "$k" != "title" && "$k" != "use_when" && "$k" != "called_from" ]]; then + if [[ -n "$extras" ]]; then + extras="${extras}, ${k}" + else + extras="$k" + fi + fi + done <<< "$FM_KEYS" + fi + + if [[ -n "$extras" ]]; then + local level="warning" + if (( strict )) || [[ "$fail_extra_keys" == "1" ]]; then + level="error" + fi + emit_and_count "$level" "$rel" "Runbook frontmatter" \ + "Runbook '${rel}' has extra frontmatter key(s): ${extras}. Prefer keeping runbooks to {title,use_when,called_from} unless you have a strong reason." + fi + + # --- suspicious gate-waiver language --- + if check_suspicious_gate_waiver "$path"; then + local level="warning" + (( strict )) && level="error" + emit_and_count "$level" "$rel" "Potential gate waiver" \ + "Runbook '${rel}' appears to suggest waiving skill-enforced gates: '${SUSPICIOUS_MATCH}'. Runbooks are additive only; skill gates win." + fi +} + +# ── iter_runbooks ───────────────────────────────────────────────────────── + +iter_runbooks() { + local dir="$1" + if [[ ! -d "$dir" ]]; then + return + fi + find "$dir" -name '*.md' -type f | sort +} + +# ── main ────────────────────────────────────────────────────────────────── + +main() { + if ! load_config; then + return 2 + fi + + local fail_missing_called_from=0 + env_flag "HARNESS_FAIL_ON_MISSING_RUNBOOK_CALLED_FROM" "0" && fail_missing_called_from=1 || true + + local fail_extra_keys=0 + env_flag "HARNESS_FAIL_ON_EXTRA_RUNBOOK_FRONTMATTER" "0" && fail_extra_keys=1 || true + + local runbooks_dir="$REPO_ROOT/docs/runbooks" + + echo "he-runbooks-lint: starting" + echo "Repro: bash scripts/ci/he-runbooks-lint.sh" + + # --- expected runbooks from config --- + local expected_runbooks + expected_runbooks="$(jq -r '(.expected_runbooks // .required_runbooks // []) | if type == "array" then .[] else empty end' "$CONFIG_PATH" 2>/dev/null)" || true + + if [[ -n "$expected_runbooks" ]]; then + while IFS= read -r rb; do + [[ -z "$rb" ]] && continue + if [[ ! -f "$REPO_ROOT/$rb" ]]; then + emit_and_count "warning" "$rb" "Expected runbook missing" \ + "Missing runbook: '${rb}'. Policy: runbooks are additive and should not block forward progress. Fix: create it (run he-bootstrap) or remove it from expected_runbooks in config." + fi + done <<< "$expected_runbooks" + fi + + # --- lint each runbook --- + while IFS= read -r path; do + [[ -z "$path" ]] && continue + lint_runbook "$path" "$fail_missing_called_from" "$fail_extra_keys" + done < <(iter_runbooks "$runbooks_dir") + + # --- summary --- + if (( ERRORS > 0 )); then + echo "he-runbooks-lint: FAIL (${ERRORS} error(s), ${WARNINGS} warning(s))" >&2 + return 1 + fi + + echo "he-runbooks-lint: OK (${WARNINGS} warning(s))" + return 0 +} + +main "$@" diff --git a/scripts/ci/he-specs-lint.sh b/scripts/ci/he-specs-lint.sh new file mode 100755 index 0000000..bb14b28 --- /dev/null +++ b/scripts/ci/he-specs-lint.sh @@ -0,0 +1,258 @@ +#!/bin/bash +set -euo pipefail + +# ── Repo root relative to script location (scripts/ci/he-specs-lint.sh) ── +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +DEFAULT_CONFIG_PATH="scripts/ci/he-docs-config.json" + +# ── Default required headings ── +DEFAULT_REQUIRED_HEADINGS=( + "## Purpose / Big Picture" + "## Scope" + "## Non-Goals" + "## Risks" + "## Rollout" + "## Validation and Acceptance Signals" + "## Requirements" + "## Success Criteria" + "## Priority" + "## Initial Milestone Candidates" + "## Revision Notes" +) + +DEFAULT_TRIVIAL_REQUIRED_HEADINGS=( + "## Purpose / Big Picture" + "## Requirements" + "## Success Criteria" +) + +# ── Counters ── +errors=0 +warnings=0 + +# ── Helpers ── + +gh_annotate() { + local level="$1" file="$2" title="$3" msg="$4" + if [[ -n "$file" ]]; then + echo "::${level} file=${file},title=${title}::${msg}" + else + echo "::${level} title=${title}::${msg}" + fi +} + +emit() { + local level="$1" file="$2" title="$3" msg="$4" + gh_annotate "$level" "$file" "$title" "$msg" + local upper + upper="$(echo "$level" | tr '[:lower:]' '[:upper:]')" + echo "${upper}: ${msg}" >&2 + if [[ "$level" == "error" ]]; then + (( errors++ )) || true + else + (( warnings++ )) || true + fi +} + +# Extract frontmatter block (content between first --- and second ---). +# Returns via stdout; returns 1 if no valid frontmatter found. +extract_frontmatter() { + local file="$1" + local first_line + first_line="$(head -n1 "$file")" + # Trim whitespace + first_line="$(echo "$first_line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [[ "$first_line" != "---" ]]; then + return 1 + fi + # Print lines between first --- and second ---, exclusive + awk 'NR==1 && /^[[:space:]]*---[[:space:]]*$/ { found=1; next } + found && /^[[:space:]]*---[[:space:]]*$/ { exit } + found { print }' "$file" + # Verify we actually found a closing --- + local count + count="$(awk '/^[[:space:]]*---[[:space:]]*$/ { c++ } c==2 { print c; exit }' "$file")" + if [[ "$count" != "2" ]]; then + return 1 + fi + return 0 +} + +# Parse frontmatter key-value pairs into an associative array. +# Usage: parse_frontmatter "$frontmatter_text" +# Sets global associative array FM_KV. +parse_frontmatter() { + local fm_text="$1" + FM_KV=() + while IFS= read -r raw_line; do + # Trim + local line + line="$(echo "$raw_line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + # Skip empty lines and comments + [[ -z "$line" || "$line" == \#* ]] && continue + # Must contain a colon + [[ "$line" != *:* ]] && continue + local key val + key="$(echo "$line" | cut -d: -f1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + val="$(echo "$line" | cut -d: -f2- | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + FM_KV["$key"]="$val" + done <<< "$fm_text" +} + +# Check if file contains an exact line match. +has_exact_line() { + local file="$1" needle="$2" + grep -qFx "$needle" "$file" +} + +# Check for placeholder tokens in file text. +check_placeholders() { + local file_rel="$1" file_path="$2" fail_ph="$3" + shift 3 + local patterns=("$@") + for p in "${patterns[@]}"; do + [[ -z "$p" ]] && continue + if grep -qF "$p" "$file_path"; then + local msg="Spec '${file_rel}' contains placeholder token '${p}'." + if [[ "$fail_ph" == "1" ]]; then + emit "error" "$file_rel" "Placeholder token" "$msg" + else + emit "warning" "$file_rel" "Placeholder token" "${msg} (Set HARNESS_FAIL_ON_ARTIFACT_PLACEHOLDERS=1 to enforce.)" + fi + break + fi + done +} + +# ── Load config ── +load_config() { + local config_rel="${HARNESS_DOCS_CONFIG:-$DEFAULT_CONFIG_PATH}" + local config_path="${REPO_ROOT}/${config_rel}" + if [[ ! -f "$config_path" ]]; then + echo "Error: he-specs-lint missing/invalid config: Missing config '${config_rel}'. Fix: create it (bootstrap should do this) or set HARNESS_DOCS_CONFIG." >&2 + exit 2 + fi + # Validate it's a JSON object + if ! jq -e 'type == "object"' "$config_path" > /dev/null 2>&1; then + echo "Error: he-specs-lint missing/invalid config: Config must be a JSON object." >&2 + exit 2 + fi + CONFIG_PATH="$config_path" +} + +# ── Check a single spec file ── +check_spec() { + local file_path="$1" + local rel="${file_path#"${REPO_ROOT}"/}" + + # Extract frontmatter + local fm_text + if ! fm_text="$(extract_frontmatter "$file_path")"; then + emit "error" "$rel" "Missing YAML frontmatter" \ + "Spec '${rel}' must start with YAML frontmatter delimited by '---' lines." + return + fi + + # Parse frontmatter key-value pairs + declare -A FM_KV + parse_frontmatter "$fm_text" + + # Required frontmatter keys from config + local required_keys_json + required_keys_json="$(jq -r '(.required_spec_frontmatter_keys // []) | if type == "array" then .[] else empty end' "$CONFIG_PATH" 2>/dev/null)" || true + if [[ -n "$required_keys_json" ]]; then + while IFS= read -r k; do + [[ -z "$k" ]] && continue + if [[ -z "${FM_KV[$k]+x}" ]]; then + emit "error" "$rel" "Missing frontmatter key" \ + "Spec '${rel}' missing YAML frontmatter key '${k}:'." + fi + done <<< "$required_keys_json" + fi + + # Validate plan_mode + local plan_mode="${FM_KV[plan_mode]:-}" + plan_mode="$(echo "$plan_mode" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [[ -n "$plan_mode" && "$plan_mode" != "trivial" && "$plan_mode" != "lightweight" && "$plan_mode" != "execution" ]]; then + emit "error" "$rel" "Invalid plan_mode" \ + "Spec '${rel}' has invalid plan_mode '${plan_mode}' (must be 'trivial', 'lightweight', or 'execution')." + fi + + # Validate spike_recommended + local spike_rec="${FM_KV[spike_recommended]:-}" + spike_rec="$(echo "$spike_rec" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [[ -n "$spike_rec" && "$spike_rec" != "yes" && "$spike_rec" != "no" ]]; then + emit "error" "$rel" "Invalid spike_recommended" \ + "Spec '${rel}' has invalid spike_recommended '${spike_rec}' (must be 'yes' or 'no')." + fi + + # Required headings + local -a required_headings + if [[ "$plan_mode" == "trivial" ]]; then + required_headings=("${DEFAULT_TRIVIAL_REQUIRED_HEADINGS[@]}") + else + required_headings=("${DEFAULT_REQUIRED_HEADINGS[@]}") + fi + for h in "${required_headings[@]}"; do + if ! has_exact_line "$file_path" "$h"; then + emit "error" "$rel" "Missing heading" \ + "Spec '${rel}' missing required heading line '${h}'." + fi + done + + # Placeholder patterns + local -a placeholder_patterns=() + local patterns_json + patterns_json="$(jq -r '(.artifact_placeholder_patterns // []) | if type == "array" then .[] else empty end' "$CONFIG_PATH" 2>/dev/null)" || true + if [[ -n "$patterns_json" ]]; then + while IFS= read -r p; do + [[ -n "$p" ]] && placeholder_patterns+=("$p") + done <<< "$patterns_json" + fi + + local fail_ph="${HARNESS_FAIL_ON_ARTIFACT_PLACEHOLDERS:-0}" + if [[ ${#placeholder_patterns[@]} -gt 0 ]]; then + check_placeholders "$rel" "$file_path" "$fail_ph" "${placeholder_patterns[@]}" + fi +} + +# ── Main ── +main() { + load_config + + echo "he-specs-lint: starting" + echo "Repro: bash scripts/ci/he-specs-lint.sh" + + local specs_dir="${REPO_ROOT}/docs/specs" + if [[ ! -d "$specs_dir" ]]; then + echo "he-specs-lint: OK (docs/specs not present)" + exit 0 + fi + + # Collect spec files (*.md excluding README.md and index.md), sorted + local -a files=() + while IFS= read -r -d '' f; do + local basename + basename="$(basename "$f")" + [[ "$basename" == "README.md" || "$basename" == "index.md" ]] && continue + files+=("$f") + done < <(find "$specs_dir" -maxdepth 1 -name '*.md' -print0 | sort -z) + + if [[ ${#files[@]} -eq 0 ]]; then + echo "he-specs-lint: OK (no spec files)" + exit 0 + fi + + for f in "${files[@]}"; do + check_spec "$f" + done + + if [[ $errors -gt 0 ]]; then + echo "he-specs-lint: FAIL (${errors} error(s), ${warnings} warning(s))" >&2 + exit 1 + fi + echo "he-specs-lint: OK (${warnings} warning(s))" + exit 0 +} + +main "$@" diff --git a/scripts/ci/he-spikes-lint.sh b/scripts/ci/he-spikes-lint.sh new file mode 100755 index 0000000..255a864 --- /dev/null +++ b/scripts/ci/he-spikes-lint.sh @@ -0,0 +1,249 @@ +#!/bin/bash +set -euo pipefail + +# --------------------------------------------------------------------------- +# he-spikes-lint.sh — Lint spike documents under docs/spikes +# --------------------------------------------------------------------------- + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +DEFAULT_CONFIG_PATH="scripts/ci/he-docs-config.json" + +# Default required headings (one per line for easy iteration) +DEFAULT_REQUIRED_HEADINGS=( + "## Context" + "## Validation Goal" + "## Approach" + "## Findings" + "## Decisions" + "## Recommendation" + "## Impact on Upstream Docs" + "## Spike Code" + "## Remaining Unknowns" + "## Time Spent" + "## Revision Notes" +) + +# Counters +errors=0 +warnings=0 + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +gh_annotate() { + local level="$1" file="$2" title="$3" msg="$4" + if [[ -n "$file" ]]; then + echo "::${level} file=${file},title=${title}::${msg}" + else + echo "::${level} title=${title}::${msg}" + fi +} + +emit() { + local level="$1" file="$2" title="$3" msg="$4" + gh_annotate "$level" "$file" "$title" "$msg" + local upper + upper="$(echo "$level" | tr '[:lower:]' '[:upper:]')" + echo "${upper}: ${msg}" >&2 + if [[ "$level" == "error" ]]; then + (( errors++ )) || true + else + (( warnings++ )) || true + fi +} + +# Extract YAML frontmatter (text between first two --- lines, exclusive). +# Prints frontmatter to stdout. Returns 1 if no valid frontmatter found. +extract_frontmatter() { + local file="$1" + local first_line + first_line="$(head -n1 "$file" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [[ "$first_line" != "---" ]]; then + return 1 + fi + # Find the closing --- (skip line 1, start from line 2) + local line_num=0 + local found=0 + while IFS= read -r line; do + line_num=$((line_num + 1)) + if [[ $line_num -eq 1 ]]; then + continue + fi + local trimmed + trimmed="$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [[ "$trimmed" == "---" ]]; then + found=1 + break + fi + echo "$line" + done < "$file" + if [[ $found -eq 0 ]]; then + return 1 + fi + return 0 +} + +# Extract keys from frontmatter text (stdin). +# Outputs one key per line. +frontmatter_keys() { + while IFS= read -r raw; do + local line + line="$(echo "$raw" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + # skip blank lines and comments + [[ -z "$line" ]] && continue + [[ "$line" == \#* ]] && continue + # must contain a colon + [[ "$line" != *:* ]] && continue + # extract key (everything before first colon), trimmed + local key + key="$(echo "$line" | cut -d: -f1 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + echo "$key" + done +} + +# Check if a file contains an exact full line matching the needle. +has_exact_line() { + local file="$1" needle="$2" + grep -qFx "$needle" "$file" +} + +# --------------------------------------------------------------------------- +# Config loading +# --------------------------------------------------------------------------- + +load_config() { + local config_rel="${HARNESS_DOCS_CONFIG:-$DEFAULT_CONFIG_PATH}" + local config_path="$REPO_ROOT/$config_rel" + if [[ ! -f "$config_path" ]]; then + echo "Error: he-spikes-lint missing/invalid config: Missing config '${config_rel}'. Fix: create it (bootstrap should do this) or set HARNESS_DOCS_CONFIG." >&2 + exit 2 + fi + # Validate it is a JSON object + if ! jq -e 'type == "object"' "$config_path" >/dev/null 2>&1; then + echo "Error: he-spikes-lint missing/invalid config: Config must be a JSON object." >&2 + exit 2 + fi + CONFIG_PATH="$config_path" +} + +# Read a JSON array from config as newline-delimited strings. +config_string_array() { + local key="$1" + jq -r "(.${key} // []) | if type == \"array\" then .[] else empty end" "$CONFIG_PATH" 2>/dev/null | while IFS= read -r v; do + # only emit strings + echo "$v" + done +} + +# --------------------------------------------------------------------------- +# Per-spike checks +# --------------------------------------------------------------------------- + +check_placeholders() { + local rel="$1" file="$2" fail_ph="$3" + shift 3 + local patterns=("$@") + for p in "${patterns[@]}"; do + [[ -z "$p" ]] && continue + if grep -qF "$p" "$file"; then + local msg="Spike '${rel}' contains placeholder token '${p}'." + if [[ "$fail_ph" == "1" ]]; then + emit "error" "$rel" "Placeholder token" "$msg" + else + emit "warning" "$rel" "Placeholder token" "${msg} (Set HARNESS_FAIL_ON_ARTIFACT_PLACEHOLDERS=1 to enforce.)" + fi + break + fi + done +} + +check_spike() { + local file="$1" + local rel="${file#"$REPO_ROOT"/}" + + # --- frontmatter --- + local fm + if ! fm="$(extract_frontmatter "$file")"; then + emit "error" "$rel" "Missing YAML frontmatter" \ + "Spike '${rel}' must start with YAML frontmatter delimited by '---' lines." + return + fi + + # Check required frontmatter keys + local fm_keys + fm_keys="$(echo "$fm" | frontmatter_keys)" + + local required_keys + required_keys="$(config_string_array "required_spike_frontmatter_keys")" + + if [[ -n "$required_keys" ]]; then + while IFS= read -r k; do + [[ -z "$k" ]] && continue + if ! echo "$fm_keys" | grep -qFx "$k"; then + emit "error" "$rel" "Missing frontmatter key" \ + "Spike '${rel}' missing YAML frontmatter key '${k}:'." + fi + done <<< "$required_keys" + fi + + # --- required headings --- + for h in "${DEFAULT_REQUIRED_HEADINGS[@]}"; do + if ! has_exact_line "$file" "$h"; then + emit "error" "$rel" "Missing heading" \ + "Spike '${rel}' missing required heading line '${h}'." + fi + done + + # --- placeholder tokens --- + local placeholder_patterns=() + while IFS= read -r p; do + [[ -z "$p" ]] && continue + placeholder_patterns+=("$p") + done < <(config_string_array "artifact_placeholder_patterns") + + local fail_ph="${HARNESS_FAIL_ON_ARTIFACT_PLACEHOLDERS:-0}" + if [[ ${#placeholder_patterns[@]} -gt 0 ]]; then + check_placeholders "$rel" "$file" "$fail_ph" "${placeholder_patterns[@]}" + fi +} + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + +load_config + +echo "he-spikes-lint: starting" +echo "Repro: bash scripts/ci/he-spikes-lint.sh" + +spikes_dir="$REPO_ROOT/docs/spikes" +if [[ ! -d "$spikes_dir" ]]; then + echo "he-spikes-lint: OK (docs/spikes not present)" + exit 0 +fi + +# Collect spike files sorted +spike_files=() +while IFS= read -r -d '' f; do + spike_files+=("$f") +done < <(find "$spikes_dir" -maxdepth 1 -name '*-spike.md' -print0 | sort -z) + +if [[ ${#spike_files[@]} -eq 0 ]]; then + echo "he-spikes-lint: OK (no spike files)" + exit 0 +fi + +for f in "${spike_files[@]}"; do + check_spike "$f" +done + +if [[ $errors -gt 0 ]]; then + echo "he-spikes-lint: FAIL (${errors} error(s), ${warnings} warning(s))" >&2 + exit 1 +fi + +echo "he-spikes-lint: OK (${warnings} warning(s))" +exit 0 diff --git a/scripts/claude-datafog-setup.sh b/scripts/claude-datafog-setup.sh new file mode 100755 index 0000000..9c5297e --- /dev/null +++ b/scripts/claude-datafog-setup.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash + +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: + scripts/claude-datafog-setup.sh [options] + +Options: + --policy-url Policy service endpoint (default: http://localhost:8080) + --shim-bin Path to datafog-shim binary (default: build ./cmd/datafog-shim) + --claude-bin Path to claude binary (default: claude in PATH) + --shim-dir Shim install dir (default: $DATAFOG_SHIM_DIR or ~/.datafog/shims) + --event-sink NDJSON sink path (default: ~/.datafog/decisions.ndjson) + --api-token Optional token forwarded to policy checks + --mode Enforcement mode for shim calls (default: enforced) + --install-git Also gate git commands through the same shim family + --dry-run Print planned actions without changing files + --help Show this help text +EOF +} + +POLICY_URL="http://localhost:8080" +SHIM_BIN="${DATAFOG_SHIM_BINARY:-}" +CLAUDE_BIN="${DATAFOG_CLAUDE_BINARY:-}" +SHIM_DIR="${DATAFOG_SHIM_DIR:-${HOME}/.datafog/shims}" +EVENT_SINK="${DATAFOG_SHIM_EVENT_SINK:-${HOME}/.datafog/decisions.ndjson}" +MODE="enforced" +API_TOKEN="${DATAFOG_API_TOKEN:-${DATAFOG_SHIM_API_TOKEN:-}}" +DRY_RUN=0 +INSTALL_GIT=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --policy-url) + [[ $# -ge 2 ]] || { echo "missing --policy-url value" >&2; exit 1; } + POLICY_URL="$2" + shift 2 + ;; + --shim-bin) + [[ $# -ge 2 ]] || { echo "missing --shim-bin value" >&2; exit 1; } + SHIM_BIN="$2" + shift 2 + ;; + --claude-bin) + [[ $# -ge 2 ]] || { echo "missing --claude-bin value" >&2; exit 1; } + CLAUDE_BIN="$2" + shift 2 + ;; + --shim-dir) + [[ $# -ge 2 ]] || { echo "missing --shim-dir value" >&2; exit 1; } + SHIM_DIR="$2" + shift 2 + ;; + --event-sink) + [[ $# -ge 2 ]] || { echo "missing --event-sink value" >&2; exit 1; } + EVENT_SINK="$2" + shift 2 + ;; + --api-token) + [[ $# -ge 2 ]] || { echo "missing --api-token value" >&2; exit 1; } + API_TOKEN="$2" + shift 2 + ;; + --mode) + [[ $# -ge 2 ]] || { echo "missing --mode value" >&2; exit 1; } + MODE="$2" + if [[ "$MODE" != "enforced" && "$MODE" != "observe" ]]; then + echo "--mode must be enforced or observe" >&2 + exit 1 + fi + shift 2 + ;; + --install-git) + INSTALL_GIT=1 + shift + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "unknown flag: $1" >&2 + usage + exit 1 + ;; + esac +done + +if [[ -z "${CLAUDE_BIN}" ]]; then + CLAUDE_BIN="$(command -v claude || true)" + if [[ -z "${CLAUDE_BIN}" ]]; then + echo "claude binary not found in PATH. Pass --claude-bin ." >&2 + exit 1 + fi +fi + +if [[ -z "${SHIM_BIN}" ]]; then + ROOT_DIR="$(git rev-parse --show-toplevel)" + SHIM_BIN="${ROOT_DIR}/datafog-shim" +fi + +if [[ ! -x "${SHIM_BIN}" ]]; then + if (( DRY_RUN )); then + echo "[dry-run] would build datafog-shim at ${SHIM_BIN}" + else + echo "building datafog-shim at ${SHIM_BIN}" + go build -o "${SHIM_BIN}" ./cmd/datafog-shim + fi +fi + +if (( DRY_RUN )); then + echo "[dry-run] would install shim for claude" +else + mkdir -p "${SHIM_DIR}" +fi + +shim_args=( + "--force" + "--policy-url" "${POLICY_URL}" + "--mode" "${MODE}" + "--event-sink" "${EVENT_SINK}" + "--shim-dir" "${SHIM_DIR}" +) + +if [[ -n "${API_TOKEN}" ]]; then + shim_args+=( "--api-token" "${API_TOKEN}" ) +fi + +run_cmd=( + "${SHIM_BIN}" + "hooks" + "install" + "${shim_args[@]}" + "--adapter" "claude" + "--target" "${CLAUDE_BIN}" + "claude" +) + +if (( DRY_RUN )); then + printf '[dry-run] %q ' "${run_cmd[@]}" + printf '\n' +else + echo "installing claude shim: ${SHIM_DIR}/claude" + "${run_cmd[@]}" +fi + +if (( INSTALL_GIT )); then + GIT_BIN="$(command -v git || true)" + if [[ -z "${GIT_BIN}" ]]; then + echo "git not found in PATH; skipping git shim install" >&2 + else + if (( DRY_RUN )); then + printf '[dry-run] %q --adapter vcs --target %q git\n' \ + "${SHIM_BIN} hooks install" "${GIT_BIN}" + else + "${SHIM_BIN}" hooks install "${shim_args[@]}" --adapter vcs --target "${GIT_BIN}" git || { + echo "failed to install git shim" >&2 + exit 1 + } + fi + fi +fi + +CONFIG_DIR="${HOME}/.datafog" +if (( DRY_RUN )); then + echo "[dry-run] would write ${CONFIG_DIR}/claude-datafog.env" +else + mkdir -p "${CONFIG_DIR}" +fi + +ENV_FILE="${CONFIG_DIR}/claude-datafog.env" + +if (( DRY_RUN )); then + echo "[dry-run] would write environment file: ${ENV_FILE}" +else + cat >"${ENV_FILE}" <>"${ENV_FILE}" + fi + echo "wrote environment helper: ${ENV_FILE}" +fi + +cat < Policy service endpoint (default: http://localhost:8080) + --shim-bin Path to datafog-shim binary (default: build ./cmd/datafog-shim) + --codex-bin Path to codex binary (default: codex in PATH) + --shim-dir Shim install dir (default: $DATAFOG_SHIM_DIR or ~/.datafog/shims) + --event-sink NDJSON sink path (default: ~/.datafog/decisions.ndjson) + --api-token Optional token forwarded to policy checks + --mode Enforcement mode for shim calls (default: enforced) + --install-git Also gate git commands through the same shim + --dry-run Print planned actions without changing files + --help Show this help text +EOF +} + +POLICY_URL="http://localhost:8080" +SHIM_BIN="${DATAFOG_SHIM_BINARY:-}" +CODEX_BIN="${DATAFOG_CODEX_BINARY:-}" +SHIM_DIR="${DATAFOG_SHIM_DIR:-${HOME}/.datafog/shims}" +EVENT_SINK="${DATAFOG_SHIM_EVENT_SINK:-${HOME}/.datafog/decisions.ndjson}" +MODE="enforced" +API_TOKEN="${DATAFOG_API_TOKEN:-${DATAFOG_SHIM_API_TOKEN:-}}" +DRY_RUN=0 +INSTALL_GIT=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --policy-url) + [[ $# -ge 2 ]] || { echo "missing --policy-url value" >&2; exit 1; } + POLICY_URL="$2" + shift 2 + ;; + --shim-bin) + [[ $# -ge 2 ]] || { echo "missing --shim-bin value" >&2; exit 1; } + SHIM_BIN="$2" + shift 2 + ;; + --codex-bin) + [[ $# -ge 2 ]] || { echo "missing --codex-bin value" >&2; exit 1; } + CODEX_BIN="$2" + shift 2 + ;; + --shim-dir) + [[ $# -ge 2 ]] || { echo "missing --shim-dir value" >&2; exit 1; } + SHIM_DIR="$2" + shift 2 + ;; + --event-sink) + [[ $# -ge 2 ]] || { echo "missing --event-sink value" >&2; exit 1; } + EVENT_SINK="$2" + shift 2 + ;; + --api-token) + [[ $# -ge 2 ]] || { echo "missing --api-token value" >&2; exit 1; } + API_TOKEN="$2" + shift 2 + ;; + --mode) + [[ $# -ge 2 ]] || { echo "missing --mode value" >&2; exit 1; } + MODE="$2" + if [[ "$MODE" != "enforced" && "$MODE" != "observe" ]]; then + echo "--mode must be enforced or observe" >&2 + exit 1 + fi + shift 2 + ;; + --install-git) + INSTALL_GIT=1 + shift + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "unknown flag: $1" >&2 + usage + exit 1 + ;; + esac +done + +if [[ -z "${CODEX_BIN}" ]]; then + CODEX_BIN="$(command -v codex || true)" + if [[ -z "${CODEX_BIN}" ]]; then + echo "codex binary not found in PATH. Pass --codex-bin ." >&2 + exit 1 + fi +fi + +if [[ -z "${SHIM_BIN}" ]]; then + ROOT_DIR="$(git rev-parse --show-toplevel)" + SHIM_BIN="${ROOT_DIR}/datafog-shim" +fi + +if [[ ! -x "${SHIM_BIN}" ]]; then + if (( DRY_RUN )); then + echo "[dry-run] would build datafog-shim at ${SHIM_BIN}" + else + echo "building datafog-shim at ${SHIM_BIN}" + go build -o "${SHIM_BIN}" ./cmd/datafog-shim + fi +fi + +if (( DRY_RUN )); then + echo "[dry-run] would install shim for codex" +else + mkdir -p "${SHIM_DIR}" +fi + +shim_args=( + "--force" + "--policy-url" "${POLICY_URL}" + "--mode" "${MODE}" + "--event-sink" "${EVENT_SINK}" + "--shim-dir" "${SHIM_DIR}" +) + +if [[ -n "${API_TOKEN}" ]]; then + shim_args+=( "--api-token" "${API_TOKEN}" ) +fi + +run_cmd=( + "${SHIM_BIN}" + "hooks" + "install" + "${shim_args[@]}" + "--adapter" "codex" + "--target" "${CODEX_BIN}" + "codex" +) + +if (( DRY_RUN )); then + printf '[dry-run] %q ' "${run_cmd[@]}" + printf '\n' +else + echo "installing codex shim: ${SHIM_DIR}/codex" + "${run_cmd[@]}" +fi + +if (( INSTALL_GIT )); then + if (( DRY_RUN )); then + printf '[dry-run] %q --adapter vcs --target "$(command -v git)" git\n' \ + "${SHIM_BIN} hooks install" + else + "${SHIM_BIN}" hooks install "${shim_args[@]}" --adapter vcs --target "$(command -v git)" git || { + echo "failed to install git shim" >&2 + exit 1 + } + fi +fi + +if (( DRY_RUN )); then + echo "[dry-run] would prepend ${SHIM_DIR} to PATH in shell startup file" +else + echo "shim install complete" +fi + +CONFIG_DIR="${HOME}/.datafog" +mkdir -p "${CONFIG_DIR}" +ENV_FILE="${CONFIG_DIR}/codex-datafog.env" + +cat >"${ENV_FILE}" <>"${ENV_FILE}" +fi + +if (( DRY_RUN )); then + echo "[dry-run] would write ${ENV_FILE}" +else + echo "wrote environment helper: ${ENV_FILE}" +fi + +cat < Datafog policy URL (default: http://localhost:8080) + --shim-bin datafog-shim binary (default: build ./datafog-shim if missing) + --codex-bin codex binary (default: codex in PATH) + --claude-bin claude binary (default: claude in PATH) + --shim-dir Shim install dir to check (default: ~/.datafog/shims) + --event-sink NDJSON event sink (default: ~/.datafog/decisions.ndjson) + --mode Shim mode (default: enforced) + --out-dir Directory for generated reports (default: docs/generated/datafog-demo-reports) + --dry-run Do not execute commands, only emit scaffolded report + --skip-live Skip live action probes, only emit preflight checks + --help Show this help text +EOF +} + +POLICY_URL="${DATAFOG_POLICY_URL:-http://localhost:8080}" +SHIM_BIN="${DATAFOG_SHIM_BINARY:-}" +CODEX_BIN="${DATAFOG_CODEX_BINARY:-}" +CLAUDE_BIN="${DATAFOG_CLAUDE_BINARY:-}" +SHIM_DIR="${DATAFOG_SHIM_DIR:-${HOME}/.datafog/shims}" +EVENT_SINK="${DATAFOG_SHIM_EVENT_SINK:-${HOME}/.datafog/decisions.ndjson}" +MODE="${DATAFOG_SHIM_MODE:-enforced}" +OUT_DIR="docs/generated/datafog-demo-reports" +DRY_RUN=0 +SKIP_LIVE=0 + +while [[ $# -gt 0 ]]; do + case "$1" in + --policy-url) + [[ $# -ge 2 ]] || { echo "missing --policy-url value" >&2; exit 1; } + POLICY_URL="$2" + shift 2 + ;; + --shim-bin) + [[ $# -ge 2 ]] || { echo "missing --shim-bin value" >&2; exit 1; } + SHIM_BIN="$2" + shift 2 + ;; + --codex-bin) + [[ $# -ge 2 ]] || { echo "missing --codex-bin value" >&2; exit 1; } + CODEX_BIN="$2" + shift 2 + ;; + --claude-bin) + [[ $# -ge 2 ]] || { echo "missing --claude-bin value" >&2; exit 1; } + CLAUDE_BIN="$2" + shift 2 + ;; + --shim-dir) + [[ $# -ge 2 ]] || { echo "missing --shim-dir value" >&2; exit 1; } + SHIM_DIR="$2" + shift 2 + ;; + --event-sink) + [[ $# -ge 2 ]] || { echo "missing --event-sink value" >&2; } + EVENT_SINK="$2" + shift 2 + ;; + --mode) + [[ $# -ge 2 ]] || { echo "missing --mode value" >&2; exit 1; } + MODE="$2" + if [[ "$MODE" != "enforced" && "$MODE" != "observe" ]]; then + echo "--mode must be enforced or observe" >&2 + exit 1 + fi + shift 2 + ;; + --out-dir) + [[ $# -ge 2 ]] || { echo "missing --out-dir value" >&2; exit 1; } + OUT_DIR="$2" + shift 2 + ;; + --dry-run) + DRY_RUN=1 + shift + ;; + --skip-live) + SKIP_LIVE=1 + shift + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "unknown flag: $1" >&2 + usage + exit 1 + ;; + esac +done + +if [[ -z "$CODEX_BIN" ]]; then + CODEX_BIN="$(command -v codex || true)" +fi + +if [[ -z "$CLAUDE_BIN" ]]; then + CLAUDE_BIN="$(command -v claude || true)" +fi + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" +WORKDIR="$(mktemp -d "${TMPDIR:-/tmp}/datafog-demo-XXXXXX")" +trap 'rm -rf "${WORKDIR}"' EXIT +RUN_ID="$(date -u +"%Y%m%dT%H%M%SZ")" +REPORT_MD="${OUT_DIR}/datafog-agent-demo-${RUN_ID}.md" +REPORT_CSV="${OUT_DIR}/datafog-agent-demo-${RUN_ID}.csv" + +if [[ -z "$SHIM_BIN" ]]; then + SHIM_BIN="${REPO_ROOT}/datafog-shim" +fi + +mkdir -p "$OUT_DIR" + +if [[ "$MODE" == "observe" ]]; then + DECISION_HINT="observe mode: decisions are logged, denials are not blocking" +elif [[ "$MODE" == "enforced" ]]; then + DECISION_HINT="enforced mode: denials block execution" +else + DECISION_HINT="mode=$MODE" +fi + +LAST_RC=0 +LAST_DECISION="n/a" +LAST_RECEIPT="n/a" +LAST_STDOUT="n/a" +LAST_STDERR="n/a" + +RESULT_KEYS=() +RESULT_EXIT=() +RESULT_DECISION=() +RESULT_RECEIPT=() +RESULT_NOTES=() +RESULT_OUTCOME=() +RESULT_COMMAND=() + +result_index_for_key() { + local key="$1" + local i + + for i in "${!RESULT_KEYS[@]}"; do + if [[ "${RESULT_KEYS[$i]}" == "$key" ]]; then + echo "$i" + return 0 + fi + done + echo "-1" + return 1 +} + +result_get() { + local key="$1" + local field="$2" + local default="${3:-}" + local idx + + idx="$(result_index_for_key "$key" || true)" + if [[ -z "$idx" || "$idx" == "-1" ]]; then + echo "$default" + return 0 + fi + + case "$field" in + exit) + echo "${RESULT_EXIT[$idx]:-$default}" + ;; + decision) + echo "${RESULT_DECISION[$idx]:-$default}" + ;; + receipt) + echo "${RESULT_RECEIPT[$idx]:-$default}" + ;; + notes) + echo "${RESULT_NOTES[$idx]:-$default}" + ;; + outcome) + echo "${RESULT_OUTCOME[$idx]:-$default}" + ;; + command) + echo "${RESULT_COMMAND[$idx]:-$default}" + ;; + *) + echo "$default" + ;; + esac +} + +result_set() { + local key="$1" + local field="$2" + local value="$3" + local idx + + idx="$(result_index_for_key "$key" || true)" + if [[ -z "$idx" || "$idx" == "-1" ]]; then + idx="${#RESULT_KEYS[@]}" + RESULT_KEYS+=("$key") + RESULT_EXIT+=("") + RESULT_DECISION+=("") + RESULT_RECEIPT+=("") + RESULT_NOTES+=("") + RESULT_OUTCOME+=("") + RESULT_COMMAND+=("") + fi + case "$field" in + exit) + RESULT_EXIT[$idx]="$value" + ;; + decision) + RESULT_DECISION[$idx]="$value" + ;; + receipt) + RESULT_RECEIPT[$idx]="$value" + ;; + notes) + RESULT_NOTES[$idx]="$value" + ;; + command) + RESULT_COMMAND[$idx]="$value" + ;; + outcome) + RESULT_OUTCOME[$idx]="$value" + ;; + *) + ;; + esac +} + +SCENARIOS=( + "cli-help:Open help command" + "read-secret:Read .env.secret" + "write-output:Write output artifact" + "delete-artifact:Delete artifact" +) + +CONTROL_SCENARIOS=( + "policy-outage:Policy API outage fail-closed" + "decide-redaction:Decide API returns allow_with_redaction" + "transform-mask:Transform API masks PII" +) + +result_key() { + local agent=$1 + local mode=$2 + local probe=$3 + printf "%s|%s|%s" "$agent" "$mode" "$probe" +} + +probe_outcome() { + local mode=$1 + local rc=$2 + local decision=$3 + local notes=$4 + + if [[ "$mode" == "without-datafog" ]]; then + if [[ "$rc" == "0" ]]; then + echo "ALLOWED" + elif [[ "$notes" == "missing binary" ]]; then + echo "SKIP" + else + echo "FAILED" + fi + return + fi + + if [[ "$notes" == "datafog-shim missing" || "$notes" == "shim path missing" || "$notes" == "skip-live enabled" ]]; then + echo "SKIP" + return + fi + if [[ "$notes" == *"call decide API"* || "$notes" == *"No such host"* || "$notes" == *"connection refused"* ]]; then + echo "ERROR" + return + fi + if [[ "$decision" == "deny" ]]; then + echo "BLOCKED" + return + fi + if [[ "$decision" == "transform" ]]; then + echo "TRANSFORM" + return + fi + if [[ "$decision" == "allow_with_redaction" ]]; then + echo "ALLOWED_WITH_REDACTION" + return + fi + if [[ "$rc" == "0" ]]; then + echo "ALLOWED" + elif [[ "$rc" == "0" || "$rc" == "1" ]]; then + echo "BLOCKED" + else + echo "ERROR" + fi +} + +record_probe_result() { + local agent=$1 + local mode=$2 + local probe=$3 + local command=$4 + local rc=$5 + local decision=$6 + local receipt=$7 + local notes=$8 + + local key + key="$(result_key "$agent" "$mode" "$probe")" + local outcome + outcome="$(probe_outcome "$mode" "$rc" "$decision" "$notes")" + result_set "$key" "exit" "$rc" + result_set "$key" "decision" "$decision" + result_set "$key" "receipt" "$receipt" + result_set "$key" "notes" "$notes" + result_set "$key" "command" "$command" + result_set "$key" "outcome" "$outcome" +} + +probe_status_text() { + local agent=$1 + local mode=$2 + local probe=$3 + local key + key="$(result_key "$agent" "$mode" "$probe")" + local outcome + local rc + local decision + local notes + + outcome="$(result_get "$key" "outcome" "UNKNOWN")" + rc="$(result_get "$key" "exit" "n/a")" + decision="$(result_get "$key" "decision" "n/a")" + notes="$(result_get "$key" "notes" "n/a")" + + printf "%s (rc=%s)" "$outcome" "$rc" + if [[ "$outcome" == "ALLOWED" && -n "$decision" && "$decision" != "n/a" ]]; then + printf " [decision=%s]" "$decision" + fi + if [[ "$outcome" == "ERROR" ]]; then + printf " (%s)" "$notes" + fi +} + +scenario_label() { + local scenario=$1 + case "$scenario" in + cli-help) echo "CLI help" ;; + read-secret) echo "Read secret file" ;; + write-output) echo "Write output" ;; + delete-artifact) echo "Delete file" ;; + *) echo "$scenario" ;; + esac +} + +scenario_risk() { + local scenario=$1 + case "$scenario" in + read-secret) echo "HIGH: sensitive file read" ;; + write-output) echo "MED: output write" ;; + delete-artifact) echo "HIGH: destructive delete" ;; + cli-help) echo "LOW: informational" ;; + *) echo "UNKNOWN" ;; + esac +} + +extract_from_file() { + local file=$1 + local key=$2 + awk -v key="$key" '{ + idx = index($0, key "=") + if (idx > 0) { + value = substr($0, idx + length(key) + 1) + sub(/ .*/, "", value) + print value + exit + } + }' "$file" +} + +json_value() { + local payload=$1 + local expr=$2 + local default=${3:-n/a} + + if ! command -v jq >/dev/null 2>&1; then + echo "$default" + return 0 + fi + + local value + value="$(printf "%s" "$payload" | jq -r "$expr" 2>/dev/null || true)" + if [[ -z "$value" || "$value" == "null" ]]; then + echo "$default" + return 0 + fi + echo "$value" +} + +run_http_probe() { + local method=$1 + local url=$2 + local body=$3 + + local out_file + local err_file + out_file="$(mktemp)" + err_file="$(mktemp)" + + LAST_RC=0 + LAST_DECISION="n/a" + LAST_RECEIPT="n/a" + LAST_STDOUT="" + LAST_STDERR="" + + if (( DRY_RUN )); then + LAST_STDERR="DRY-RUN for: ${method} ${url}" + rm -f "$out_file" "$err_file" + return 0 + fi + + set +e + if [[ "$method" == "GET" ]]; then + curl -fsS -m 8 "$url" >"$out_file" 2>"$err_file" + else + curl -fsS -m 8 -H 'Content-Type: application/json' -X "$method" -d "$body" "$url" >"$out_file" 2>"$err_file" + fi + LAST_RC=$? + set -e + LAST_STDOUT="$(awk 'NR==1 { print; exit }' "$out_file")" + LAST_STDERR="$(awk 'NR==1 { print; exit }' "$err_file")" + [[ -z "$LAST_STDOUT" ]] && LAST_STDOUT="(none)" + [[ -z "$LAST_STDERR" ]] && LAST_STDERR="(none)" + + rm -f "$out_file" "$err_file" +} + +run_capture() { + local label=$1 + shift + + local out_file + local err_file + out_file="$(mktemp)" + err_file="$(mktemp)" + + LAST_RC=0 + LAST_DECISION="n/a" + LAST_RECEIPT="n/a" + LAST_STDOUT="" + LAST_STDERR="" + + if (( DRY_RUN )); then + LAST_STDERR="DRY-RUN for: $label" + return 0 + fi + + set +e + "$@" >"$out_file" 2>"$err_file" + LAST_RC=$? + set -e + LAST_DECISION="$(extract_from_file "$err_file" "decision" || true)" + LAST_RECEIPT="$(extract_from_file "$err_file" "receipt" || true)" + LAST_STDOUT="$(awk 'NR==1 { print; exit }' "$out_file")" + LAST_STDERR="$(awk 'NR==1 { print; exit }' "$err_file")" + [[ -z "$LAST_STDOUT" ]] && LAST_STDOUT="(none)" + [[ -z "$LAST_STDERR" ]] && LAST_STDERR="(none)" + LAST_STDOUT=${LAST_STDOUT//$'\n'/} + LAST_STDERR=${LAST_STDERR//$'\n'/} + + rm -f "$out_file" "$err_file" +} + +escape_csv() { + local value=$1 + value="${value//\"/\"\"}" + printf '"%s"' "$value" +} + +append_markdown_row() { + printf "| %s | %s | %s | %s | %s | %s | %s | %s |\n" \ + "$1" "$2" "$3" "$4" "$5" "$6" "$7" "$8" >>"$REPORT_MD" +} + +append_csv_row() { + printf "%s,%s,%s,%s,%s,%s,%s,%s\n" \ + "$(escape_csv "$1")" \ + "$(escape_csv "$2")" \ + "$(escape_csv "$3")" \ + "$(escape_csv "$4")" \ + "$(escape_csv "$5")" \ + "$(escape_csv "$6")" \ + "$(escape_csv "$7")" \ + "$(escape_csv "$8")" >>"$REPORT_CSV" +} + +emit_preflight() { + cat >"$REPORT_MD" </dev/null || echo unknown) + +## Preflight + +| Check | Status | Path | +| --- | --- | --- | +EOF + + if [[ -x "$SHIM_BIN" ]]; then + printf "| datafog-shim binary | OK | %s |\n" "$SHIM_BIN" >>"$REPORT_MD" + else + printf "| datafog-shim binary | MISSING | %s |\n" "$SHIM_BIN" >>"$REPORT_MD" + fi + if [[ -x "${SHIM_DIR}/codex" ]]; then + printf "| codex shim | OK | %s |\n" "${SHIM_DIR}/codex" >>"$REPORT_MD" + else + printf "| codex shim | MISSING | %s/codex |\n" "$SHIM_DIR" >>"$REPORT_MD" + fi + if [[ -x "${SHIM_DIR}/claude" ]]; then + printf "| claude shim | OK | %s |\n" "${SHIM_DIR}/claude" >>"$REPORT_MD" + else + printf "| claude shim | MISSING | %s/claude |\n" "$SHIM_DIR" >>"$REPORT_MD" + fi + if [[ -n "${CODEX_BIN}" ]]; then + printf "| codex binary | OK | %s |\n" "$CODEX_BIN" >>"$REPORT_MD" + else + printf "| codex binary | NOT FOUND | codex |\n" >>"$REPORT_MD" + fi + if [[ -n "${CLAUDE_BIN}" ]]; then + printf "| claude binary | OK | %s |\n" "$CLAUDE_BIN" >>"$REPORT_MD" + else + printf "| claude binary | NOT FOUND | claude |\n" >>"$REPORT_MD" + fi + if command -v curl >/dev/null 2>&1; then + if curl -fsS "${POLICY_URL}/health" >/dev/null 2>&1; then + printf "| policy endpoint health | OK | %s/health |\n" "$POLICY_URL" >>"$REPORT_MD" + else + printf "| policy endpoint health | UNREACHABLE | %s/health |\n" "$POLICY_URL" >>"$REPORT_MD" + fi + else + printf "| policy endpoint health | UNKNOWN | curl unavailable |\n" >>"$REPORT_MD" + fi + + printf "\n## Baseline and Runtime Matrix\n\n" >>"$REPORT_MD" + printf "| Cell | Mode | Probe | Command | Exit | Decision | Receipt | Notes |\n" >>"$REPORT_MD" + printf "| --- | --- | --- | --- | --- | --- | --- | --- |\n" >>"$REPORT_MD" + + cat >"$REPORT_CSV" <>"$REPORT_MD" + printf "| --- | --- | --- | --- | --- | --- |\n" >>"$REPORT_MD" + + for s in "${SCENARIOS[@]}"; do + scenario_name="${s%%:*}" + label="$(scenario_label "$scenario_name")" + before_key="$(result_key "$adapter" "without-datafog" "$scenario_name")" + after_key="$(result_key "$adapter" "with-datafog" "$scenario_name")" + before_out="$(result_get "$before_key" "outcome" "UNKNOWN")" + after_out="$(result_get "$after_key" "outcome" "UNKNOWN")" + before_decision="$(result_get "$before_key" "decision" "n/a")" + after_decision="$(result_get "$after_key" "decision" "n/a")" + before_receipt="$(result_get "$before_key" "receipt" "n/a")" + after_receipt="$(result_get "$after_key" "receipt" "n/a")" + printf "| %s | %s | %s | %s | %s | %s |\n" \ + "$label" \ + "$(scenario_risk "$scenario_name")" \ + "$(result_text "$adapter" "without-datafog" "$scenario_name")" \ + "$(result_text "$adapter" "with-datafog" "$scenario_name")" \ + "$(outcome_delta "$before_out" "$after_out")" \ + "$(result_change_impact "$adapter" "$scenario_name" "$before_out" "$after_out" "$before_decision" "$after_decision" "$before_receipt" "$after_receipt")" \ + >>"$REPORT_MD" + done + + printf "\n" >>"$REPORT_MD" +} + +emit_blocked_story() { + local blocked_count=0 + local adapter + local s + local scenario + local key + local outcome + local decision + local receipt + local notes + + printf "\n## Bad actions this run (with-datafog mode)\n\n" >>"$REPORT_MD" + printf "| Adapter | Scenario | Risk | Decision | Receipt | Why it was blocked |\n" >>"$REPORT_MD" + printf "| --- | --- | --- | --- | --- | --- |\n" >>"$REPORT_MD" + + for adapter in codex claude; do + for s in "${SCENARIOS[@]}"; do + scenario="${s%%:*}" + key="$(result_key "$adapter" "with-datafog" "$scenario")" + outcome="$(result_get "$key" "outcome" "UNKNOWN")" + decision="$(result_get "$key" "decision" "n/a")" + receipt="$(result_get "$key" "receipt" "n/a")" + notes="$(result_get "$key" "notes" "n/a")" + if [[ "$outcome" == "BLOCKED" ]]; then + ((blocked_count += 1)) + printf "| %s | %s | %s | %s | %s | %s |\n" \ + "$adapter" \ + "$(scenario_label "$scenario")" \ + "$(scenario_risk "$scenario")" \ + "${decision:-n/a}" \ + "${receipt:-n/a}" \ + "${notes:-n/a}" \ + >>"$REPORT_MD" + fi + done + done + + if (( blocked_count == 0 )); then + printf "No explicit policy blocks recorded.\n\n" >>"$REPORT_MD" + fi +} + +run_control_policy_outage_probe() { + local adapter="codex" + local probe="policy-outage" + local fail_url="http://127.0.0.1:1" + + if [[ ! -x "$SHIM_BIN" ]]; then + record_probe_result \ + "control" \ + "policy" \ + "$probe" \ + "shim run --policy-url $fail_url" \ + "n/a" \ + "n/a" \ + "n/a" \ + "datafog-shim missing" + return + fi + + run_capture "$probe" \ + "$SHIM_BIN" \ + run \ + --adapter "$adapter" \ + --policy-url "$fail_url" \ + --mode "$MODE" \ + --target /bin/echo -- "policy outage control probe" + + record_probe_result \ + "control" \ + "policy" \ + "$probe" \ + "datafog-shim run --policy-url $fail_url --target /bin/echo" \ + "$LAST_RC" \ + "${LAST_DECISION:-n/a}" \ + "${LAST_RECEIPT:-n/a}" \ + "${LAST_STDERR:-n/a}" +} + +run_control_decide_probe() { + local probe="decide-redaction" + local payload='{"action":{"type":"file.write","tool":"fs","resource":"notes.txt"},"text":"Contact alice@example.com for invoice details."}' + + if ! command -v curl >/dev/null 2>&1; then + record_probe_result \ + "control" \ + "policy" \ + "$probe" \ + "POST ${POLICY_URL}/v1/decide" \ + "n/a" \ + "n/a" \ + "n/a" \ + "curl unavailable" + return + fi + + run_http_probe "POST" "${POLICY_URL}/v1/decide" "$payload" + local decision + local matches + local plan + local notes + if (( LAST_RC == 0 )); then + decision="$(json_value "$LAST_STDOUT" '.decision' 'n/a')" + matches="$(json_value "$LAST_STDOUT" '.matched_rules | join(",")' 'n/a')" + plan="$(json_value "$LAST_STDOUT" '.transform_plan | tostring' 'n/a')" + notes="decision=${decision}; matches=${matches}; transform_plan=${plan}" + else + decision="n/a" + notes="curl/endpoint failed: ${LAST_STDERR}" + fi + + record_probe_result \ + "control" \ + "policy" \ + "$probe" \ + "POST ${POLICY_URL}/v1/decide" \ + "$LAST_RC" \ + "$decision" \ + "n/a" \ + "$notes" +} + +run_control_transform_probe() { + local probe="transform-mask" + local payload='{"text":"Please email alice@example.com for invoice details.","mode":"mask"}' + + if ! command -v curl >/dev/null 2>&1; then + record_probe_result \ + "control" \ + "policy" \ + "$probe" \ + "POST ${POLICY_URL}/v1/transform" \ + "n/a" \ + "n/a" \ + "n/a" \ + "curl unavailable" + return + fi + + run_http_probe "POST" "${POLICY_URL}/v1/transform" "$payload" + local output + local count + local modes + local notes + if (( LAST_RC == 0 )); then + output="$(json_value "$LAST_STDOUT" '.output' 'n/a')" + count="$(json_value "$LAST_STDOUT" '.stats.entities_transformed' 'n/a')" + modes="$(json_value "$LAST_STDOUT" '.stats.modes_applied' 'n/a')" + notes="output=${output}; entities_transformed=${count}; modes=${modes}" + else + output="n/a" + notes="curl/endpoint failed: ${LAST_STDERR}" + fi + + record_probe_result \ + "control" \ + "policy" \ + "$probe" \ + "POST ${POLICY_URL}/v1/transform" \ + "$LAST_RC" \ + "transform" \ + "n/a" \ + "$notes" +} + +emit_control_checks() { + local probe_name + local key + local outcome + local decision + local notes + local description + + printf "\n## Policy API control checks\n\n" >>"$REPORT_MD" + printf "| Check | Outcome | Decision | Notes |\n" >>"$REPORT_MD" + printf "| --- | --- | --- | --- |\n" >>"$REPORT_MD" + + for c in "${CONTROL_SCENARIOS[@]}"; do + probe_name="${c%%:*}" + description="${c#*:}" + key="$(result_key "control" "policy" "$probe_name")" + outcome="$(result_get "$key" "outcome" "UNKNOWN")" + decision="$(result_get "$key" "decision" "n/a")" + notes="$(result_get "$key" "notes" "n/a")" + printf "| %s | %s | %s | %s |\n" \ + "$description" \ + "$outcome" \ + "${decision:-n/a}" \ + "${notes:-n/a}" \ + >>"$REPORT_MD" + done + printf "\n" >>"$REPORT_MD" +} + +emit_risk_catalog() { + printf "\n## High-value checks this demo evaluates\n\n" >>"$REPORT_MD" + printf "%s\n" "- Read operation against .env.secret" >>"$REPORT_MD" + printf "%s\n" "- Delete operation against workspace artifact" >>"$REPORT_MD" + printf "%s\n" "- Output write (\`printf 'report=clean' > write.out\`)" >>"$REPORT_MD" + printf "\n" >>"$REPORT_MD" +} + +reset_demo_workspace() { + local dir=$1 + mkdir -p "$dir" + printf 'DATAFOG_FAKE_KEY=ak_test_12345\n' >"${dir}/.env.secret" + printf 'artifact\n' >"${dir}/artifact.txt" + printf 'notes\n' >"${dir}/notes.txt" +} + +run_cli_probe() { + local agent=$1 + local bin=$2 + local mode=$3 + local command_label=$4 + local command_desc=$5 + local command=("${@:6}") + + if [[ -z "$bin" ]]; then + LAST_RC="n/a" + LAST_DECISION="n/a" + LAST_RECEIPT="n/a" + LAST_STDERR="missing binary" + append_markdown_row "$agent" "$mode" "$command_label" "$command_desc" "n/a" "n/a" "n/a" "$LAST_STDERR" + append_csv_row "$agent" "$mode" "$command_label" "$command_desc" "n/a" "n/a" "n/a" "$LAST_STDERR" + record_probe_result "$agent" "$mode" "$command_label" "$command_desc" "$LAST_RC" "$LAST_DECISION" "$LAST_RECEIPT" "$LAST_STDERR" + return + fi + + run_capture "$command_label" "${command[@]}" + append_markdown_row "$agent" "$mode" "$command_label" "$command_desc" "$LAST_RC" "${LAST_DECISION:-n/a}" "${LAST_RECEIPT:-n/a}" "$LAST_STDERR" + append_csv_row "$agent" "$mode" "$command_label" "$command_desc" "$LAST_RC" "${LAST_DECISION:-n/a}" "${LAST_RECEIPT:-n/a}" "$LAST_STDERR" + record_probe_result "$agent" "$mode" "$command_label" "$command_desc" "$LAST_RC" "${LAST_DECISION:-n/a}" "${LAST_RECEIPT:-n/a}" "$LAST_STDERR" +} + +run_datafog_action_probe() { + local adapter=$1 + local mode=$2 + local scenario=$3 + local command_desc=$4 + local target_cmd=$5 + local workspace=${6:-} + + if [[ -z "$workspace" ]]; then + workspace="$WORKDIR/$adapter-$mode-${scenario// /-}" + fi + reset_demo_workspace "$workspace" + if [[ "$mode" == "without-datafog" ]]; then + run_capture "$adapter $mode $scenario" /bin/sh -lc "cd '${workspace}' && ${target_cmd}" + else + if [[ ! -x "$SHIM_BIN" ]]; then + LAST_RC="n/a" + LAST_DECISION="n/a" + LAST_RECEIPT="n/a" + LAST_STDERR="datafog-shim missing" + else + run_capture "$adapter $mode $scenario" \ + "$SHIM_BIN" \ + run \ + --adapter "$adapter" \ + --policy-url "$POLICY_URL" \ + --event-sink "$EVENT_SINK" \ + --mode "$MODE" \ + --target /bin/sh -- -lc "cd '${workspace}' && ${target_cmd}" + fi + fi + append_markdown_row "$adapter" "$mode" "$scenario" "$command_desc" "$LAST_RC" "${LAST_DECISION:-n/a}" "${LAST_RECEIPT:-n/a}" "$LAST_STDERR" + append_csv_row "$adapter" "$mode" "$scenario" "$command_desc" "$LAST_RC" "${LAST_DECISION:-n/a}" "${LAST_RECEIPT:-n/a}" "$LAST_STDERR" + record_probe_result "$adapter" "$mode" "$scenario" "$command_desc" "$LAST_RC" "${LAST_DECISION:-n/a}" "${LAST_RECEIPT:-n/a}" "$LAST_STDERR" +} + +build_command_table() { + local adapter=$1 + local bin=$2 + local no_shim="${SHIM_DIR}/${adapter}" + + # Baseline: native command. + run_cli_probe "$adapter" "$bin" "without-datafog" "cli-help" "bin --help" "$bin" "--help" + + # Datafog wrapper path check. + if (( SKIP_LIVE )); then + LAST_RC="n/a" + LAST_DECISION="n/a" + LAST_RECEIPT="n/a" + LAST_STDERR="skip-live enabled" + record_probe_result "$adapter" "with-datafog" "cli-help" "install + datafog shim required" "$LAST_RC" "$LAST_DECISION" "$LAST_RECEIPT" "$LAST_STDERR" + append_markdown_row "$adapter" "with-datafog" "cli-help" "install + datafog shim required" "n/a" "n/a" "n/a" "$LAST_STDERR" + append_csv_row "$adapter" "with-datafog" "cli-help" "install + datafog shim required" "n/a" "n/a" "n/a" "$LAST_STDERR" + else + if [[ -x "$no_shim" ]]; then + run_cli_probe "$adapter" "$no_shim" "with-datafog" "cli-help" "shim --help (shim install check)" "$no_shim" "--help" + else + LAST_RC="n/a" + LAST_DECISION="n/a" + LAST_RECEIPT="n/a" + LAST_STDERR="shim path missing" + record_probe_result "$adapter" "with-datafog" "cli-help" "install + datafog shim required" "$LAST_RC" "$LAST_DECISION" "$LAST_RECEIPT" "$LAST_STDERR" + append_markdown_row "$adapter" "with-datafog" "cli-help" "install + datafog shim required" "n/a" "n/a" "n/a" "$LAST_STDERR" + append_csv_row "$adapter" "with-datafog" "cli-help" "install + datafog shim required" "n/a" "n/a" "n/a" "$LAST_STDERR" + fi + fi +} + +emit_action_matrix_rows() { + local adapter=$1 + local bin=$2 + local dir="$WORKDIR/$adapter-actions" + + run_datafog_action_probe "$adapter" "without-datafog" "read-secret" "cat .env.secret" "cat .env.secret" "$dir" + run_datafog_action_probe "$adapter" "with-datafog" "read-secret" "cat .env.secret" "cat .env.secret" "$dir" + + run_datafog_action_probe "$adapter" "without-datafog" "write-output" "printf 'report=clean' > write.out" "printf 'report=clean' > write.out" "$dir" + run_datafog_action_probe "$adapter" "with-datafog" "write-output" "printf 'report=clean' > write.out" "printf 'report=clean' > write.out" "$dir" + + run_datafog_action_probe "$adapter" "without-datafog" "delete-artifact" "rm -f artifact.txt" "rm -f artifact.txt" "$dir" + run_datafog_action_probe "$adapter" "with-datafog" "delete-artifact" "rm -f artifact.txt" "rm -f artifact.txt" "$dir" +} + +echo_markdown_summary() { + for adapter in codex claude; do + emit_story_matrix "$adapter" + done + emit_blocked_story + emit_control_checks + emit_risk_catalog + + printf "\n## Interpretation notes\n\n" >>"$REPORT_MD" + printf "%s\n" "- This report is ordered as: baseline before Datafog, then with-datafog enforcement." >>"$REPORT_MD" + printf "%s\n" "- If an action appears in \"Bad actions this run,\" Datafog blocked or transformed it with a decision." >>"$REPORT_MD" + printf "%s\n" "- \"High-value checks\" are the explicit operations this harness validates for risky behavior." >>"$REPORT_MD" + printf "%s\n\n" "- If paths are \`n/a\`, install wrappers or point \`--shim-bin\` at a built \`datafog-shim\` binary." >>"$REPORT_MD" + printf "## Next run\n\n" >>"$REPORT_MD" + printf "%s\n" "- Keep the same report template and run with \`--mode observe\` first, then switch to \`--mode enforced\` after policy tuning." >>"$REPORT_MD" + printf "%s\n" "- Use \`export PATH=\"${SHIM_DIR}:\$PATH\"\` after setup to ensure managed shims are hit for real agent traffic." >>"$REPORT_MD" +} + +emit_preflight + +for adapter in codex claude; do + if [[ "$adapter" == "codex" ]]; then + bin="$CODEX_BIN" + else + bin="$CLAUDE_BIN" + fi + build_command_table "$adapter" "$bin" +done + +if (( SKIP_LIVE )); then + cat < /tmp/datafog_demo_report_notice.txt +Datafog demo report generated in preflight-only mode. +EOF +else + for adapter in codex claude; do + emit_action_matrix_rows "$adapter" "" + done +fi + +run_control_policy_outage_probe +run_control_decide_probe +run_control_transform_probe + +echo_markdown_summary + +printf "\nReport generated:\n%s\n" "$REPORT_MD" +printf "\nCSV generated:\n%s\n" "$REPORT_CSV" diff --git a/scripts/runbooks/select-runbooks.sh b/scripts/runbooks/select-runbooks.sh new file mode 100755 index 0000000..e393e22 --- /dev/null +++ b/scripts/runbooks/select-runbooks.sh @@ -0,0 +1,154 @@ +#!/bin/bash +set -euo pipefail + +# Select runbooks whose called_from frontmatter matches a skill or step name. +# Prints matching runbook paths (relative to repo root) to stdout. + +# --- Repo root: two parents up from this script's directory --- +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" + +# --- CLI argument parsing --- +SKILL="" +STEP="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --skill) + SKILL="$2" + shift 2 + ;; + --step) + STEP="$2" + shift 2 + ;; + *) + echo "Usage: $0 --skill [--step ]" >&2 + exit 1 + ;; + esac +done + +if [[ -z "$SKILL" ]]; then + echo "Error: --skill is required" >&2 + exit 1 +fi + +# --- Main logic --- +RUNBOOKS_DIR="$REPO_ROOT/docs/runbooks" + +if [[ ! -d "$RUNBOOKS_DIR" ]]; then + exit 0 +fi + +# Extract the frontmatter block (between first --- and next ---). +# Parse called_from entries. Print the file path if skill or step matches. +process_file() { + local file="$1" + local in_frontmatter=0 + local in_called_from=0 + local first_line=1 + local called_from_items=() + + while IFS= read -r line || [[ -n "$line" ]]; do + local trimmed + trimmed="$(echo "$line" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + + # First non-empty consideration: frontmatter must start at line 1 with --- + if [[ "$first_line" -eq 1 ]]; then + first_line=0 + if [[ "$trimmed" == "---" ]]; then + in_frontmatter=1 + continue + else + # No frontmatter + return + fi + fi + + # Inside frontmatter + if [[ "$in_frontmatter" -eq 1 ]]; then + # Closing delimiter + if [[ "$trimmed" == "---" ]]; then + break + fi + + # Skip empty lines and comments + if [[ -z "$trimmed" || "$trimmed" == \#* ]]; then + # Empty lines inside a YAML list block: keep scanning + if [[ "$in_called_from" -eq 1 && -z "$trimmed" ]]; then + continue + fi + continue + fi + + # If we're collecting YAML list items for called_from + if [[ "$in_called_from" -eq 1 ]]; then + # Check if this is a list item (starts with -) + if [[ "$trimmed" == -* ]]; then + local item + item="$(echo "$trimmed" | sed "s/^-[[:space:]]*//;s/^[\"']//;s/[\"']$//")" + if [[ -n "$item" ]]; then + called_from_items+=("$item") + fi + continue + else + # Not a list item; if it contains a colon it's a new key — stop collecting + if echo "$trimmed" | grep -q ':'; then + in_called_from=0 + # Fall through to process this line as a new key + else + continue + fi + fi + fi + + # Check for key: value lines + if echo "$trimmed" | grep -q ':'; then + local key val + key="$(echo "$trimmed" | sed 's/^\([^:]*\):.*/\1/' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + val="$(echo "$trimmed" | sed 's/^[^:]*://' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + + if [[ "$key" == "called_from" ]]; then + # Inline list: called_from: [a, b] + if [[ "$val" == \[* ]]; then + local inner + inner="$(echo "$val" | sed 's/^\[//;s/\]$//' | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [[ -n "$inner" ]]; then + IFS=',' read -ra parts <<< "$inner" + for part in "${parts[@]}"; do + local cleaned + cleaned="$(echo "$part" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" + if [[ -n "$cleaned" ]]; then + called_from_items+=("$cleaned") + fi + done + fi + else + # YAML list form — start collecting on subsequent lines + in_called_from=1 + fi + fi + fi + fi + done < "$file" + + # Check for matches + for item in "${called_from_items[@]+"${called_from_items[@]}"}"; do + if [[ "$item" == "$SKILL" ]]; then + echo "${file#"$REPO_ROOT"/}" + return + fi + if [[ -n "$STEP" && "$item" == "$STEP" ]]; then + echo "${file#"$REPO_ROOT"/}" + return + fi + done +} + +# Find all .md files, sorted for deterministic output +while IFS= read -r mdfile; do + process_file "$mdfile" +done < <(find "$RUNBOOKS_DIR" -name '*.md' -type f | sort) + +exit 0