From 46635e7f042b15370da806f1b2e0854262d40c65 Mon Sep 17 00:00:00 2001 From: Bryan Frimin Date: Mon, 16 Mar 2026 23:19:28 +0100 Subject: [PATCH] Add Lima sandbox environment for parallel feature testing Implement a complete sandbox system for testing multiple features in parallel using git worktrees and Lima VMs. Each worktree gets its own isolated VM with Docker, full service stack, and unique IP via vzNAT networking. - contrib/lima/provision.sh: Idempotent provisioning script (Docker, Go 1.26.1, Node.js 24, npm 11.8.0, Go tools, mkcert) - contrib/lima/probo.yaml: Lima VM template with vz vmType, Rosetta, vzNAT, virtiofs mount - contrib/lima/sandbox.sh: Lifecycle CLI (create, start, stop, restart, delete, ssh, exec, status, list) - contrib/lima/README.md: Human documentation with prerequisites, quickstart, troubleshooting - contrib/claude/sandbox.md: Agent reference doc for sandbox usage patterns - GNUmakefile: Convenience targets for sandbox.sh commands - AGENTS.md: Updated reference documentation index Signed-off-by: Bryan Frimin --- AGENTS.md | 1 + GNUmakefile | 24 ++++++ contrib/claude/sandbox.md | 73 ++++++++++++++++ contrib/lima/README.md | 88 +++++++++++++++++++ contrib/lima/probo.yaml | 33 +++++++ contrib/lima/provision.sh | 86 +++++++++++++++++++ contrib/lima/sandbox.sh | 176 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 481 insertions(+) create mode 100644 contrib/claude/sandbox.md create mode 100644 contrib/lima/README.md create mode 100644 contrib/lima/probo.yaml create mode 100755 contrib/lima/provision.sh create mode 100755 contrib/lima/sandbox.sh diff --git a/AGENTS.md b/AGENTS.md index fde2924fc..3baffb635 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -33,6 +33,7 @@ Detailed guides for specific subsystems live in `contrib/claude/`: - [`contrib/claude/go-testing.md`](contrib/claude/go-testing.md) — Go test conventions (parallel, require vs assert, naming) - [`contrib/claude/go-worker.md`](contrib/claude/go-worker.md) — Go worker pattern (poll-based, bounded concurrency, FOR UPDATE SKIP LOCKED) - [`contrib/claude/go-service.md`](contrib/claude/go-service.md) — Go service orchestration (Run, graceful shutdown, crash propagation) +- [`contrib/claude/sandbox.md`](contrib/claude/sandbox.md) — Lima sandbox environments (create, manage, access services) ## API Surface Rules diff --git a/GNUmakefile b/GNUmakefile index ecc1cf9a4..c4edf6465 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -311,6 +311,30 @@ apps/console/dist/index.html apps/trust/dist/index.html: $(ECHO) dev-server > $@ +.PHONY: sandbox-create +sandbox-create: ## Create a Lima sandbox VM for this worktree + ./contrib/lima/sandbox.sh create + +.PHONY: sandbox-start +sandbox-start: ## Start the Lima sandbox VM + ./contrib/lima/sandbox.sh start + +.PHONY: sandbox-stop +sandbox-stop: ## Stop (hibernate) the Lima sandbox VM + ./contrib/lima/sandbox.sh stop + +.PHONY: sandbox-delete +sandbox-delete: ## Delete the Lima sandbox VM + ./contrib/lima/sandbox.sh delete + +.PHONY: sandbox-ssh +sandbox-ssh: ## Open a shell in the Lima sandbox VM + ./contrib/lima/sandbox.sh ssh + +.PHONY: sandbox-status +sandbox-status: ## Show Lima sandbox VM status and IP + ./contrib/lima/sandbox.sh status + .PHONY: deadcode deadcode: $(GO_TOOL) deadcode ./... | grep -v "With" | grep -v "UnmarshalBigIntScalar" | grep -v "^e2e/" \ No newline at end of file diff --git a/contrib/claude/sandbox.md b/contrib/claude/sandbox.md new file mode 100644 index 000000000..7fdafc524 --- /dev/null +++ b/contrib/claude/sandbox.md @@ -0,0 +1,73 @@ +# Sandbox Environments + +## When to use + +Use a sandbox when you need to: +- Run `make stack-up` (Docker services: Postgres, SeaweedFS, etc.) +- Test changes end-to-end with `make dev` or `make test-e2e` +- Build the full binary with `make build` +- Run any command that requires Docker or the full service stack + +## Quick reference + +```bash +# Create a sandbox (first time only) +./contrib/lima/sandbox.sh create + +# Start an existing sandbox +./contrib/lima/sandbox.sh start + +# Run commands inside the sandbox +./contrib/lima/sandbox.sh exec -- make stack-up +./contrib/lima/sandbox.sh exec -- make build +./contrib/lima/sandbox.sh exec -- make dev +./contrib/lima/sandbox.sh exec -- make test + +# Get the VM IP and service URLs +./contrib/lima/sandbox.sh status + +# Interactive shell +./contrib/lima/sandbox.sh ssh + +# Stop (shutdown — preserves disk and Docker images, but running processes are lost) +./contrib/lima/sandbox.sh stop + +# Delete entirely +./contrib/lima/sandbox.sh delete +``` + +## Accessing services + +After `sandbox.sh status`, use the VM IP to access services from the host: + +| Service | URL | +|---|---| +| Console | `http://:5173` | +| API | `http://:8080` | +| Grafana | `http://:3001` | +| Mailpit | `http://:8025` | +| Keycloak | `http://:8082` | +| PostgreSQL | `psql -h -U probod` | + +## Common workflows + +**Build and test:** +```bash +./contrib/lima/sandbox.sh exec -- make stack-up +./contrib/lima/sandbox.sh exec -- make build +./contrib/lima/sandbox.sh exec -- make test +``` + +**Run e2e tests:** +```bash +./contrib/lima/sandbox.sh exec -- make stack-up +./contrib/lima/sandbox.sh exec -- make test-e2e +``` + +**Restart after code changes:** +Code changes are reflected immediately (virtiofs mount). Just re-run the +relevant make target — no need to restart the VM. + +```bash +./contrib/lima/sandbox.sh exec -- make dev +``` diff --git a/contrib/lima/README.md b/contrib/lima/README.md new file mode 100644 index 000000000..06ddd678f --- /dev/null +++ b/contrib/lima/README.md @@ -0,0 +1,88 @@ +# Lima Sandbox Environments + +Run isolated dev environments using [Lima](https://lima-vm.io/) VMs. Each git +worktree gets its own VM with Docker, standard ports, and a unique IP — no port +conflicts between sandboxes. + +## Prerequisites + +```bash +brew install lima jq +``` + +## Quickstart + +```bash +# Create and start a sandbox for the current worktree +./contrib/lima/sandbox.sh create + +# Check status and get the VM IP +./contrib/lima/sandbox.sh status + +# Start the Docker stack inside the VM +./contrib/lima/sandbox.sh exec -- make stack-up + +# Build and run the dev server +./contrib/lima/sandbox.sh exec -- make build +./contrib/lima/sandbox.sh exec -- make dev + +# Access services from your host browser using the VM IP +# e.g. http://192.168.105.2:5173 +``` + +## Commands + +| Command | Description | +|---|---| +| `./contrib/lima/sandbox.sh create [--cpus C] [--memory M] [--disk D]` | Create and start a new VM | +| `./contrib/lima/sandbox.sh start` | Start a stopped VM | +| `./contrib/lima/sandbox.sh stop` | Shut down the VM (preserves disk and Docker images) | +| `./contrib/lima/sandbox.sh restart` | Stop + start | +| `./contrib/lima/sandbox.sh delete` | Remove the VM entirely | +| `./contrib/lima/sandbox.sh ssh` | Interactive shell at `/workspace` | +| `./contrib/lima/sandbox.sh exec -- CMD` | Run a command in the VM | +| `./contrib/lima/sandbox.sh status` | Show VM state, IP, and service URLs | +| `./contrib/lima/sandbox.sh list` | List all `probo-*` VMs | + +## Architecture + +``` +Host (macOS) +├── worktree: ~/Developer/probo/delhi → VM "probo-delhi" (192.168.105.x) +├── worktree: ~/Developer/probo/feature-a → VM "probo-feature-a" (192.168.105.y) +└── worktree: ~/Developer/probo/feature-b → VM "probo-feature-b" (192.168.105.z) +``` + +Each VM: +- Mounts the worktree at `/workspace` via virtiofs (read-write) +- Runs Docker + docker-compose inside the VM +- Forwards the host SSH agent +- Gets its own IP via vzNAT — all services on standard ports + +## Makefile targets + +Convenience wrappers are available: + +```bash +make sandbox-create +make sandbox-start +make sandbox-stop +make sandbox-delete +make sandbox-ssh +make sandbox-status +``` + +## Troubleshooting + +**VM won't start**: Check `limactl list` for stale entries. Delete with +`./contrib/lima/sandbox.sh delete` and recreate. + +**Slow file I/O**: The worktree is mounted via virtiofs which is fast for most +operations. If `node_modules` is slow, consider symlinking it to VM-local disk. + +**Docker permission denied**: The provision script adds the lima user to the +docker group. If you see permission errors, restart the VM with +`./contrib/lima/sandbox.sh restart`. + +**Can't reach VM IP from host**: Ensure vzNAT networking is working. Run +`./contrib/lima/sandbox.sh status` to verify the IP is assigned. diff --git a/contrib/lima/probo.yaml b/contrib/lima/probo.yaml new file mode 100644 index 000000000..769dde5ce --- /dev/null +++ b/contrib/lima/probo.yaml @@ -0,0 +1,33 @@ +# Copyright (c) 2025, 2026 Probo Inc. +# SPDX-License-Identifier: ISC + +images: + - location: "https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img" + arch: "x86_64" + - location: "https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-arm64.img" + arch: "aarch64" + +vmType: vz +vmOpts: + vz: + rosetta: + enabled: true + binfmt: true + +cpus: 4 +memory: 8GiB +disk: 50GiB + +networks: + - vzNAT: true + +mounts: [] + +ssh: + forwardAgent: true + +provision: + - mode: system + script: | + #!/bin/bash + /workspace/contrib/lima/provision.sh diff --git a/contrib/lima/provision.sh b/contrib/lima/provision.sh new file mode 100755 index 000000000..832c1024c --- /dev/null +++ b/contrib/lima/provision.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# Copyright (c) 2025, 2026 Probo Inc. +# SPDX-License-Identifier: ISC + +set -euo pipefail + +export DEBIAN_FRONTEND=noninteractive + +GO_VERSION="1.26.1" +NODE_MAJOR=24 +NPM_VERSION="11.8.0" + +GOTESTSUM_VERSION="v1.13.0" +GOLANGCI_LINT_VERSION="v2.11.3" +GOW_VERSION="v0.0.0-20260225145757-ff0f6779ab4c" +MKCERT_VERSION="v1.4.4" + +apt-get update -qq +apt-get install -y -qq \ + build-essential \ + git \ + curl \ + jq \ + parallel \ + ca-certificates \ + gnupg \ + lsb-release \ + unzip + +if ! command -v docker &>/dev/null; then + install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg \ + | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + chmod a+r /etc/apt/keyrings/docker.gpg + + echo \ + "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] \ + https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ + | tee /etc/apt/sources.list.d/docker.list > /dev/null + + apt-get update -qq + apt-get install -y -qq \ + docker-ce \ + docker-ce-cli \ + containerd.io \ + docker-buildx-plugin \ + docker-compose-plugin +fi + +usermod -aG docker "${LIMA_CIDATA_USER:-lima}" 2>/dev/null || true + +if [ ! -d "/usr/local/go" ] || ! /usr/local/go/bin/go version | grep -q "go${GO_VERSION}"; then + rm -rf /usr/local/go + ARCH=$(dpkg --print-architecture) + curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${ARCH}.tar.gz" \ + | tar -C /usr/local -xzf - +fi + +cat > /etc/profile.d/go.sh << 'GOEOF' +export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH" +GOEOF +chmod +x /etc/profile.d/go.sh + +export PATH="/usr/local/go/bin:$PATH" + +GOBIN=/usr/local/bin /usr/local/go/bin/go install "gotest.tools/gotestsum@${GOTESTSUM_VERSION}" +GOBIN=/usr/local/bin /usr/local/go/bin/go install "github.com/golangci/golangci-lint/v2/cmd/golangci-lint@${GOLANGCI_LINT_VERSION}" +GOBIN=/usr/local/bin /usr/local/go/bin/go install "github.com/mitranim/gow@${GOW_VERSION}" + +if ! command -v node &>/dev/null || ! node --version | grep -q "v${NODE_MAJOR}"; then + curl -fsSL "https://deb.nodesource.com/setup_${NODE_MAJOR}.x" | bash - + apt-get install -y -qq nodejs +fi + +npm install -g "npm@${NPM_VERSION}" + +if ! command -v mkcert &>/dev/null; then + GOBIN=/usr/local/bin /usr/local/go/bin/go install "filippo.io/mkcert@${MKCERT_VERSION}" +fi +mkcert -install 2>/dev/null || true + +LIMA_USER="${LIMA_CIDATA_USER:-lima}" +LIMA_HOME=$(eval echo "~${LIMA_USER}") +mkdir -p /root/.parallel "${LIMA_HOME}/.parallel" +touch /root/.parallel/will-cite "${LIMA_HOME}/.parallel/will-cite" +chown -R "${LIMA_USER}:${LIMA_USER}" "${LIMA_HOME}/.parallel" diff --git a/contrib/lima/sandbox.sh b/contrib/lima/sandbox.sh new file mode 100755 index 000000000..1c7fd2e04 --- /dev/null +++ b/contrib/lima/sandbox.sh @@ -0,0 +1,176 @@ +#!/bin/bash +# Copyright (c) 2025, 2026 Probo Inc. +# SPDX-License-Identifier: ISC + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +TEMPLATE="${SCRIPT_DIR}/probo.yaml" + +REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" +WORKTREE_NAME="$(basename "${REPO_ROOT}")" +VM_NAME="probo-${WORKTREE_NAME}" + +usage() { + cat < [options] + +Commands: + create [--cpus C] [--memory M] [--disk D] Create a new sandbox VM + start Start a stopped sandbox + stop Stop the sandbox + restart Stop + start the sandbox + delete Delete the sandbox entirely + ssh Open interactive shell in /workspace + exec -- CMD Run a command in the sandbox + status Show sandbox state, IP, and services + list List all probo-* VMs + +VM name: ${VM_NAME} (derived from worktree directory) +EOF + exit 1 +} + +get_vm_ip() { + limactl shell "${VM_NAME}" ip -4 -j addr show dev lima0 2>/dev/null \ + | jq -r '.[0].addr_info[0].local // empty' 2>/dev/null || true +} + +get_vm_status() { + local status + status=$(limactl list --json 2>/dev/null \ + | jq -r "select(.name == \"${VM_NAME}\") | .status" 2>/dev/null) || true + echo "${status:-NotFound}" +} + +cmd_create() { + local cpus="" memory="" disk="" + + while [[ $# -gt 0 ]]; do + case "$1" in + --cpus) cpus="$2"; shift 2 ;; + --memory) memory="$2"; shift 2 ;; + --disk) disk="$2"; shift 2 ;; + *) echo "Unknown option: $1"; usage ;; + esac + done + + echo "Creating sandbox: ${VM_NAME}" + echo "Worktree: ${REPO_ROOT}" + + local -a create_args=( + --name "${VM_NAME}" + --tty=false + --set ".mounts = [{\"location\": \"${REPO_ROOT}\", \"mountPoint\": \"/workspace\", \"writable\": true}]" + --mount-type virtiofs + ) + + if [[ -n "${cpus}" ]]; then + create_args+=(--cpus "${cpus}") + fi + if [[ -n "${memory}" ]]; then + create_args+=(--memory "${memory}") + fi + if [[ -n "${disk}" ]]; then + create_args+=(--disk "${disk}") + fi + + limactl create "${create_args[@]}" "${TEMPLATE}" + limactl start "${VM_NAME}" + + echo "" + cmd_status +} + +cmd_start() { + echo "Starting sandbox: ${VM_NAME}" + limactl start "${VM_NAME}" + echo "" + cmd_status +} + +cmd_stop() { + echo "Stopping sandbox: ${VM_NAME}" + limactl stop "${VM_NAME}" + echo "Sandbox stopped." +} + +cmd_restart() { + cmd_stop + echo "" + cmd_start +} + +cmd_delete() { + echo "Deleting sandbox: ${VM_NAME}" + limactl delete --force "${VM_NAME}" + echo "Sandbox deleted." +} + +cmd_ssh() { + exec limactl shell --workdir /workspace "${VM_NAME}" +} + +cmd_exec() { + limactl shell --workdir /workspace "${VM_NAME}" "$@" +} + +cmd_status() { + local status ip + status="$(get_vm_status)" + ip="$(get_vm_ip)" + + echo "Sandbox: ${VM_NAME}" + echo "State: ${status}" + echo "IP: ${ip:-"-"}" + + if [[ "${status}" == "Running" && -n "${ip}" ]]; then + echo "" + echo "Services (use VM IP to access from host):" + echo " Console: http://${ip}:5173" + echo " API: http://${ip}:8080" + echo " Grafana: http://${ip}:3001" + echo " Mailpit: http://${ip}:8025" + echo " Keycloak: http://${ip}:8082" + echo " PostgreSQL: psql -h ${ip} -U probod" + fi +} + +cmd_list() { + printf "%-25s %-12s %s\n" "NAME" "STATE" "IP" + + limactl list --json 2>/dev/null | jq -r ' + select(.name | startswith("probo-")) | + [.name, .status] | @tsv + ' | while IFS=$'\t' read -r name status; do + local ip="-" + if [[ "${status}" == "Running" ]]; then + ip=$(limactl shell "${name}" ip -4 -j addr show dev lima0 2>/dev/null \ + | jq -r '.[0].addr_info[0].local // "-"' 2>/dev/null || echo "-") + fi + printf "%-25s %-12s %s\n" "${name}" "${status}" "${ip}" + done +} + +if [[ $# -lt 1 ]]; then + usage +fi + +command="$1" +shift + +case "${command}" in + create) cmd_create "$@" ;; + start) cmd_start ;; + stop) cmd_stop ;; + restart) cmd_restart ;; + delete) cmd_delete ;; + ssh) cmd_ssh ;; + exec) + if [[ "${1:-}" == "--" ]]; then shift; fi + cmd_exec "$@" + ;; + status) cmd_status ;; + list) cmd_list ;; + *) echo "Unknown command: ${command}"; usage ;; +esac