CLean up site generation
This commit is contained in:
James Thompson
Binary file not shown.
197
scripts/check_env_sync.py
Normal file
197
scripts/check_env_sync.py
Normal file
@@ -0,0 +1,197 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Sync .env.example: keep structure/comments, merge in keys from .env."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
SKIP_DIRS = {".git", "site", "__pycache__"}
|
||||
|
||||
|
||||
class EnvParseError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def discover_pairs(repo_root: Path) -> list[tuple[Path, Path]]:
|
||||
pairs: list[tuple[Path, Path]] = []
|
||||
for example_path in sorted(repo_root.rglob(".env.example")):
|
||||
rel_parts = set(example_path.relative_to(repo_root).parts)
|
||||
if rel_parts & SKIP_DIRS:
|
||||
continue
|
||||
env_path = example_path.with_name(".env")
|
||||
pairs.append((example_path, env_path))
|
||||
return pairs
|
||||
|
||||
|
||||
def parse_env_key_values(path: Path) -> tuple[list[str], dict[str, str]]:
|
||||
keys: list[str] = []
|
||||
values: dict[str, str] = {}
|
||||
seen: set[str] = set()
|
||||
|
||||
for idx, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
|
||||
if "=" not in line:
|
||||
raise EnvParseError(f"{path}: line {idx} is not KEY=VALUE format")
|
||||
|
||||
key, value = line.split("=", 1)
|
||||
key = key.strip()
|
||||
if not key:
|
||||
raise EnvParseError(f"{path}: line {idx} has empty key")
|
||||
|
||||
if key in seen:
|
||||
raise EnvParseError(f"{path}: duplicate key '{key}'")
|
||||
seen.add(key)
|
||||
keys.append(key)
|
||||
values[key] = value
|
||||
|
||||
return keys, values
|
||||
|
||||
|
||||
def parse_env_keys(path: Path) -> list[str]:
|
||||
keys, _ = parse_env_key_values(path)
|
||||
return keys
|
||||
|
||||
|
||||
def merge_env_into_example(env_path: Path, example_values: dict[str, str]) -> str:
|
||||
"""
|
||||
Build .env.example from .env structure.
|
||||
.env defines: keys, order, comments.
|
||||
.env.example provides: default values (or placeholders for new keys).
|
||||
"""
|
||||
out_lines: list[str] = []
|
||||
|
||||
# Read .env to get structure/order/comments, fill in values from .env.example
|
||||
for raw_line in env_path.read_text(encoding="utf-8").splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or "=" not in raw_line:
|
||||
out_lines.append(raw_line)
|
||||
continue
|
||||
|
||||
key, value_from_env = raw_line.split("=", 1)
|
||||
key = key.strip()
|
||||
|
||||
# Use default from .env.example if it exists, otherwise use placeholder
|
||||
if key in example_values:
|
||||
out_lines.append(f"{key}={example_values[key]}")
|
||||
else:
|
||||
# New key not in template: use placeholder
|
||||
out_lines.append(f"{key}=__CHANGEME__")
|
||||
|
||||
return "\n".join(out_lines) + "\n"
|
||||
|
||||
|
||||
def compare_env(example_path: Path, env_path: Path) -> list[str]:
|
||||
"""Check if .env.example matches .env schema."""
|
||||
errors: list[str] = []
|
||||
|
||||
if not env_path.exists():
|
||||
rel_env = env_path.relative_to(REPO_ROOT)
|
||||
errors.append(f"Missing .env (schema): {rel_env}")
|
||||
return errors
|
||||
|
||||
try:
|
||||
env_keys, _ = parse_env_key_values(env_path)
|
||||
example_keys = parse_env_keys(example_path) if example_path.exists() else []
|
||||
except EnvParseError as exc:
|
||||
errors.append(str(exc))
|
||||
return errors
|
||||
|
||||
example_set = set(example_keys)
|
||||
env_set = set(env_keys)
|
||||
|
||||
missing_in_example = [k for k in env_keys if k not in example_set]
|
||||
|
||||
if missing_in_example:
|
||||
rel_example = example_path.relative_to(REPO_ROOT)
|
||||
errors.append(f"{rel_example}: missing keys from .env schema: {', '.join(missing_in_example)}")
|
||||
|
||||
return errors
|
||||
|
||||
|
||||
def fix_env_sync(env_path: Path, example_path: Path) -> tuple[bool, str | None]:
|
||||
"""Update .env.example: .env defines schema, .env.example provides defaults."""
|
||||
rel_env = env_path.relative_to(REPO_ROOT)
|
||||
rel_example = example_path.relative_to(REPO_ROOT)
|
||||
|
||||
if not env_path.exists():
|
||||
return False, f"Missing .env file (schema source): {rel_env}"
|
||||
|
||||
try:
|
||||
_, example_values = parse_env_key_values(example_path) if example_path.exists() else ([], {})
|
||||
desired = merge_env_into_example(env_path, example_values)
|
||||
except EnvParseError as exc:
|
||||
return False, str(exc)
|
||||
|
||||
current = example_path.read_text(encoding="utf-8") if example_path.exists() else None
|
||||
if current == desired:
|
||||
return False, None
|
||||
|
||||
example_path.write_text(desired, encoding="utf-8")
|
||||
if current is None:
|
||||
return True, f"Created {rel_example} from {rel_env} schema"
|
||||
return True, f"Synced {rel_example} to match {rel_env}"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--fix",
|
||||
action="store_true",
|
||||
help="Update .env.example from .env schema, keeping default values",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
pairs = discover_pairs(REPO_ROOT)
|
||||
if not pairs:
|
||||
print("No .env.example files found.")
|
||||
return 0
|
||||
|
||||
if args.fix:
|
||||
changed: list[str] = []
|
||||
errors: list[str] = []
|
||||
|
||||
for example_path, env_path in pairs:
|
||||
did_change, message = fix_env_sync(env_path, example_path)
|
||||
if message:
|
||||
if did_change:
|
||||
changed.append(message)
|
||||
else:
|
||||
errors.append(message)
|
||||
|
||||
if changed:
|
||||
print("Synced .env.example from .env schema:\n")
|
||||
for item in changed:
|
||||
print(f"- {item}")
|
||||
print("\nRe-stage changed .env.example files and commit again.")
|
||||
return 1
|
||||
|
||||
if errors:
|
||||
print("Environment file sync failed:\n")
|
||||
for err in errors:
|
||||
print(f"- {err}")
|
||||
return 1
|
||||
|
||||
print(f"Environment files in sync ({len(pairs)} checked).")
|
||||
return 0
|
||||
|
||||
all_errors: list[str] = []
|
||||
for example_path, env_path in pairs:
|
||||
all_errors.extend(compare_env(example_path, env_path))
|
||||
|
||||
if all_errors:
|
||||
print("Environment file validation failed:\n")
|
||||
for err in all_errors:
|
||||
print(f"- {err}")
|
||||
return 1
|
||||
|
||||
print(f"Environment files look good ({len(pairs)} checked).")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -12,6 +12,11 @@ REPO_ROOT = Path(__file__).parent.parent
|
||||
SERVICES_DIR = REPO_ROOT
|
||||
DOCS_DIR = REPO_ROOT / "docs" / "services"
|
||||
|
||||
DISPLAY_NAME_OVERRIDES = {
|
||||
"qbittorrent": "qBittorrent",
|
||||
"n8n": "N8N",
|
||||
}
|
||||
|
||||
# Load root .env.example for shared variables
|
||||
def load_root_env_example():
|
||||
env_vars = {}
|
||||
@@ -168,264 +173,108 @@ def format_security(sec):
|
||||
return []
|
||||
|
||||
|
||||
def generate_service_md(service_name, compose_data, root_env, service_env):
|
||||
def generate_service_md(service_name, service_dir):
|
||||
"""Generate service doc that includes docker-compose.yml, Dockerfile, README, and .env.example."""
|
||||
lines = []
|
||||
lines.append(f"# {service_name}")
|
||||
lines.append("")
|
||||
lines.append(f"> Auto-generated from `docker-compose.yml`")
|
||||
lines.append("")
|
||||
|
||||
services = compose_data.get("services", {})
|
||||
if not services:
|
||||
lines.append("No services defined.")
|
||||
return "\n".join(lines)
|
||||
# Include README.md first if exists
|
||||
readme_path = service_dir / "README.md"
|
||||
if readme_path.exists():
|
||||
lines.append(f'--8<-- "{service_dir.name}/README.md"')
|
||||
|
||||
for svc_name, svc in services.items():
|
||||
if len(services) > 1:
|
||||
lines.append(f"## `{svc_name}`")
|
||||
lines.append("")
|
||||
|
||||
# Image
|
||||
image = svc.get("image", "N/A")
|
||||
container = svc.get("container_name", svc_name)
|
||||
lines.append(f"| Field | Value |")
|
||||
lines.append(f"|-------|-------|")
|
||||
lines.append(f"| **Image** | `{image}` |")
|
||||
lines.append(f"| **Container** | `{container}` |")
|
||||
lines.append(f"| **Restart** | `{svc.get('restart', 'no')}` |")
|
||||
lines.append("> Below are the configuration files for this service. For details on how to deploy or customize, refer to the README above or the official documentation for the service.")
|
||||
# Include docker-compose.yml
|
||||
compose_file = service_dir / "docker-compose.yml"
|
||||
if compose_file.exists():
|
||||
lines.append("## Docker Compose Configuration")
|
||||
lines.append("")
|
||||
lines.append("```yaml")
|
||||
lines.append(f'--8<-- "{service_dir.name}/docker-compose.yml"')
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
|
||||
# Command
|
||||
cmd = format_command(svc.get("command"))
|
||||
if cmd:
|
||||
lines.append("### Command")
|
||||
lines.append("")
|
||||
lines.append("```")
|
||||
lines.append(" ".join(cmd))
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
# Include Dockerfile if exists
|
||||
dockerfile_path = service_dir / "Dockerfile"
|
||||
if dockerfile_path.exists():
|
||||
lines.append("## Dockerfile")
|
||||
lines.append("")
|
||||
lines.append("```dockerfile")
|
||||
lines.append(f'--8<-- "{service_dir.name}/Dockerfile"')
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
|
||||
# Environment
|
||||
env = format_env(svc.get("environment"))
|
||||
env_file = svc.get("env_file")
|
||||
env_from_file = []
|
||||
if env_file:
|
||||
if isinstance(env_file, str):
|
||||
env_file = [env_file]
|
||||
env_from_file = env_file
|
||||
|
||||
if env or env_from_file:
|
||||
lines.append("### Environment")
|
||||
lines.append("")
|
||||
if env_from_file:
|
||||
lines.append("**Env files:** " + ", ".join(f"`{e}`" for e in env_from_file))
|
||||
lines.append("")
|
||||
if env:
|
||||
lines.append("| Variable | Value |")
|
||||
lines.append("|----------|-------|")
|
||||
for key, val in env:
|
||||
if "${" in val or val == "":
|
||||
lines.append(f"| `{key}` | `{val}` |")
|
||||
else:
|
||||
lines.append(f"| `{key}` | `{val}` |")
|
||||
lines.append("")
|
||||
|
||||
# Service secrets from .env.example
|
||||
if service_env:
|
||||
lines.append("### Secrets (from `.env.example`)")
|
||||
lines.append("")
|
||||
lines.append("These variables must be set in the service's `.env` file:")
|
||||
lines.append("")
|
||||
lines.append("| Variable | Default |")
|
||||
lines.append("|----------|---------|")
|
||||
for key, val in service_env.items():
|
||||
lines.append(f"| `{key}` | `{val}` |")
|
||||
lines.append("")
|
||||
|
||||
# Ports
|
||||
ports = format_ports(svc.get("ports"))
|
||||
if ports:
|
||||
lines.append("### Ports")
|
||||
lines.append("")
|
||||
lines.append("| Host | Container | Protocol |")
|
||||
lines.append("|------|-----------|----------|")
|
||||
for host, container, proto in ports:
|
||||
lines.append(f"| `{host}` | `{container}` | {proto} |")
|
||||
lines.append("")
|
||||
|
||||
# Volumes
|
||||
volumes = format_volumes(svc.get("volumes"))
|
||||
if volumes:
|
||||
lines.append("### Volumes")
|
||||
lines.append("")
|
||||
lines.append("| Host Path | Container Path | Mode |")
|
||||
lines.append("|-----------|----------------|------|")
|
||||
for host, container, mode in volumes:
|
||||
lines.append(f"| `{host}` | `{container}` | {mode} |")
|
||||
lines.append("")
|
||||
|
||||
# Networks
|
||||
nets = format_networks(svc.get("networks"))
|
||||
if nets:
|
||||
lines.append("### Networks")
|
||||
lines.append("")
|
||||
for name, ntype in nets:
|
||||
lines.append(f"- `{name}` ({ntype})")
|
||||
lines.append("")
|
||||
|
||||
# Depends on
|
||||
depends = format_depends(svc.get("depends_on"))
|
||||
if depends:
|
||||
lines.append("### Dependencies")
|
||||
lines.append("")
|
||||
for d in depends:
|
||||
lines.append(f"- `{d}`")
|
||||
lines.append("")
|
||||
|
||||
# Capabilities
|
||||
caps = format_cap_add(svc.get("cap_add"))
|
||||
if caps:
|
||||
lines.append("### Capabilities")
|
||||
lines.append("")
|
||||
lines.append(", ".join(f"`{c}`" for c in caps))
|
||||
lines.append("")
|
||||
|
||||
# Security
|
||||
sec = format_security(svc.get("security_opt"))
|
||||
if sec:
|
||||
lines.append("### Security Options")
|
||||
lines.append("")
|
||||
for s in sec:
|
||||
lines.append(f"- `{s}`")
|
||||
lines.append("")
|
||||
|
||||
# Labels
|
||||
labels = format_labels(svc.get("labels"))
|
||||
if labels:
|
||||
lines.append("### Labels")
|
||||
lines.append("")
|
||||
lines.append("| Key | Value |")
|
||||
lines.append("|-----|-------|")
|
||||
for key, val in labels:
|
||||
lines.append(f"| `{key}` | `{val}` |")
|
||||
lines.append("")
|
||||
|
||||
# Devices
|
||||
devices = svc.get("devices")
|
||||
if devices:
|
||||
lines.append("### Devices")
|
||||
lines.append("")
|
||||
for d in devices:
|
||||
if isinstance(d, str):
|
||||
lines.append(f"- `{d}`")
|
||||
lines.append("")
|
||||
|
||||
# tmpfs
|
||||
tmpfs = svc.get("tmpfs")
|
||||
if tmpfs:
|
||||
lines.append("### Tmpfs")
|
||||
lines.append("")
|
||||
if isinstance(tmpfs, list):
|
||||
for t in tmpfs:
|
||||
lines.append(f"- `{t}`")
|
||||
lines.append("")
|
||||
|
||||
# shm_size
|
||||
shm = svc.get("shm_size")
|
||||
if shm:
|
||||
lines.append(f"### SHM Size: `{shm}`")
|
||||
lines.append("")
|
||||
|
||||
# resource limits
|
||||
mem = svc.get("mem_limit")
|
||||
swap = svc.get("memswap_limit")
|
||||
if mem or swap:
|
||||
lines.append("### Resources")
|
||||
lines.append("")
|
||||
if mem:
|
||||
lines.append(f"- Memory limit: `{mem}`")
|
||||
if swap:
|
||||
lines.append(f"- Memory+Swap limit: `{swap}`")
|
||||
lines.append("")
|
||||
|
||||
# Compose-level volumes (named)
|
||||
comp_volumes = compose_data.get("volumes")
|
||||
if comp_volumes and isinstance(comp_volumes, dict):
|
||||
lines.append("### Named Volumes (compose-level)")
|
||||
lines.append("")
|
||||
for vname, vcfg in comp_volumes.items():
|
||||
external = False
|
||||
if isinstance(vcfg, dict):
|
||||
external = vcfg.get("external", False)
|
||||
ntype = "external" if external else "managed by compose"
|
||||
lines.append(f"- `{vname}` ({ntype})")
|
||||
lines.append("")
|
||||
|
||||
# Compose-level networks
|
||||
comp_nets = compose_data.get("networks")
|
||||
if comp_nets and isinstance(comp_nets, dict):
|
||||
lines.append("### Networks (compose-level)")
|
||||
lines.append("")
|
||||
for nname, ncfg in comp_nets.items():
|
||||
external = False
|
||||
if isinstance(ncfg, dict):
|
||||
external = ncfg.get("external", False)
|
||||
ntype = "external" if external else "internal"
|
||||
lines.append(f"- `{nname}` ({ntype})")
|
||||
lines.append("")
|
||||
|
||||
lines.append("---")
|
||||
# Include .env.example if exists
|
||||
env_example_path = service_dir / ".env.example"
|
||||
if env_example_path.exists():
|
||||
lines.append("## Environment Variables (`.env.example`)")
|
||||
lines.append("")
|
||||
lines.append("```bash")
|
||||
lines.append(f'--8<-- "{service_dir.name}/.env.example"')
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_index(all_services):
|
||||
lines = []
|
||||
lines.append("# Homelab Documentation")
|
||||
lines.append("")
|
||||
lines.append("> Auto-generated from `docker-compose.yml` files")
|
||||
lines.append("")
|
||||
|
||||
lines.append("## Data Classification")
|
||||
lines.append("")
|
||||
lines.append("| Type | Example | Git Repo? | Backup? | Location |")
|
||||
lines.append("|------|---------|-----------|---------|----------|")
|
||||
lines.append("| **Source Configs** | `docker-compose.yml`, `.env.example`, `Makefile` | Yes | No | `/srv/homelab-infra` |")
|
||||
lines.append("| **Service Secrets** | `.env` (DB passwords, API keys) | No | Yes | `/srv/homelab-infra/<service>/.env` |")
|
||||
lines.append("| **Runtime Configs** | `acme.json`, service configs | No | Yes | `/mnt/storage/docker-data/<service>` |")
|
||||
lines.append("| **Persistent Data** | DB data, uploads, media | No | Yes | `/mnt/storage/docker-data/<service>` |")
|
||||
lines.append("")
|
||||
|
||||
lines.append("## Services")
|
||||
lines.append("")
|
||||
lines.append("| Service | Image | Status |")
|
||||
lines.append("|---------|-------|--------|")
|
||||
for name, data in sorted(all_services.items()):
|
||||
services = data.get("services", {})
|
||||
images = []
|
||||
for svc in services.values():
|
||||
img = svc.get("image", "N/A")
|
||||
images.append(f"`{img}`")
|
||||
status = "active" if "planned" not in name.lower() else "planned"
|
||||
lines.append(f"| [{name}](services/{name}.md) | {', '.join(images)} | {status} |")
|
||||
lines.append("")
|
||||
|
||||
lines.append("## Quick Start")
|
||||
lines.append("")
|
||||
lines.append("```bash")
|
||||
lines.append("cp .env.example .env")
|
||||
lines.append("for svc in */; do [ -f \"$svc/.env.example\" ] && cp \"$svc/.env.example\" \"$svc/.env\"; done")
|
||||
lines.append("docker network create web")
|
||||
lines.append("make up")
|
||||
lines.append("```")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines)
|
||||
def service_display_name(service_name):
|
||||
if service_name in DISPLAY_NAME_OVERRIDES:
|
||||
return DISPLAY_NAME_OVERRIDES[service_name]
|
||||
return " ".join(part.capitalize() for part in service_name.split("-"))
|
||||
|
||||
|
||||
def update_mkdocs_nav(service_names):
|
||||
mkdocs_file = REPO_ROOT / "mkdocs.yml"
|
||||
cfg = {}
|
||||
if mkdocs_file.exists():
|
||||
existing = yaml.safe_load(mkdocs_file.read_text())
|
||||
if isinstance(existing, dict):
|
||||
cfg = existing
|
||||
|
||||
existing_nav = cfg.get("nav", [])
|
||||
nav_without_services = []
|
||||
for item in existing_nav:
|
||||
if isinstance(item, dict) and "Services" in item:
|
||||
continue
|
||||
nav_without_services.append(item)
|
||||
|
||||
if not nav_without_services:
|
||||
nav_without_services = [
|
||||
{"Home": "index.md"},
|
||||
{"Recovery Reference": "recovery.md"},
|
||||
]
|
||||
|
||||
services_nav = []
|
||||
for name in sorted(service_names):
|
||||
services_nav.append({service_display_name(name): f"services/{name}.md"})
|
||||
|
||||
cfg["nav"] = nav_without_services + [{"Services": services_nav}]
|
||||
|
||||
if "site_name" not in cfg:
|
||||
cfg["site_name"] = "Homelab Documentation"
|
||||
if "theme" not in cfg:
|
||||
cfg["theme"] = {"name": "material", "palette": {"scheme": "slate"}}
|
||||
|
||||
mkdocs_file.write_text(yaml.safe_dump(cfg, sort_keys=False))
|
||||
print(f"Updated {mkdocs_file.relative_to(REPO_ROOT)}")
|
||||
|
||||
|
||||
def cleanup_stale_service_docs(service_names):
|
||||
DOCS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
valid = set(service_names)
|
||||
for md_file in DOCS_DIR.glob("*.md"):
|
||||
if md_file.stem not in valid:
|
||||
md_file.unlink()
|
||||
print(f"Removed stale {md_file.relative_to(REPO_ROOT)}")
|
||||
|
||||
|
||||
def main():
|
||||
root_env = load_root_env_example()
|
||||
all_services = {}
|
||||
|
||||
# Find all service directories with docker-compose.yml
|
||||
@@ -450,21 +299,20 @@ def main():
|
||||
if not compose_data:
|
||||
continue
|
||||
|
||||
service_env = load_service_env_example(item)
|
||||
all_services[service_name] = compose_data
|
||||
|
||||
# Generate service doc
|
||||
md = generate_service_md(service_name, compose_data, root_env, service_env)
|
||||
# Generate service doc (stub that includes files)
|
||||
md = generate_service_md(service_name, item)
|
||||
out_file = DOCS_DIR / f"{service_name}.md"
|
||||
out_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_file.write_text(md)
|
||||
print(f"Generated {out_file.relative_to(REPO_ROOT)}")
|
||||
|
||||
# Generate index
|
||||
index_md = generate_index(all_services)
|
||||
index_file = REPO_ROOT / "docs" / "index.md"
|
||||
index_file.write_text(index_md)
|
||||
print(f"Generated {index_file.relative_to(REPO_ROOT)}")
|
||||
# Remove docs for services that no longer exist
|
||||
cleanup_stale_service_docs(all_services.keys())
|
||||
|
||||
# Keep mkdocs.yml service nav in sync with discovered service folders
|
||||
update_mkdocs_nav(all_services.keys())
|
||||
|
||||
print(f"\nDone. Generated docs for {len(all_services)} services.")
|
||||
|
||||
|
||||
Reference in New Issue
Block a user