feat(ci): complete security audit governance and resilient CI control lanes
This commit is contained in:
Chummy
Executable
+354
@@ -0,0 +1,354 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate a CI/CD change audit report for GitHub workflows and CI scripts.
|
||||
|
||||
The report is designed for change-control traceability and light policy checks:
|
||||
- enumerate changed CI-related files
|
||||
- summarize line churn
|
||||
- capture newly introduced action references
|
||||
- detect unpinned `uses:` action references
|
||||
- detect risky pipe-to-shell commands (e.g. `curl ... | sh`)
|
||||
- detect newly introduced `pull_request_target` triggers in supported YAML forms
|
||||
- detect broad `permissions: write-all` grants
|
||||
- detect newly introduced `${{ secrets.* }}` references
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Iterable
|
||||
|
||||
|
||||
AUDIT_PREFIXES = (
|
||||
".github/workflows/",
|
||||
".github/actions/",
|
||||
".github/codeql/",
|
||||
"scripts/ci/",
|
||||
".githooks/",
|
||||
)
|
||||
AUDIT_FILES = {
|
||||
".github/dependabot.yml",
|
||||
"deny.toml",
|
||||
".gitleaks.toml",
|
||||
}
|
||||
USES_RE = re.compile(r"^\+\s*(?:-\s*)?uses:\s*([^\s#]+)")
|
||||
SECRETS_RE = re.compile(r"\$\{\{\s*secrets\.([A-Za-z0-9_]+)\s*}}")
|
||||
SHA_PIN_RE = re.compile(r"^[0-9a-f]{40}$")
|
||||
PIPE_TO_SHELL_RE = re.compile(r"\b(?:curl|wget)\b.*\|\s*(?:sh|bash)\b")
|
||||
PERMISSION_WRITE_RE = re.compile(r"^\+\s*([a-z-]+):\s*write\s*$")
|
||||
PERMISSIONS_WRITE_ALL_RE = re.compile(r"^\+\s*permissions\s*:\s*write-all\s*$", re.IGNORECASE)
|
||||
|
||||
|
||||
def line_adds_pull_request_target(added_text: str) -> bool:
|
||||
# Support the three common YAML forms:
|
||||
# 1) pull_request_target:
|
||||
# 2) - pull_request_target
|
||||
# 3) on: [push, pull_request_target]
|
||||
normalized = added_text.split("#", 1)[0].strip().lower()
|
||||
if not normalized:
|
||||
return False
|
||||
if normalized.startswith("pull_request_target:"):
|
||||
return True
|
||||
if normalized == "- pull_request_target":
|
||||
return True
|
||||
if normalized.startswith("on:") and "[" in normalized and "]" in normalized:
|
||||
return "pull_request_target" in normalized
|
||||
return False
|
||||
|
||||
|
||||
def run(cmd: list[str]) -> str:
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True, check=False)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"Command failed ({proc.returncode}): {' '.join(cmd)}\n{proc.stderr}")
|
||||
return proc.stdout
|
||||
|
||||
|
||||
def is_ci_path(path: str) -> bool:
|
||||
return path in AUDIT_FILES or path.startswith(AUDIT_PREFIXES)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FileAudit:
|
||||
path: str
|
||||
status: str
|
||||
added: int = 0
|
||||
deleted: int = 0
|
||||
added_actions: list[str] = field(default_factory=list)
|
||||
unpinned_actions: list[str] = field(default_factory=list)
|
||||
added_secret_refs: list[str] = field(default_factory=list)
|
||||
added_pipe_to_shell: list[str] = field(default_factory=list)
|
||||
added_write_permissions: list[str] = field(default_factory=list)
|
||||
added_pull_request_target: int = 0
|
||||
|
||||
@property
|
||||
def risk_level(self) -> str:
|
||||
if (
|
||||
self.unpinned_actions
|
||||
or self.added_pipe_to_shell
|
||||
or self.added_pull_request_target
|
||||
or "write-all" in self.added_write_permissions
|
||||
):
|
||||
return "high"
|
||||
if self.added_secret_refs or self.added_actions or self.added_write_permissions:
|
||||
return "medium"
|
||||
return "low"
|
||||
|
||||
|
||||
def parse_changed_files(base_sha: str, head_sha: str) -> list[tuple[str, str]]:
|
||||
out = run(["git", "diff", "--name-status", "--find-renames", base_sha, head_sha])
|
||||
changed: list[tuple[str, str]] = []
|
||||
for line in out.splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
parts = line.split("\t")
|
||||
status = parts[0]
|
||||
path = parts[-1]
|
||||
changed.append((status, path))
|
||||
return changed
|
||||
|
||||
|
||||
def parse_numstat(base_sha: str, head_sha: str, path: str) -> tuple[int, int]:
|
||||
out = run(["git", "diff", "--numstat", base_sha, head_sha, "--", path]).strip()
|
||||
if not out:
|
||||
return (0, 0)
|
||||
parts = out.split("\t")
|
||||
if len(parts) < 3:
|
||||
return (0, 0)
|
||||
add_raw, del_raw = parts[0], parts[1]
|
||||
try:
|
||||
added = 0 if add_raw == "-" else int(add_raw)
|
||||
deleted = 0 if del_raw == "-" else int(del_raw)
|
||||
except ValueError:
|
||||
return (0, 0)
|
||||
return (added, deleted)
|
||||
|
||||
|
||||
def parse_patch_added_lines(base_sha: str, head_sha: str, path: str) -> Iterable[str]:
|
||||
out = run(["git", "diff", "-U0", base_sha, head_sha, "--", path])
|
||||
for line in out.splitlines():
|
||||
if not line.startswith("+") or line.startswith("+++"):
|
||||
continue
|
||||
yield line
|
||||
|
||||
|
||||
def action_is_pinned(action_ref: str) -> bool:
|
||||
if action_ref.startswith("./"):
|
||||
return True
|
||||
if "@" not in action_ref:
|
||||
return False
|
||||
version = action_ref.rsplit("@", 1)[1]
|
||||
return bool(SHA_PIN_RE.fullmatch(version))
|
||||
|
||||
|
||||
def build_markdown(
|
||||
audits: list[FileAudit],
|
||||
*,
|
||||
base_sha: str,
|
||||
head_sha: str,
|
||||
violations: list[str],
|
||||
) -> str:
|
||||
lines: list[str] = []
|
||||
lines.append("# CI/CD Change Audit")
|
||||
lines.append("")
|
||||
lines.append(f"- Base SHA: `{base_sha}`")
|
||||
lines.append(f"- Head SHA: `{head_sha}`")
|
||||
lines.append(f"- Audited files: `{len(audits)}`")
|
||||
lines.append(
|
||||
f"- Policy violations: `{len(violations)}` "
|
||||
"(currently: unpinned `uses:`, pipe-to-shell commands, broad "
|
||||
"`permissions: write-all`, and new `pull_request_target` triggers)"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
if violations:
|
||||
lines.append("## Violations")
|
||||
for entry in violations:
|
||||
lines.append(f"- {entry}")
|
||||
lines.append("")
|
||||
|
||||
if not audits:
|
||||
lines.append("No CI/CD files changed in this diff.")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
lines.append("## File Summary")
|
||||
lines.append("")
|
||||
lines.append(
|
||||
"| Path | Status | +Lines | -Lines | New Actions | New Secret Refs | "
|
||||
"Pipe-to-Shell | New `*: write` | New `pull_request_target` | Risk |"
|
||||
)
|
||||
lines.append("| --- | --- | ---:| ---:| ---:| ---:| ---:| ---:| ---:| --- |")
|
||||
for audit in sorted(audits, key=lambda x: x.path):
|
||||
lines.append(
|
||||
f"| `{audit.path}` | `{audit.status}` | {audit.added} | {audit.deleted} | "
|
||||
f"{len(audit.added_actions)} | {len(audit.added_secret_refs)} | "
|
||||
f"{len(audit.added_pipe_to_shell)} | {len(set(audit.added_write_permissions))} | "
|
||||
f"{audit.added_pull_request_target} | "
|
||||
f"`{audit.risk_level}` |"
|
||||
)
|
||||
lines.append("")
|
||||
|
||||
medium_or_high = [a for a in audits if a.risk_level in {"medium", "high"}]
|
||||
if medium_or_high:
|
||||
lines.append("## Detailed Review Targets")
|
||||
for audit in sorted(medium_or_high, key=lambda x: x.path):
|
||||
lines.append(f"### `{audit.path}`")
|
||||
if audit.added_actions:
|
||||
lines.append("- Added `uses:` references:")
|
||||
for action in audit.added_actions:
|
||||
pin_state = "pinned" if action not in audit.unpinned_actions else "unpinned"
|
||||
lines.append(f" - `{action}` ({pin_state})")
|
||||
if audit.added_secret_refs:
|
||||
lines.append("- Added secret references:")
|
||||
for secret_key in sorted(set(audit.added_secret_refs)):
|
||||
lines.append(f" - `secrets.{secret_key}`")
|
||||
if audit.added_pipe_to_shell:
|
||||
lines.append("- Added pipe-to-shell commands (high risk):")
|
||||
for cmd in audit.added_pipe_to_shell:
|
||||
lines.append(f" - `{cmd}`")
|
||||
if audit.added_write_permissions:
|
||||
lines.append("- Added write permissions:")
|
||||
for permission_name in sorted(set(audit.added_write_permissions)):
|
||||
if permission_name == "write-all":
|
||||
lines.append(" - `permissions: write-all`")
|
||||
else:
|
||||
lines.append(f" - `{permission_name}: write`")
|
||||
if audit.added_pull_request_target:
|
||||
lines.append("- Added `pull_request_target` trigger (high risk):")
|
||||
lines.append(" - Review event payload usage and token scope carefully.")
|
||||
lines.append("")
|
||||
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Generate CI/CD change audit report.")
|
||||
parser.add_argument("--base-sha", required=True, help="Base commit SHA")
|
||||
parser.add_argument("--head-sha", default="HEAD", help="Head commit SHA (default: HEAD)")
|
||||
parser.add_argument("--output-json", required=True, help="Output JSON path")
|
||||
parser.add_argument("--output-md", required=True, help="Output Markdown path")
|
||||
parser.add_argument(
|
||||
"--fail-on-violations",
|
||||
action="store_true",
|
||||
help="Return non-zero when policy violations are found",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
changed = parse_changed_files(args.base_sha, args.head_sha)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 2
|
||||
|
||||
audits: list[FileAudit] = []
|
||||
violations: list[str] = []
|
||||
for status, path in changed:
|
||||
if not is_ci_path(path):
|
||||
continue
|
||||
|
||||
added, deleted = parse_numstat(args.base_sha, args.head_sha, path)
|
||||
audit = FileAudit(path=path, status=status, added=added, deleted=deleted)
|
||||
|
||||
for line in parse_patch_added_lines(args.base_sha, args.head_sha, path):
|
||||
added_text = line[1:].strip()
|
||||
|
||||
uses_match = USES_RE.search(line)
|
||||
if uses_match:
|
||||
action_ref = uses_match.group(1).strip()
|
||||
audit.added_actions.append(action_ref)
|
||||
if not action_is_pinned(action_ref):
|
||||
audit.unpinned_actions.append(action_ref)
|
||||
violations.append(
|
||||
f"{path}: unpinned action reference introduced -> `{action_ref}`"
|
||||
)
|
||||
|
||||
for secret_name in SECRETS_RE.findall(line):
|
||||
audit.added_secret_refs.append(secret_name)
|
||||
|
||||
if PIPE_TO_SHELL_RE.search(added_text):
|
||||
command = added_text[:220]
|
||||
audit.added_pipe_to_shell.append(command)
|
||||
violations.append(
|
||||
f"{path}: pipe-to-shell command introduced -> `{command}`"
|
||||
)
|
||||
|
||||
permission_match = PERMISSION_WRITE_RE.match(line)
|
||||
if permission_match:
|
||||
audit.added_write_permissions.append(permission_match.group(1))
|
||||
if PERMISSIONS_WRITE_ALL_RE.match(line):
|
||||
audit.added_write_permissions.append("write-all")
|
||||
violations.append(
|
||||
f"{path}: `permissions: write-all` introduced; scope permissions minimally."
|
||||
)
|
||||
|
||||
if line_adds_pull_request_target(added_text):
|
||||
audit.added_pull_request_target += 1
|
||||
violations.append(
|
||||
f"{path}: `pull_request_target` trigger introduced -> `{added_text[:180]}`; "
|
||||
"manual security review required."
|
||||
)
|
||||
|
||||
audits.append(audit)
|
||||
|
||||
summary = {
|
||||
"total_changed_files": len(changed),
|
||||
"audited_files": len(audits),
|
||||
"added_lines": sum(a.added for a in audits),
|
||||
"deleted_lines": sum(a.deleted for a in audits),
|
||||
"new_actions": sum(len(a.added_actions) for a in audits),
|
||||
"new_unpinned_actions": sum(len(a.unpinned_actions) for a in audits),
|
||||
"new_secret_references": sum(len(a.added_secret_refs) for a in audits),
|
||||
"new_pipe_to_shell_commands": sum(len(a.added_pipe_to_shell) for a in audits),
|
||||
"new_write_permissions": sum(len(set(a.added_write_permissions)) for a in audits),
|
||||
"new_pull_request_target_triggers": sum(a.added_pull_request_target for a in audits),
|
||||
"violations": len(violations),
|
||||
}
|
||||
payload = {
|
||||
"generated_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"base_sha": args.base_sha,
|
||||
"head_sha": args.head_sha,
|
||||
"summary": summary,
|
||||
"files": [
|
||||
{
|
||||
"path": a.path,
|
||||
"status": a.status,
|
||||
"added": a.added,
|
||||
"deleted": a.deleted,
|
||||
"added_actions": a.added_actions,
|
||||
"unpinned_actions": a.unpinned_actions,
|
||||
"added_secret_refs": sorted(set(a.added_secret_refs)),
|
||||
"added_pipe_to_shell": a.added_pipe_to_shell,
|
||||
"added_write_permissions": sorted(set(a.added_write_permissions)),
|
||||
"added_pull_request_target": a.added_pull_request_target,
|
||||
"risk_level": a.risk_level,
|
||||
}
|
||||
for a in sorted(audits, key=lambda x: x.path)
|
||||
],
|
||||
"violations": violations,
|
||||
}
|
||||
|
||||
json_path = Path(args.output_json)
|
||||
md_path = Path(args.output_md)
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
md_path.write_text(
|
||||
build_markdown(audits, base_sha=args.base_sha, head_sha=args.head_sha, violations=violations),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
if args.fail_on_violations and violations:
|
||||
print("CI/CD change audit violations found:", file=sys.stderr)
|
||||
for item in violations:
|
||||
print(f"- {item}", file=sys.stderr)
|
||||
return 3
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,215 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate deny.toml policy hygiene for advisory ignore exceptions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import tomllib # Python 3.11+
|
||||
except ModuleNotFoundError: # pragma: no cover
|
||||
import tomli as tomllib # type: ignore
|
||||
|
||||
TICKET_RE = re.compile(r"^[A-Z]+-\d+$")
|
||||
|
||||
|
||||
def parse_iso_date(raw: str) -> dt.date | None:
|
||||
try:
|
||||
return dt.date.fromisoformat(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def build_markdown(report: dict) -> str:
|
||||
lines: list[str] = []
|
||||
lines.append("# deny.toml Policy Guard")
|
||||
lines.append("")
|
||||
lines.append(f"- Generated at: `{report['generated_at']}`")
|
||||
lines.append(f"- Ignore entries: `{report['ignore_count']}`")
|
||||
lines.append(f"- Governance entries: `{report['governance_entries']}`")
|
||||
lines.append(f"- Violations: `{len(report['violations'])}`")
|
||||
lines.append(f"- Warnings: `{len(report['warnings'])}`")
|
||||
lines.append("")
|
||||
if report["violations"]:
|
||||
lines.append("## Violations")
|
||||
for item in report["violations"]:
|
||||
lines.append(f"- {item}")
|
||||
lines.append("")
|
||||
if report["warnings"]:
|
||||
lines.append("## Warnings")
|
||||
for item in report["warnings"]:
|
||||
lines.append(f"- {item}")
|
||||
lines.append("")
|
||||
if not report["violations"]:
|
||||
lines.append("No policy violations found.")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Validate deny.toml advisory ignore policy.")
|
||||
parser.add_argument("--deny-file", default="deny.toml")
|
||||
parser.add_argument("--governance-file", default=".github/security/deny-ignore-governance.json")
|
||||
parser.add_argument("--warn-days", type=int, default=30)
|
||||
parser.add_argument("--output-json", required=True)
|
||||
parser.add_argument("--output-md", required=True)
|
||||
parser.add_argument("--fail-on-violation", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
deny_path = Path(args.deny_file)
|
||||
content = tomllib.loads(deny_path.read_text(encoding="utf-8"))
|
||||
advisories = content.get("advisories", {})
|
||||
ignore = advisories.get("ignore", [])
|
||||
|
||||
violations: list[str] = []
|
||||
warnings: list[str] = []
|
||||
normalized: list[dict] = []
|
||||
today = dt.date.today()
|
||||
|
||||
if not isinstance(ignore, list):
|
||||
violations.append("`advisories.ignore` must be a list.")
|
||||
ignore = []
|
||||
|
||||
governance_path = Path(args.governance_file)
|
||||
governance_map: dict[str, dict] = {}
|
||||
if governance_path.exists():
|
||||
governance_raw = json.loads(governance_path.read_text(encoding="utf-8"))
|
||||
governance_entries = governance_raw.get("advisories", [])
|
||||
if not isinstance(governance_entries, list):
|
||||
violations.append("deny governance file: `advisories` must be a list.")
|
||||
governance_entries = []
|
||||
for idx, entry in enumerate(governance_entries):
|
||||
if not isinstance(entry, dict):
|
||||
violations.append(f"deny governance advisory[{idx}] must be an object.")
|
||||
continue
|
||||
adv_id = str(entry.get("id", "")).strip()
|
||||
owner = str(entry.get("owner", "")).strip()
|
||||
reason = str(entry.get("reason", "")).strip()
|
||||
ticket = str(entry.get("ticket", "")).strip()
|
||||
expires_on = str(entry.get("expires_on", "")).strip()
|
||||
|
||||
if not adv_id:
|
||||
violations.append(f"deny governance advisory[{idx}] missing required field `id`.")
|
||||
continue
|
||||
if adv_id in governance_map:
|
||||
violations.append(f"deny governance contains duplicate advisory id `{adv_id}`.")
|
||||
if not owner:
|
||||
violations.append(f"deny governance `{adv_id}` missing required field `owner`.")
|
||||
if not reason:
|
||||
violations.append(f"deny governance `{adv_id}` missing required field `reason`.")
|
||||
elif len(reason) < 12:
|
||||
violations.append(
|
||||
f"deny governance `{adv_id}` reason is too short; provide actionable context."
|
||||
)
|
||||
if not expires_on:
|
||||
violations.append(f"deny governance `{adv_id}` missing required field `expires_on`.")
|
||||
parsed_expires = None
|
||||
else:
|
||||
parsed_expires = parse_iso_date(expires_on)
|
||||
if parsed_expires is None:
|
||||
violations.append(
|
||||
f"deny governance `{adv_id}` has invalid `expires_on` (`{expires_on}`); "
|
||||
"use YYYY-MM-DD."
|
||||
)
|
||||
elif parsed_expires < today:
|
||||
violations.append(
|
||||
f"deny governance `{adv_id}` expired on `{expires_on}`; renew or remove ignore."
|
||||
)
|
||||
elif parsed_expires <= (today + dt.timedelta(days=max(0, args.warn_days))):
|
||||
warnings.append(
|
||||
f"deny governance `{adv_id}` expires soon on `{expires_on}`; schedule review."
|
||||
)
|
||||
if not ticket:
|
||||
warnings.append(f"deny governance `{adv_id}` missing tracking `ticket`.")
|
||||
elif not TICKET_RE.fullmatch(ticket):
|
||||
warnings.append(
|
||||
f"deny governance `{adv_id}` ticket `{ticket}` does not match KEY-123 format."
|
||||
)
|
||||
|
||||
governance_map[adv_id] = {
|
||||
"id": adv_id,
|
||||
"owner": owner,
|
||||
"reason": reason,
|
||||
"ticket": ticket,
|
||||
"expires_on": expires_on,
|
||||
}
|
||||
else:
|
||||
violations.append(f"deny governance file not found: `{governance_path}`")
|
||||
|
||||
ignore_ids: set[str] = set()
|
||||
for idx, entry in enumerate(ignore):
|
||||
if isinstance(entry, str):
|
||||
violations.append(
|
||||
f"ignore[{idx}] uses legacy string format (`{entry}`); use table form with `id` + `reason`."
|
||||
)
|
||||
normalized.append({"id": entry, "reason": "", "legacy": True})
|
||||
continue
|
||||
|
||||
if not isinstance(entry, dict):
|
||||
violations.append(f"ignore[{idx}] must be a table/object.")
|
||||
continue
|
||||
|
||||
adv_id = str(entry.get("id", "")).strip()
|
||||
reason = str(entry.get("reason", "")).strip()
|
||||
expires = str(entry.get("expires", "")).strip()
|
||||
if not adv_id:
|
||||
violations.append(f"ignore[{idx}] is missing required field `id`.")
|
||||
if not reason:
|
||||
violations.append(f"ignore[{idx}] (`{adv_id or 'unknown'}`) is missing required field `reason`.")
|
||||
elif len(reason) < 12:
|
||||
violations.append(
|
||||
f"ignore[{idx}] (`{adv_id or 'unknown'}`) reason is too short; provide actionable mitigation context."
|
||||
)
|
||||
|
||||
normalized.append({"id": adv_id, "reason": reason, "expires": expires, "legacy": False})
|
||||
if adv_id:
|
||||
ignore_ids.add(adv_id)
|
||||
if adv_id not in governance_map:
|
||||
violations.append(
|
||||
f"ignore[{idx}] (`{adv_id}`) has no governance metadata in `{governance_path}`."
|
||||
)
|
||||
|
||||
stale_governance = sorted([adv_id for adv_id in governance_map if adv_id not in ignore_ids])
|
||||
for adv_id in stale_governance:
|
||||
warnings.append(
|
||||
f"deny governance entry `{adv_id}` exists but advisory is not currently ignored in deny.toml."
|
||||
)
|
||||
|
||||
report = {
|
||||
"schema_version": "zeroclaw.audit.v1",
|
||||
"event_type": "deny_policy_guard",
|
||||
"generated_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"deny_file": str(deny_path),
|
||||
"governance_file": str(governance_path),
|
||||
"ignore_count": len(normalized),
|
||||
"governance_entries": len(governance_map),
|
||||
"unmanaged_ignores": sorted([item["id"] for item in normalized if item.get("id") and item["id"] not in governance_map]),
|
||||
"stale_governance": stale_governance,
|
||||
"warnings": warnings,
|
||||
"violations": violations,
|
||||
"ignores": normalized,
|
||||
"governance": [governance_map[k] for k in sorted(governance_map)],
|
||||
}
|
||||
|
||||
json_path = Path(args.output_json)
|
||||
md_path = Path(args.output_md)
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
md_path.write_text(build_markdown(report), encoding="utf-8")
|
||||
|
||||
if args.fail_on_violation and violations:
|
||||
print("deny policy violations found:", file=sys.stderr)
|
||||
for item in violations:
|
||||
print(f"- {item}", file=sys.stderr)
|
||||
return 3
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Wrap workflow artifacts into a normalized audit event envelope."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Emit normalized audit event envelope.")
|
||||
parser.add_argument("--event-type", required=True)
|
||||
parser.add_argument("--input-json", required=True)
|
||||
parser.add_argument("--output-json", required=True)
|
||||
parser.add_argument("--artifact-name", default="")
|
||||
parser.add_argument("--retention-days", type=int, default=0)
|
||||
args = parser.parse_args()
|
||||
|
||||
payload = json.loads(Path(args.input_json).read_text(encoding="utf-8"))
|
||||
event = {
|
||||
"schema_version": "zeroclaw.audit.v1",
|
||||
"event_type": args.event_type,
|
||||
"generated_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"run_context": {
|
||||
"repository": os.getenv("GITHUB_REPOSITORY", ""),
|
||||
"workflow": os.getenv("GITHUB_WORKFLOW", ""),
|
||||
"run_id": os.getenv("GITHUB_RUN_ID", ""),
|
||||
"run_attempt": os.getenv("GITHUB_RUN_ATTEMPT", ""),
|
||||
"sha": os.getenv("GITHUB_SHA", ""),
|
||||
"ref": os.getenv("GITHUB_REF", ""),
|
||||
"actor": os.getenv("GITHUB_ACTOR", ""),
|
||||
},
|
||||
"payload": payload,
|
||||
}
|
||||
if args.artifact_name or args.retention_days > 0:
|
||||
artifact_meta: dict[str, object] = {}
|
||||
if args.artifact_name:
|
||||
artifact_meta["name"] = args.artifact_name
|
||||
if args.retention_days > 0:
|
||||
artifact_meta["retention_days"] = args.retention_days
|
||||
event["artifact"] = artifact_meta
|
||||
|
||||
out = Path(args.output_json)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(json.dumps(event, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,98 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Run a single retry probe for failed test jobs and emit flake telemetry artifacts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def parse_bool(value: str) -> bool:
|
||||
return value.strip().lower() in {"1", "true", "yes", "y", "on"}
|
||||
|
||||
|
||||
def run_retry(command: str) -> tuple[int, int]:
|
||||
started = time.perf_counter()
|
||||
proc = subprocess.run(command, shell=True, check=False)
|
||||
elapsed_ms = int((time.perf_counter() - started) * 1000)
|
||||
return (proc.returncode, elapsed_ms)
|
||||
|
||||
|
||||
def build_markdown(report: dict) -> str:
|
||||
lines: list[str] = []
|
||||
lines.append("# Test Flake Retry Probe")
|
||||
lines.append("")
|
||||
lines.append(f"- Generated at: `{report['generated_at']}`")
|
||||
lines.append(f"- Initial test result: `{report['initial_test_result']}`")
|
||||
lines.append(f"- Retry attempted: `{report['retry_attempted']}`")
|
||||
lines.append(f"- Classification: `{report['classification']}`")
|
||||
lines.append(f"- Block on flake: `{report['block_on_flake']}`")
|
||||
if report["retry_attempted"]:
|
||||
lines.append(f"- Retry exit code: `{report['retry_exit_code']}`")
|
||||
lines.append(f"- Retry duration (ms): `{report['retry_duration_ms']}`")
|
||||
lines.append("")
|
||||
if report["classification"] == "flake_suspected":
|
||||
lines.append("Detected flaky pattern: first run failed, retry run passed.")
|
||||
elif report["classification"] == "persistent_failure":
|
||||
lines.append("Detected persistent failure: first run failed and retry run failed.")
|
||||
else:
|
||||
lines.append("No retry probe needed because initial test run did not fail.")
|
||||
lines.append("")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Emit flaky-test retry probe artifacts.")
|
||||
parser.add_argument("--initial-result", required=True, help="needs.test.result from workflow context")
|
||||
parser.add_argument("--retry-command", required=True, help="Command to rerun failed tests once")
|
||||
parser.add_argument("--output-json", required=True)
|
||||
parser.add_argument("--output-md", required=True)
|
||||
parser.add_argument("--block-on-flake", default="false", help="Whether suspected flakes should fail the job")
|
||||
args = parser.parse_args()
|
||||
|
||||
initial = args.initial_result.strip().lower()
|
||||
block_on_flake = parse_bool(args.block_on_flake)
|
||||
|
||||
retry_attempted = False
|
||||
retry_exit_code: int | None = None
|
||||
retry_duration_ms = 0
|
||||
classification = "not_applicable"
|
||||
|
||||
if initial == "failure":
|
||||
retry_attempted = True
|
||||
retry_exit_code, retry_duration_ms = run_retry(args.retry_command)
|
||||
if retry_exit_code == 0:
|
||||
classification = "flake_suspected"
|
||||
else:
|
||||
classification = "persistent_failure"
|
||||
|
||||
report = {
|
||||
"schema_version": "zeroclaw.audit.v1",
|
||||
"event_type": "test_flake_retry_probe",
|
||||
"generated_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"initial_test_result": initial,
|
||||
"retry_attempted": retry_attempted,
|
||||
"retry_exit_code": retry_exit_code,
|
||||
"retry_duration_ms": retry_duration_ms,
|
||||
"classification": classification,
|
||||
"block_on_flake": block_on_flake,
|
||||
}
|
||||
|
||||
json_path = Path(args.output_json)
|
||||
md_path = Path(args.output_md)
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_path.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
md_path.write_text(build_markdown(report), encoding="utf-8")
|
||||
|
||||
if classification == "flake_suspected" and block_on_flake:
|
||||
return 3
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate an in-toto/SLSA-style provenance statement for a built artifact."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def sha256_file(path: Path) -> str:
|
||||
h = hashlib.sha256()
|
||||
with path.open("rb") as f:
|
||||
for chunk in iter(lambda: f.read(1024 * 1024), b""):
|
||||
h.update(chunk)
|
||||
return h.hexdigest()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Generate provenance statement for artifact.")
|
||||
parser.add_argument("--artifact", required=True)
|
||||
parser.add_argument("--subject-name", default="zeroclaw")
|
||||
parser.add_argument("--output", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
artifact = Path(args.artifact)
|
||||
digest = sha256_file(artifact)
|
||||
now = dt.datetime.now(dt.timezone.utc).isoformat()
|
||||
|
||||
statement = {
|
||||
"_type": "https://in-toto.io/Statement/v1",
|
||||
"subject": [{"name": args.subject_name, "digest": {"sha256": digest}}],
|
||||
"predicateType": "https://slsa.dev/provenance/v1",
|
||||
"predicate": {
|
||||
"buildDefinition": {
|
||||
"buildType": "https://zeroclaw.dev/ci/release-fast",
|
||||
"externalParameters": {
|
||||
"repository": os.getenv("GITHUB_REPOSITORY", ""),
|
||||
"ref": os.getenv("GITHUB_REF", ""),
|
||||
"workflow": os.getenv("GITHUB_WORKFLOW", ""),
|
||||
},
|
||||
"internalParameters": {
|
||||
"sha": os.getenv("GITHUB_SHA", ""),
|
||||
"run_id": os.getenv("GITHUB_RUN_ID", ""),
|
||||
"run_attempt": os.getenv("GITHUB_RUN_ATTEMPT", ""),
|
||||
},
|
||||
"resolvedDependencies": [],
|
||||
},
|
||||
"runDetails": {
|
||||
"builder": {
|
||||
"id": f"https://github.com/{os.getenv('GITHUB_REPOSITORY', '')}/actions/runs/{os.getenv('GITHUB_RUN_ID', '')}"
|
||||
},
|
||||
"metadata": {
|
||||
"invocationId": os.getenv("GITHUB_RUN_ID", ""),
|
||||
"startedOn": now,
|
||||
"finishedOn": now,
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
out = Path(args.output)
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.write_text(json.dumps(statement, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+29
@@ -0,0 +1,29 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Install pinned gitleaks binary into a writable bin directory.
|
||||
# Usage: ./scripts/ci/install_gitleaks.sh <bin_dir> [version]
|
||||
|
||||
BIN_DIR="${1:-${RUNNER_TEMP:-/tmp}/bin}"
|
||||
VERSION="${2:-${GITLEAKS_VERSION:-v8.24.2}}"
|
||||
ARCHIVE="gitleaks_${VERSION#v}_linux_x64.tar.gz"
|
||||
CHECKSUMS="gitleaks_${VERSION#v}_checksums.txt"
|
||||
BASE_URL="https://github.com/gitleaks/gitleaks/releases/download/${VERSION}"
|
||||
|
||||
mkdir -p "${BIN_DIR}"
|
||||
tmp_dir="$(mktemp -d)"
|
||||
trap 'rm -rf "${tmp_dir}"' EXIT
|
||||
|
||||
curl -sSfL "${BASE_URL}/${ARCHIVE}" -o "${tmp_dir}/${ARCHIVE}"
|
||||
curl -sSfL "${BASE_URL}/${CHECKSUMS}" -o "${tmp_dir}/${CHECKSUMS}"
|
||||
|
||||
grep " ${ARCHIVE}\$" "${tmp_dir}/${CHECKSUMS}" > "${tmp_dir}/gitleaks.sha256"
|
||||
(
|
||||
cd "${tmp_dir}"
|
||||
sha256sum -c gitleaks.sha256
|
||||
)
|
||||
|
||||
tar -xzf "${tmp_dir}/${ARCHIVE}" -C "${tmp_dir}" gitleaks
|
||||
install -m 0755 "${tmp_dir}/gitleaks" "${BIN_DIR}/gitleaks"
|
||||
|
||||
echo "Installed gitleaks ${VERSION} to ${BIN_DIR}/gitleaks"
|
||||
Executable
+54
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Install a pinned syft binary into a writable bin directory.
|
||||
# Usage: ./scripts/ci/install_syft.sh <bin_dir> [version]
|
||||
|
||||
BIN_DIR="${1:-${RUNNER_TEMP:-/tmp}/bin}"
|
||||
VERSION="${2:-${SYFT_VERSION:-v1.42.1}}"
|
||||
|
||||
os_name="$(uname -s | tr '[:upper:]' '[:lower:]')"
|
||||
case "$os_name" in
|
||||
linux|darwin) ;;
|
||||
*)
|
||||
echo "Unsupported OS for syft installer: ${os_name}" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
arch_name="$(uname -m)"
|
||||
case "$arch_name" in
|
||||
x86_64|amd64) arch_name="amd64" ;;
|
||||
aarch64|arm64) arch_name="arm64" ;;
|
||||
armv7l) arch_name="armv7" ;;
|
||||
*)
|
||||
echo "Unsupported architecture for syft installer: ${arch_name}" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
|
||||
ARCHIVE="syft_${VERSION#v}_${os_name}_${arch_name}.tar.gz"
|
||||
CHECKSUMS="syft_${VERSION#v}_checksums.txt"
|
||||
BASE_URL="https://github.com/anchore/syft/releases/download/${VERSION}"
|
||||
|
||||
mkdir -p "${BIN_DIR}"
|
||||
tmp_dir="$(mktemp -d)"
|
||||
trap 'rm -rf "${tmp_dir}"' EXIT
|
||||
|
||||
curl -sSfL "${BASE_URL}/${ARCHIVE}" -o "${tmp_dir}/${ARCHIVE}"
|
||||
curl -sSfL "${BASE_URL}/${CHECKSUMS}" -o "${tmp_dir}/${CHECKSUMS}"
|
||||
|
||||
awk -v target="${ARCHIVE}" '$2 == target {print $1 " " $2}' "${tmp_dir}/${CHECKSUMS}" > "${tmp_dir}/syft.sha256"
|
||||
if [ ! -s "${tmp_dir}/syft.sha256" ]; then
|
||||
echo "Missing checksum entry for ${ARCHIVE} in ${CHECKSUMS}" >&2
|
||||
exit 1
|
||||
fi
|
||||
(
|
||||
cd "${tmp_dir}"
|
||||
sha256sum -c syft.sha256
|
||||
)
|
||||
|
||||
tar -xzf "${tmp_dir}/${ARCHIVE}" -C "${tmp_dir}" syft
|
||||
install -m 0755 "${tmp_dir}/syft" "${BIN_DIR}/syft"
|
||||
|
||||
echo "Installed syft ${VERSION} to ${BIN_DIR}/syft"
|
||||
@@ -1,589 +1,152 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build provider/model connectivity matrix for CI and local inspection.
|
||||
|
||||
The script runs `zeroclaw doctor models --provider <id>` against a contract-defined
|
||||
provider set, classifies failures, applies noise-control policy, and emits:
|
||||
- machine-readable JSON report
|
||||
- markdown summary (also appended to GITHUB_STEP_SUMMARY when available)
|
||||
- optional raw log for deep triage
|
||||
|
||||
Exit code is non-zero only when policy says the run should gate (unless --report-only).
|
||||
"""
|
||||
"""Probe provider API endpoints and generate connectivity matrix artifacts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import socket
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
AUTH_HINTS = [
|
||||
"401",
|
||||
"403",
|
||||
"unauthorized",
|
||||
"forbidden",
|
||||
"invalid api key",
|
||||
"requires api key",
|
||||
"api key",
|
||||
"token",
|
||||
"insufficient quota",
|
||||
"insufficient balance",
|
||||
"permission denied",
|
||||
]
|
||||
|
||||
RATE_LIMIT_HINTS = [
|
||||
"429",
|
||||
"rate limit",
|
||||
"too many requests",
|
||||
]
|
||||
|
||||
NETWORK_HINTS = [
|
||||
"timed out",
|
||||
"timeout",
|
||||
"network",
|
||||
"connection refused",
|
||||
"connection reset",
|
||||
"dns",
|
||||
"temporary failure in name resolution",
|
||||
"failed to connect",
|
||||
"tls",
|
||||
"certificate",
|
||||
"could not resolve host",
|
||||
"operation timed out",
|
||||
]
|
||||
|
||||
UNAVAILABLE_HINTS = [
|
||||
"404",
|
||||
"not found",
|
||||
"service unavailable",
|
||||
"provider returned an empty model list",
|
||||
"does not support live model discovery",
|
||||
"unsupported",
|
||||
]
|
||||
|
||||
MODEL_COUNT_PATTERNS = [
|
||||
re.compile(r"Refreshed '\\S+' model cache with (\\d+) models", re.IGNORECASE),
|
||||
re.compile(r"with (\\d+) models", re.IGNORECASE),
|
||||
]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProviderContract:
|
||||
name: str
|
||||
provider: str
|
||||
required: bool
|
||||
secret_env: str | None
|
||||
timeout_sec: int
|
||||
retries: int
|
||||
notes: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class ProviderResult:
|
||||
name: str
|
||||
provider: str
|
||||
required: bool
|
||||
secret_env: str | None
|
||||
status: str
|
||||
category: str
|
||||
gate: bool
|
||||
attempts: int
|
||||
timeout_sec: int
|
||||
retries: int
|
||||
message: str
|
||||
model_count: int | None
|
||||
started_at: str
|
||||
ended_at: str
|
||||
duration_ms: int
|
||||
notes: str
|
||||
|
||||
|
||||
def utc_now() -> str:
|
||||
return dt.datetime.now(tz=dt.timezone.utc).isoformat(timespec="seconds")
|
||||
|
||||
|
||||
def clip(text: str, max_chars: int = 280) -> str:
|
||||
clean = " ".join(text.strip().split())
|
||||
if len(clean) <= max_chars:
|
||||
return clean
|
||||
return clean[: max_chars - 3] + "..."
|
||||
|
||||
|
||||
def classify_failure(raw: str) -> str:
|
||||
lower = raw.lower()
|
||||
|
||||
if any(hint in lower for hint in RATE_LIMIT_HINTS):
|
||||
return "rate_limit"
|
||||
if any(hint in lower for hint in AUTH_HINTS):
|
||||
return "auth"
|
||||
if any(hint in lower for hint in NETWORK_HINTS):
|
||||
return "network"
|
||||
if any(hint in lower for hint in UNAVAILABLE_HINTS):
|
||||
return "unavailable"
|
||||
return "other"
|
||||
|
||||
|
||||
def parse_model_count(output: str) -> int | None:
|
||||
for pattern in MODEL_COUNT_PATTERNS:
|
||||
m = pattern.search(output)
|
||||
if m:
|
||||
try:
|
||||
return int(m.group(1))
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def load_contract(path: pathlib.Path) -> tuple[int, int, list[ProviderContract]]:
|
||||
raw = json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
version = int(raw.get("version", 1))
|
||||
threshold = int(raw.get("consecutive_transient_failures_to_escalate", 2))
|
||||
providers_raw = raw.get("providers", [])
|
||||
if not isinstance(providers_raw, list) or not providers_raw:
|
||||
raise ValueError("contract.providers must be a non-empty list")
|
||||
|
||||
providers: list[ProviderContract] = []
|
||||
for item in providers_raw:
|
||||
if not isinstance(item, dict):
|
||||
raise ValueError("contract.providers entries must be objects")
|
||||
|
||||
name = str(item.get("name", "")).strip()
|
||||
provider = str(item.get("provider", "")).strip()
|
||||
if not name or not provider:
|
||||
raise ValueError("provider entry requires non-empty 'name' and 'provider'")
|
||||
|
||||
timeout_sec = int(item.get("timeout_sec", 90))
|
||||
retries = int(item.get("retries", 2))
|
||||
providers.append(
|
||||
ProviderContract(
|
||||
name=name,
|
||||
provider=provider,
|
||||
required=bool(item.get("required", False)),
|
||||
secret_env=str(item["secret_env"]).strip() if item.get("secret_env") else None,
|
||||
timeout_sec=max(10, timeout_sec),
|
||||
retries=max(1, retries),
|
||||
notes=str(item.get("notes", "")).strip(),
|
||||
)
|
||||
)
|
||||
|
||||
return version, max(1, threshold), providers
|
||||
|
||||
|
||||
def load_state(path: pathlib.Path) -> dict[str, Any]:
|
||||
if not path.exists():
|
||||
return {"providers": {}}
|
||||
|
||||
def dns_check(hostname: str, port: int) -> tuple[bool, str]:
|
||||
try:
|
||||
parsed = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return {"providers": {}}
|
||||
|
||||
providers = parsed.get("providers")
|
||||
if not isinstance(providers, dict):
|
||||
providers = {}
|
||||
return {"providers": providers}
|
||||
socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM)
|
||||
return (True, "ok")
|
||||
except Exception as exc: # pragma: no cover - operational error surface
|
||||
return (False, str(exc))
|
||||
|
||||
|
||||
def save_state(path: pathlib.Path, state: dict[str, Any]) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(state, indent=2, ensure_ascii=True) + "\n", encoding="utf-8")
|
||||
def http_probe(url: str, method: str, timeout_s: int) -> tuple[bool, int | None, str, int]:
|
||||
req = urllib.request.Request(url=url, method=method, headers={"User-Agent": "zeroclaw-ci-probe/1.0"})
|
||||
start = time.perf_counter()
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=timeout_s) as resp:
|
||||
elapsed_ms = int((time.perf_counter() - start) * 1000)
|
||||
code = int(resp.getcode())
|
||||
# For connectivity probes, 2xx-4xx indicates endpoint is reachable.
|
||||
return (200 <= code < 500, code, "ok", elapsed_ms)
|
||||
except urllib.error.HTTPError as exc:
|
||||
elapsed_ms = int((time.perf_counter() - start) * 1000)
|
||||
code = int(exc.code)
|
||||
return (200 <= code < 500, code, f"http_error:{code}", elapsed_ms)
|
||||
except Exception as exc: # pragma: no cover - operational error surface
|
||||
elapsed_ms = int((time.perf_counter() - start) * 1000)
|
||||
return (False, None, str(exc), elapsed_ms)
|
||||
|
||||
|
||||
def run_probe(binary: str, contract: ProviderContract) -> tuple[bool, str, int | None, int, str]:
|
||||
"""Return (ok, category, model_count, attempts_used, message)."""
|
||||
env = os.environ.copy()
|
||||
|
||||
if contract.secret_env:
|
||||
value = env.get(contract.secret_env, "").strip()
|
||||
if value:
|
||||
env[contract.secret_env] = value
|
||||
|
||||
command = [binary, "doctor", "models", "--provider", contract.provider]
|
||||
|
||||
last_message = ""
|
||||
last_category = "other"
|
||||
last_model_count: int | None = None
|
||||
|
||||
for attempt in range(1, contract.retries + 1):
|
||||
try:
|
||||
proc = subprocess.run(
|
||||
command,
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=contract.timeout_sec,
|
||||
check=False,
|
||||
)
|
||||
combined = "\n".join([proc.stdout or "", proc.stderr or ""]).strip()
|
||||
if proc.returncode == 0:
|
||||
return (
|
||||
True,
|
||||
"ok",
|
||||
parse_model_count(combined),
|
||||
attempt,
|
||||
clip(combined, 360),
|
||||
)
|
||||
|
||||
last_message = clip(combined or f"command exited with code {proc.returncode}", 360)
|
||||
last_category = classify_failure(combined)
|
||||
last_model_count = parse_model_count(combined)
|
||||
|
||||
# Only retry transient classes.
|
||||
if last_category not in {"network", "rate_limit"}:
|
||||
return False, last_category, last_model_count, attempt, last_message
|
||||
|
||||
if attempt < contract.retries:
|
||||
time.sleep(min(5, attempt))
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
last_message = (
|
||||
f"probe timed out after {contract.timeout_sec}s for provider {contract.provider}"
|
||||
)
|
||||
last_category = "network"
|
||||
last_model_count = None
|
||||
|
||||
if attempt < contract.retries:
|
||||
time.sleep(min(5, attempt))
|
||||
|
||||
return False, last_category, last_model_count, contract.retries, last_message
|
||||
|
||||
|
||||
def build_markdown(
|
||||
report: dict[str, Any],
|
||||
binary: str,
|
||||
contract_path: pathlib.Path,
|
||||
report_only: bool,
|
||||
) -> str:
|
||||
def build_markdown(rows: list[dict], timeout_s: int, critical_failures: list[dict]) -> str:
|
||||
lines: list[str] = []
|
||||
lines.append("## Provider Connectivity Matrix")
|
||||
lines.append("# Provider Connectivity Matrix")
|
||||
lines.append("")
|
||||
lines.append(f"- Generated: `{report['generated_at']}`")
|
||||
lines.append(f"- Contract: `{contract_path}` (v{report['contract_version']})")
|
||||
lines.append(f"- Probe binary: `{binary}`")
|
||||
lines.append(f"- Mode: `{'report-only' if report_only else 'enforced'}`")
|
||||
lines.append(
|
||||
"- Summary: "
|
||||
f"{report['summary']['ok']} ok, "
|
||||
f"{report['summary']['failed']} failed, "
|
||||
f"{report['summary']['skipped']} skipped"
|
||||
)
|
||||
lines.append(
|
||||
"- Categories: "
|
||||
f"auth={report['summary']['categories']['auth']}, "
|
||||
f"network={report['summary']['categories']['network']}, "
|
||||
f"unavailable={report['summary']['categories']['unavailable']}, "
|
||||
f"rate_limit={report['summary']['categories']['rate_limit']}, "
|
||||
f"other={report['summary']['categories']['other']}"
|
||||
)
|
||||
lines.append(f"- Generated at: `{dt.datetime.now(dt.timezone.utc).isoformat()}`")
|
||||
lines.append(f"- Timeout per endpoint: `{timeout_s}s`")
|
||||
lines.append(f"- Total endpoints: `{len(rows)}`")
|
||||
lines.append(f"- Reachable endpoints: `{sum(1 for r in rows if r['reachable'])}`")
|
||||
lines.append(f"- Critical failures: `{len(critical_failures)}`")
|
||||
lines.append("")
|
||||
lines.append("| Provider | Required | Status | Category | Gate | Models | Attempts | Detail |")
|
||||
lines.append("| --- | --- | --- | --- | --- | ---: | ---: | --- |")
|
||||
|
||||
for item in report["providers"]:
|
||||
models = "-" if item["model_count"] is None else str(item["model_count"])
|
||||
lines.append("| Provider | Endpoint | Critical | DNS | HTTP | Reachable | Latency (ms) | Notes |")
|
||||
lines.append("| --- | --- | --- | --- | ---:| --- | ---:| --- |")
|
||||
for row in rows:
|
||||
lines.append(
|
||||
"| "
|
||||
f"{item['name']} (`{item['provider']}`)"
|
||||
" | "
|
||||
f"{'yes' if item['required'] else 'no'}"
|
||||
" | "
|
||||
f"{item['status']}"
|
||||
" | "
|
||||
f"{item['category']}"
|
||||
" | "
|
||||
f"{'yes' if item['gate'] else 'no'}"
|
||||
" | "
|
||||
f"{models}"
|
||||
" | "
|
||||
f"{item['attempts']}"
|
||||
" | "
|
||||
f"{item['message']}"
|
||||
" |"
|
||||
f"| `{row['provider']}` | `{row['url']}` | `{row['critical']}` | `{row['dns_ok']}` | "
|
||||
f"`{row['http_status'] if row['http_status'] is not None else 'n/a'}` | "
|
||||
f"`{row['reachable']}` | `{row['latency_ms']}` | {row['notes']} |"
|
||||
)
|
||||
|
||||
lines.append("")
|
||||
lines.append("### Local Inspection")
|
||||
lines.append("")
|
||||
lines.append("```bash")
|
||||
lines.append(
|
||||
"python3 scripts/ci/provider_connectivity_matrix.py "
|
||||
"--binary target/release-fast/zeroclaw "
|
||||
"--contract .github/connectivity/probe-contract.json "
|
||||
"--output-json connectivity-report.json "
|
||||
"--output-markdown connectivity-summary.md"
|
||||
)
|
||||
lines.append("```")
|
||||
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
if critical_failures:
|
||||
lines.append("## Critical Probe Failures")
|
||||
for row in critical_failures:
|
||||
lines.append(f"- `{row['provider']}` -> `{row['url']}` ({row['notes']})")
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser = argparse.ArgumentParser(description="Generate provider connectivity matrix.")
|
||||
parser.add_argument(
|
||||
"--contract",
|
||||
default=".github/connectivity/probe-contract.json",
|
||||
help="Path to connectivity probe contract JSON",
|
||||
"--config",
|
||||
default=".github/connectivity/providers.json",
|
||||
help="Path to providers connectivity config JSON",
|
||||
)
|
||||
parser.add_argument("--timeout", type=int, default=8, help="HTTP probe timeout in seconds")
|
||||
parser.add_argument("--output-json", required=True, help="Output JSON path")
|
||||
parser.add_argument("--output-md", required=True, help="Output markdown path")
|
||||
parser.add_argument(
|
||||
"--binary",
|
||||
default="zeroclaw",
|
||||
help="Path to zeroclaw binary used for probes",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--state-file",
|
||||
default=".ci/connectivity-state.json",
|
||||
help="State file for transient-failure tracking",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-json",
|
||||
default="connectivity-report.json",
|
||||
help="Output JSON report path",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-markdown",
|
||||
default="connectivity-summary.md",
|
||||
help="Output markdown summary path",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--raw-log",
|
||||
default=".ci/connectivity-raw.log",
|
||||
help="Output raw probe log path",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--report-only",
|
||||
"--fail-on-critical",
|
||||
action="store_true",
|
||||
help="Never fail the process, even if gate conditions are hit",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dry-run",
|
||||
action="store_true",
|
||||
help="Validate contract and emit empty report without running probes",
|
||||
help="Return non-zero if any critical endpoint is unreachable",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
contract_path = pathlib.Path(args.contract)
|
||||
output_json_path = pathlib.Path(args.output_json)
|
||||
output_markdown_path = pathlib.Path(args.output_markdown)
|
||||
state_path = pathlib.Path(args.state_file)
|
||||
raw_log_path = pathlib.Path(args.raw_log)
|
||||
config = json.loads(Path(args.config).read_text(encoding="utf-8"))
|
||||
timeout_s = int(config.get("global_timeout_seconds", args.timeout))
|
||||
providers = config.get("providers", [])
|
||||
|
||||
if not contract_path.exists():
|
||||
print(f"contract file not found: {contract_path}", file=sys.stderr)
|
||||
return 2
|
||||
rows: list[dict] = []
|
||||
for item in providers:
|
||||
provider = str(item.get("id", "")).strip()
|
||||
url = str(item.get("url", "")).strip()
|
||||
if not provider or not url:
|
||||
continue
|
||||
|
||||
try:
|
||||
contract_version, threshold, providers = load_contract(contract_path)
|
||||
except Exception as exc:
|
||||
print(f"invalid contract: {exc}", file=sys.stderr)
|
||||
return 2
|
||||
critical = bool(item.get("critical", False))
|
||||
method = str(item.get("method", "HEAD")).upper().strip()
|
||||
parsed = urllib.parse.urlparse(url)
|
||||
host = parsed.hostname or ""
|
||||
port = parsed.port or (443 if parsed.scheme == "https" else 80)
|
||||
|
||||
if not args.dry_run and not pathlib.Path(args.binary).exists() and "/" in args.binary:
|
||||
print(f"probe binary not found: {args.binary}", file=sys.stderr)
|
||||
return 2
|
||||
dns_ok, dns_note = dns_check(host, port)
|
||||
reachable = False
|
||||
http_status: int | None = None
|
||||
notes = dns_note
|
||||
latency_ms = 0
|
||||
|
||||
previous_state = load_state(state_path)
|
||||
current_state: dict[str, Any] = {"providers": {}}
|
||||
if dns_ok:
|
||||
reachable, http_status, notes, latency_ms = http_probe(url, method, timeout_s)
|
||||
if not reachable and method == "HEAD":
|
||||
# Some providers reject HEAD but respond to GET.
|
||||
reachable, http_status, notes, latency_ms = http_probe(url, "GET", timeout_s)
|
||||
|
||||
generated_at = utc_now()
|
||||
results: list[ProviderResult] = []
|
||||
raw_lines: list[str] = [
|
||||
f"# Connectivity probe raw log\n",
|
||||
f"generated_at={generated_at}\n",
|
||||
f"contract={contract_path}\n",
|
||||
f"binary={args.binary}\n",
|
||||
f"report_only={args.report_only}\n",
|
||||
f"threshold={threshold}\n",
|
||||
"\n",
|
||||
]
|
||||
rows.append(
|
||||
{
|
||||
"provider": provider,
|
||||
"url": url,
|
||||
"critical": critical,
|
||||
"dns_ok": dns_ok,
|
||||
"http_status": http_status,
|
||||
"reachable": bool(dns_ok and reachable),
|
||||
"latency_ms": latency_ms,
|
||||
"notes": notes,
|
||||
}
|
||||
)
|
||||
|
||||
if args.dry_run:
|
||||
for contract in providers:
|
||||
now = utc_now()
|
||||
results.append(
|
||||
ProviderResult(
|
||||
name=contract.name,
|
||||
provider=contract.provider,
|
||||
required=contract.required,
|
||||
secret_env=contract.secret_env,
|
||||
status="dry_run",
|
||||
category="other",
|
||||
gate=False,
|
||||
attempts=0,
|
||||
timeout_sec=contract.timeout_sec,
|
||||
retries=contract.retries,
|
||||
message="dry-run: probe skipped",
|
||||
model_count=None,
|
||||
started_at=now,
|
||||
ended_at=now,
|
||||
duration_ms=0,
|
||||
notes=contract.notes,
|
||||
)
|
||||
)
|
||||
else:
|
||||
for contract in providers:
|
||||
started = time.perf_counter()
|
||||
started_at = utc_now()
|
||||
|
||||
secret_value = ""
|
||||
if contract.secret_env:
|
||||
secret_value = os.environ.get(contract.secret_env, "").strip()
|
||||
|
||||
if contract.secret_env and not secret_value:
|
||||
category = "auth"
|
||||
status = "missing_secret_required" if contract.required else "skipped_missing_secret"
|
||||
gate = contract.required
|
||||
ended_at = utc_now()
|
||||
duration_ms = int((time.perf_counter() - started) * 1000)
|
||||
message = f"missing secret env: {contract.secret_env}"
|
||||
attempts = 0
|
||||
model_count = None
|
||||
else:
|
||||
ok, category, model_count, attempts, message = run_probe(args.binary, contract)
|
||||
status = "ok" if ok else "failed"
|
||||
|
||||
prev = previous_state["providers"].get(contract.provider, {})
|
||||
prev_transient = int(prev.get("consecutive_transient_failures", 0))
|
||||
|
||||
if ok:
|
||||
transient = 0
|
||||
elif category in {"network", "rate_limit"}:
|
||||
transient = prev_transient + 1
|
||||
else:
|
||||
transient = 0
|
||||
|
||||
immediate_gate = category in {"auth", "unavailable", "other"}
|
||||
transient_gate = category in {"network", "rate_limit"} and transient >= threshold
|
||||
gate = contract.required and (immediate_gate or transient_gate)
|
||||
|
||||
current_state["providers"][contract.provider] = {
|
||||
"name": contract.name,
|
||||
"last_status": status,
|
||||
"last_category": category,
|
||||
"last_message": message,
|
||||
"last_checked_at": utc_now(),
|
||||
"consecutive_transient_failures": transient,
|
||||
}
|
||||
|
||||
ended_at = utc_now()
|
||||
duration_ms = int((time.perf_counter() - started) * 1000)
|
||||
|
||||
if contract.provider not in current_state["providers"]:
|
||||
current_state["providers"][contract.provider] = {
|
||||
"name": contract.name,
|
||||
"last_status": status,
|
||||
"last_category": category,
|
||||
"last_message": message,
|
||||
"last_checked_at": utc_now(),
|
||||
"consecutive_transient_failures": 0,
|
||||
}
|
||||
|
||||
result = ProviderResult(
|
||||
name=contract.name,
|
||||
provider=contract.provider,
|
||||
required=contract.required,
|
||||
secret_env=contract.secret_env,
|
||||
status=status,
|
||||
category=category,
|
||||
gate=gate,
|
||||
attempts=attempts,
|
||||
timeout_sec=contract.timeout_sec,
|
||||
retries=contract.retries,
|
||||
message=clip(message),
|
||||
model_count=model_count,
|
||||
started_at=started_at,
|
||||
ended_at=ended_at,
|
||||
duration_ms=duration_ms,
|
||||
notes=contract.notes,
|
||||
)
|
||||
results.append(result)
|
||||
|
||||
raw_lines.append(
|
||||
f"[{result.ended_at}] provider={result.provider} status={result.status} "
|
||||
f"category={result.category} gate={result.gate} attempts={result.attempts} "
|
||||
f"duration_ms={result.duration_ms} message={result.message}\n"
|
||||
)
|
||||
|
||||
summary = {
|
||||
"ok": sum(1 for r in results if r.status == "ok"),
|
||||
"failed": sum(1 for r in results if r.status in {"failed", "missing_secret_required"}),
|
||||
"skipped": sum(
|
||||
1
|
||||
for r in results
|
||||
if r.status in {"skipped_missing_secret", "dry_run"}
|
||||
),
|
||||
"gate_failures": sum(1 for r in results if r.gate),
|
||||
"categories": {
|
||||
"auth": sum(1 for r in results if r.category == "auth"),
|
||||
"network": sum(1 for r in results if r.category == "network"),
|
||||
"unavailable": sum(1 for r in results if r.category == "unavailable"),
|
||||
"rate_limit": sum(1 for r in results if r.category == "rate_limit"),
|
||||
"other": sum(1 for r in results if r.category == "other"),
|
||||
},
|
||||
critical_failures = [r for r in rows if r["critical"] and not r["reachable"]]
|
||||
payload = {
|
||||
"generated_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"timeout_seconds": timeout_s,
|
||||
"total_endpoints": len(rows),
|
||||
"reachable_endpoints": sum(1 for r in rows if r["reachable"]),
|
||||
"critical_failures": len(critical_failures),
|
||||
"rows": rows,
|
||||
}
|
||||
|
||||
report = {
|
||||
"generated_at": generated_at,
|
||||
"contract_version": contract_version,
|
||||
"consecutive_transient_failures_to_escalate": threshold,
|
||||
"report_only": args.report_only,
|
||||
"summary": summary,
|
||||
"policy": {
|
||||
"required_immediate_gate_categories": ["auth", "unavailable", "other"],
|
||||
"required_transient_gate_categories": ["network", "rate_limit"],
|
||||
"required_transient_gate_threshold": threshold,
|
||||
"optional_provider_gating": "never",
|
||||
},
|
||||
"providers": [r.__dict__ for r in results],
|
||||
}
|
||||
json_path = Path(args.output_json)
|
||||
md_path = Path(args.output_md)
|
||||
json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
md_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
json_path.write_text(json.dumps(payload, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
md_path.write_text(build_markdown(rows, timeout_s, critical_failures), encoding="utf-8")
|
||||
|
||||
output_json_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_markdown_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
raw_log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
output_json_path.write_text(
|
||||
json.dumps(report, indent=2, ensure_ascii=True) + "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
markdown = build_markdown(
|
||||
report=report,
|
||||
binary=args.binary,
|
||||
contract_path=contract_path,
|
||||
report_only=args.report_only,
|
||||
)
|
||||
output_markdown_path.write_text(markdown, encoding="utf-8")
|
||||
raw_log_path.write_text("".join(raw_lines), encoding="utf-8")
|
||||
|
||||
save_state(state_path, current_state)
|
||||
|
||||
summary_path = os.environ.get("GITHUB_STEP_SUMMARY", "").strip()
|
||||
if summary_path:
|
||||
with open(summary_path, "a", encoding="utf-8") as fh:
|
||||
fh.write("\n")
|
||||
fh.write(markdown)
|
||||
|
||||
print(
|
||||
f"connectivity matrix complete: ok={summary['ok']} failed={summary['failed']} "
|
||||
f"skipped={summary['skipped']} gate_failures={summary['gate_failures']}"
|
||||
)
|
||||
print(f"report: {output_json_path}")
|
||||
print(f"summary: {output_markdown_path}")
|
||||
print(f"state: {state_path}")
|
||||
|
||||
if args.report_only or args.dry_run:
|
||||
return 0
|
||||
|
||||
return 1 if summary["gate_failures"] > 0 else 0
|
||||
if args.fail_on_critical and critical_failures:
|
||||
return 3
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
raise SystemExit(main())
|
||||
|
||||
Executable
+121
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Reproducible build probe:
|
||||
# - Build twice from clean state
|
||||
# - Compare artifact SHA256
|
||||
# - Emit JSON + markdown artifacts for auditability
|
||||
|
||||
PROFILE="${PROFILE:-release-fast}"
|
||||
BINARY_NAME="${BINARY_NAME:-zeroclaw}"
|
||||
OUTPUT_DIR="${OUTPUT_DIR:-artifacts}"
|
||||
FAIL_ON_DRIFT="${FAIL_ON_DRIFT:-false}"
|
||||
ALLOW_BUILD_ID_DRIFT="${ALLOW_BUILD_ID_DRIFT:-true}"
|
||||
|
||||
mkdir -p "${OUTPUT_DIR}"
|
||||
|
||||
host_target="$(rustc -vV | sed -n 's/^host: //p')"
|
||||
artifact_path="target/${host_target}/${PROFILE}/${BINARY_NAME}"
|
||||
|
||||
build_once() {
|
||||
local pass="$1"
|
||||
cargo clean
|
||||
cargo build --profile "${PROFILE}" --locked --target "${host_target}" --verbose
|
||||
if [ ! -f "${artifact_path}" ]; then
|
||||
echo "expected artifact not found: ${artifact_path}" >&2
|
||||
exit 2
|
||||
fi
|
||||
cp "${artifact_path}" "${OUTPUT_DIR}/repro-build-${pass}.bin"
|
||||
sha256sum "${OUTPUT_DIR}/repro-build-${pass}.bin" | awk '{print $1}'
|
||||
}
|
||||
|
||||
extract_build_id() {
|
||||
local bin="$1"
|
||||
if ! command -v readelf >/dev/null 2>&1; then
|
||||
echo ""
|
||||
return 0
|
||||
fi
|
||||
readelf -n "${bin}" 2>/dev/null | sed -n 's/^\s*Build ID: //p' | head -n 1
|
||||
}
|
||||
|
||||
is_build_id_only_drift() {
|
||||
local first="$1"
|
||||
local second="$2"
|
||||
|
||||
if ! command -v objcopy >/dev/null 2>&1; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
local tmp1 tmp2
|
||||
tmp1="$(mktemp)"
|
||||
tmp2="$(mktemp)"
|
||||
cp "${first}" "${tmp1}"
|
||||
cp "${second}" "${tmp2}"
|
||||
objcopy --remove-section .note.gnu.build-id "${tmp1}" >/dev/null 2>&1 || true
|
||||
objcopy --remove-section .note.gnu.build-id "${tmp2}" >/dev/null 2>&1 || true
|
||||
|
||||
if cmp -s "${tmp1}" "${tmp2}"; then
|
||||
rm -f "${tmp1}" "${tmp2}"
|
||||
return 0
|
||||
fi
|
||||
rm -f "${tmp1}" "${tmp2}"
|
||||
return 1
|
||||
}
|
||||
|
||||
sha1="$(build_once first)"
|
||||
sha2="$(build_once second)"
|
||||
|
||||
status="match"
|
||||
drift_reason="none"
|
||||
first_build_id=""
|
||||
second_build_id=""
|
||||
if [ "${sha1}" != "${sha2}" ]; then
|
||||
status="drift"
|
||||
drift_reason="binary_sha_mismatch"
|
||||
first_build_id="$(extract_build_id "${OUTPUT_DIR}/repro-build-first.bin")"
|
||||
second_build_id="$(extract_build_id "${OUTPUT_DIR}/repro-build-second.bin")"
|
||||
if is_build_id_only_drift "${OUTPUT_DIR}/repro-build-first.bin" "${OUTPUT_DIR}/repro-build-second.bin"; then
|
||||
status="drift_build_id_only"
|
||||
drift_reason="gnu_build_id_note_diff"
|
||||
fi
|
||||
fi
|
||||
|
||||
cat > "${OUTPUT_DIR}/reproducible-build.json" <<EOF
|
||||
{
|
||||
"schema_version": "zeroclaw.audit.v1",
|
||||
"event_type": "reproducible_build",
|
||||
"profile": "${PROFILE}",
|
||||
"target": "${host_target}",
|
||||
"binary": "${BINARY_NAME}",
|
||||
"first_sha256": "${sha1}",
|
||||
"second_sha256": "${sha2}",
|
||||
"status": "${status}",
|
||||
"drift_reason": "${drift_reason}",
|
||||
"allow_build_id_drift": "${ALLOW_BUILD_ID_DRIFT}",
|
||||
"first_build_id": "${first_build_id}",
|
||||
"second_build_id": "${second_build_id}"
|
||||
}
|
||||
EOF
|
||||
|
||||
cat > "${OUTPUT_DIR}/reproducible-build.md" <<EOF
|
||||
# Reproducible Build Check
|
||||
|
||||
- Profile: \`${PROFILE}\`
|
||||
- Target: \`${host_target}\`
|
||||
- Binary: \`${BINARY_NAME}\`
|
||||
- First SHA256: \`${sha1}\`
|
||||
- Second SHA256: \`${sha2}\`
|
||||
- Result: \`${status}\`
|
||||
- Drift reason: \`${drift_reason}\`
|
||||
- First Build ID: \`${first_build_id:-n/a}\`
|
||||
- Second Build ID: \`${second_build_id:-n/a}\`
|
||||
- Allow build-id-only drift: \`${ALLOW_BUILD_ID_DRIFT}\`
|
||||
EOF
|
||||
|
||||
if [ "${status}" = "drift" ] && [ "${FAIL_ON_DRIFT}" = "true" ]; then
|
||||
exit 3
|
||||
fi
|
||||
|
||||
if [ "${status}" = "drift_build_id_only" ] && [ "${FAIL_ON_DRIFT}" = "true" ] && [ "${ALLOW_BUILD_ID_DRIFT}" != "true" ]; then
|
||||
exit 4
|
||||
fi
|
||||
@@ -0,0 +1,209 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build and validate a rollback execution plan for CI/CD incidents."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import fnmatch
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def run_git(args: list[str], *, cwd: Path) -> str:
|
||||
proc = subprocess.run(
|
||||
["git", *args],
|
||||
cwd=str(cwd),
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"git {' '.join(args)} failed ({proc.returncode}): {proc.stderr.strip()}")
|
||||
return proc.stdout.strip()
|
||||
|
||||
|
||||
def resolve_target_ref(
|
||||
*,
|
||||
repo_root: Path,
|
||||
target_ref: str,
|
||||
tag_pattern: str,
|
||||
) -> tuple[str | None, str | None]:
|
||||
if target_ref:
|
||||
sha = run_git(["rev-parse", f"{target_ref}^{{commit}}"], cwd=repo_root)
|
||||
return (target_ref, sha)
|
||||
|
||||
# Prefer semantic version ordering for deterministic rollback target selection.
|
||||
refs = run_git(["tag", "--list", tag_pattern, "--sort=-version:refname"], cwd=repo_root)
|
||||
for ref in refs.splitlines():
|
||||
if not ref:
|
||||
continue
|
||||
try:
|
||||
sha = run_git(["rev-parse", f"{ref}^{{commit}}"], cwd=repo_root)
|
||||
except RuntimeError:
|
||||
continue
|
||||
return (ref, sha)
|
||||
|
||||
# Fallback for non-semver tag names.
|
||||
fallback_refs = run_git(
|
||||
[
|
||||
"for-each-ref",
|
||||
"--sort=-creatordate",
|
||||
"--format=%(refname:short)",
|
||||
"refs/tags",
|
||||
],
|
||||
cwd=repo_root,
|
||||
)
|
||||
for ref in fallback_refs.splitlines():
|
||||
if not ref or not fnmatch.fnmatch(ref, tag_pattern):
|
||||
continue
|
||||
try:
|
||||
sha = run_git(["rev-parse", f"{ref}^{{commit}}"], cwd=repo_root)
|
||||
except RuntimeError:
|
||||
continue
|
||||
return (ref, sha)
|
||||
return (None, None)
|
||||
|
||||
|
||||
def build_markdown(report: dict) -> str:
|
||||
lines: list[str] = []
|
||||
lines.append("# Rollback Guard Plan")
|
||||
lines.append("")
|
||||
lines.append(f"- Generated at: `{report['generated_at']}`")
|
||||
lines.append(f"- Branch: `{report['branch']}`")
|
||||
lines.append(f"- Mode: `{report['mode']}`")
|
||||
lines.append(f"- Current head: `{report['current_head_sha']}`")
|
||||
lines.append(f"- Target ref: `{report['target_ref'] or 'n/a'}`")
|
||||
lines.append(f"- Target sha: `{report['target_sha'] or 'n/a'}`")
|
||||
lines.append(f"- Ancestor check: `{report['ancestor_check']}`")
|
||||
lines.append(f"- Violations: `{len(report['violations'])}`")
|
||||
lines.append(f"- Warnings: `{len(report['warnings'])}`")
|
||||
lines.append("")
|
||||
|
||||
if report["violations"]:
|
||||
lines.append("## Violations")
|
||||
for item in report["violations"]:
|
||||
lines.append(f"- {item}")
|
||||
lines.append("")
|
||||
|
||||
if report["warnings"]:
|
||||
lines.append("## Warnings")
|
||||
for item in report["warnings"]:
|
||||
lines.append(f"- {item}")
|
||||
lines.append("")
|
||||
|
||||
lines.append("## Plan")
|
||||
lines.append(f"- Rollback strategy: `{report['strategy']}`")
|
||||
lines.append(f"- Allow non-ancestor target: `{report['allow_non_ancestor']}`")
|
||||
lines.append(f"- Ready to execute: `{report['ready_to_execute']}`")
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Validate rollback target and emit rollback execution plan.")
|
||||
parser.add_argument("--repo-root", default=".")
|
||||
parser.add_argument("--branch", default="dev")
|
||||
parser.add_argument("--mode", choices=("dry-run", "execute"), default="dry-run")
|
||||
parser.add_argument("--strategy", default="latest-release-tag")
|
||||
parser.add_argument("--target-ref", default="")
|
||||
parser.add_argument("--tag-pattern", default="v*")
|
||||
parser.add_argument("--allow-non-ancestor", action="store_true")
|
||||
parser.add_argument("--output-json", required=True)
|
||||
parser.add_argument("--output-md", required=True)
|
||||
parser.add_argument("--fail-on-violation", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
repo_root = Path(args.repo_root).resolve()
|
||||
out_json = Path(args.output_json)
|
||||
out_md = Path(args.output_md)
|
||||
|
||||
warnings: list[str] = []
|
||||
violations: list[str] = []
|
||||
|
||||
try:
|
||||
current_head_sha = run_git(["rev-parse", "HEAD"], cwd=repo_root)
|
||||
except RuntimeError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 2
|
||||
|
||||
try:
|
||||
target_ref, target_sha = resolve_target_ref(
|
||||
repo_root=repo_root,
|
||||
target_ref=args.target_ref.strip(),
|
||||
tag_pattern=args.tag_pattern,
|
||||
)
|
||||
except RuntimeError as exc:
|
||||
target_ref, target_sha = (args.target_ref.strip() or None, None)
|
||||
violations.append(f"Failed to resolve rollback target: {exc}")
|
||||
if not target_sha:
|
||||
violations.append(
|
||||
"Rollback target could not be resolved; provide `--target-ref` or ensure matching tags exist."
|
||||
)
|
||||
|
||||
ancestor_check = "unknown"
|
||||
if target_sha:
|
||||
proc = subprocess.run(
|
||||
["git", "merge-base", "--is-ancestor", target_sha, current_head_sha],
|
||||
cwd=str(repo_root),
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
)
|
||||
if proc.returncode == 0:
|
||||
ancestor_check = "pass"
|
||||
elif proc.returncode == 1:
|
||||
ancestor_check = "fail"
|
||||
msg = (
|
||||
f"Target `{target_ref}` ({target_sha}) is not an ancestor of current head "
|
||||
f"`{current_head_sha}`."
|
||||
)
|
||||
if args.allow_non_ancestor:
|
||||
warnings.append(msg)
|
||||
else:
|
||||
violations.append(msg)
|
||||
else:
|
||||
ancestor_check = "error"
|
||||
violations.append(f"Unable to evaluate ancestor relation: {proc.stderr.strip()}")
|
||||
|
||||
if target_sha == current_head_sha:
|
||||
warnings.append("Target SHA matches current head; rollback is a no-op.")
|
||||
|
||||
ready_to_execute = args.mode == "execute" and not violations
|
||||
|
||||
report = {
|
||||
"schema_version": "zeroclaw.audit.v1",
|
||||
"event_type": "rollback_guard",
|
||||
"generated_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"repo_root": str(repo_root),
|
||||
"branch": args.branch,
|
||||
"mode": args.mode,
|
||||
"strategy": args.strategy,
|
||||
"tag_pattern": args.tag_pattern,
|
||||
"target_ref": target_ref,
|
||||
"target_sha": target_sha,
|
||||
"current_head_sha": current_head_sha,
|
||||
"ancestor_check": ancestor_check,
|
||||
"allow_non_ancestor": args.allow_non_ancestor,
|
||||
"ready_to_execute": ready_to_execute,
|
||||
"warnings": warnings,
|
||||
"violations": violations,
|
||||
}
|
||||
|
||||
out_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_json.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
out_md.write_text(build_markdown(report), encoding="utf-8")
|
||||
|
||||
if args.fail_on_violation and violations:
|
||||
print("rollback guard violations found:", file=sys.stderr)
|
||||
for item in violations:
|
||||
print(f"- {item}", file=sys.stderr)
|
||||
return 3
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,243 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate gitleaks allowlist governance metadata and expiry policy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
import tomllib # Python 3.11+
|
||||
except ModuleNotFoundError: # pragma: no cover
|
||||
import tomli as tomllib # type: ignore
|
||||
|
||||
|
||||
TICKET_RE = re.compile(r"^[A-Z][A-Z0-9]+-\d+$")
|
||||
|
||||
|
||||
def parse_iso_date(raw: str) -> dt.date | None:
|
||||
try:
|
||||
return dt.date.fromisoformat(raw)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def likely_overbroad_pattern(pattern: str) -> bool:
|
||||
compact = pattern.strip()
|
||||
if compact in {".*", ".+"}:
|
||||
return True
|
||||
if compact.startswith(".*") and "/" not in compact:
|
||||
return True
|
||||
if compact.count(".*") >= 3:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def build_markdown(report: dict) -> str:
|
||||
lines: list[str] = []
|
||||
lines.append("# Secrets Governance Guard")
|
||||
lines.append("")
|
||||
lines.append(f"- Generated at: `{report['generated_at']}`")
|
||||
lines.append(f"- Gitleaks allowlist paths: `{report['allowlist_paths']}`")
|
||||
lines.append(f"- Gitleaks allowlist regexes: `{report['allowlist_regexes']}`")
|
||||
lines.append(f"- Governance entries: `{report['governance_entries']}`")
|
||||
lines.append(f"- Violations: `{len(report['violations'])}`")
|
||||
lines.append(f"- Warnings: `{len(report['warnings'])}`")
|
||||
lines.append("")
|
||||
if report["violations"]:
|
||||
lines.append("## Violations")
|
||||
for item in report["violations"]:
|
||||
lines.append(f"- {item}")
|
||||
lines.append("")
|
||||
if report["warnings"]:
|
||||
lines.append("## Warnings")
|
||||
for item in report["warnings"]:
|
||||
lines.append(f"- {item}")
|
||||
lines.append("")
|
||||
if not report["violations"] and not report["warnings"]:
|
||||
lines.append("No governance issues found.")
|
||||
lines.append("")
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def validate_metadata_entry(
|
||||
*,
|
||||
kind: str,
|
||||
entry: dict,
|
||||
warnings: list[str],
|
||||
violations: list[str],
|
||||
today: dt.date,
|
||||
warn_days: int,
|
||||
) -> str:
|
||||
pattern = str(entry.get("pattern", "")).strip()
|
||||
owner = str(entry.get("owner", "")).strip()
|
||||
reason = str(entry.get("reason", "")).strip()
|
||||
expires_on = str(entry.get("expires_on", "")).strip()
|
||||
ticket = str(entry.get("ticket", "")).strip()
|
||||
|
||||
if not pattern:
|
||||
violations.append(f"{kind}: metadata entry is missing required field `pattern`.")
|
||||
return ""
|
||||
if not owner:
|
||||
violations.append(f"{kind}: `{pattern}` is missing required field `owner`.")
|
||||
if not reason:
|
||||
violations.append(f"{kind}: `{pattern}` is missing required field `reason`.")
|
||||
elif len(reason) < 12:
|
||||
violations.append(
|
||||
f"{kind}: `{pattern}` reason is too short; provide actionable context."
|
||||
)
|
||||
if not expires_on:
|
||||
violations.append(f"{kind}: `{pattern}` is missing required field `expires_on`.")
|
||||
else:
|
||||
parsed = parse_iso_date(expires_on)
|
||||
if parsed is None:
|
||||
violations.append(
|
||||
f"{kind}: `{pattern}` has invalid `expires_on` date (`{expires_on}`). Use YYYY-MM-DD."
|
||||
)
|
||||
else:
|
||||
if parsed < today:
|
||||
violations.append(
|
||||
f"{kind}: `{pattern}` expired on `{expires_on}` and must be removed or renewed."
|
||||
)
|
||||
elif (parsed - today).days <= warn_days:
|
||||
warnings.append(
|
||||
f"{kind}: `{pattern}` expires soon on `{expires_on}`; review renewal/removal."
|
||||
)
|
||||
if not ticket:
|
||||
warnings.append(f"{kind}: `{pattern}` has no tracking `ticket`.")
|
||||
elif not TICKET_RE.fullmatch(ticket):
|
||||
warnings.append(f"{kind}: `{pattern}` ticket `{ticket}` does not match KEY-123 format.")
|
||||
if likely_overbroad_pattern(pattern):
|
||||
violations.append(f"{kind}: `{pattern}` appears over-broad and must be narrowed.")
|
||||
|
||||
return pattern
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Validate gitleaks allowlist governance policy.")
|
||||
parser.add_argument("--gitleaks-file", default=".gitleaks.toml")
|
||||
parser.add_argument(
|
||||
"--governance-file",
|
||||
default=".github/security/gitleaks-allowlist-governance.json",
|
||||
)
|
||||
parser.add_argument("--warn-days", type=int, default=21)
|
||||
parser.add_argument("--output-json", required=True)
|
||||
parser.add_argument("--output-md", required=True)
|
||||
parser.add_argument("--fail-on-violation", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
gitleaks_path = Path(args.gitleaks_file)
|
||||
governance_path = Path(args.governance_file)
|
||||
|
||||
gitleaks = tomllib.loads(gitleaks_path.read_text(encoding="utf-8"))
|
||||
governance = json.loads(governance_path.read_text(encoding="utf-8"))
|
||||
|
||||
allowlist = gitleaks.get("allowlist", {})
|
||||
configured_paths = [str(v) for v in allowlist.get("paths", []) if str(v).strip()]
|
||||
configured_regexes = [str(v) for v in allowlist.get("regexes", []) if str(v).strip()]
|
||||
|
||||
governance_paths = governance.get("paths", [])
|
||||
governance_regexes = governance.get("regexes", [])
|
||||
|
||||
warnings: list[str] = []
|
||||
violations: list[str] = []
|
||||
today = dt.datetime.now(dt.timezone.utc).date()
|
||||
|
||||
governed_path_patterns: set[str] = set()
|
||||
if not isinstance(governance_paths, list):
|
||||
violations.append("governance.paths must be an array.")
|
||||
governance_paths = []
|
||||
for entry in governance_paths:
|
||||
if not isinstance(entry, dict):
|
||||
violations.append("governance.paths entries must be objects.")
|
||||
continue
|
||||
pattern = validate_metadata_entry(
|
||||
kind="path",
|
||||
entry=entry,
|
||||
warnings=warnings,
|
||||
violations=violations,
|
||||
today=today,
|
||||
warn_days=args.warn_days,
|
||||
)
|
||||
if pattern:
|
||||
governed_path_patterns.add(pattern)
|
||||
|
||||
governed_regex_patterns: set[str] = set()
|
||||
if not isinstance(governance_regexes, list):
|
||||
violations.append("governance.regexes must be an array.")
|
||||
governance_regexes = []
|
||||
for entry in governance_regexes:
|
||||
if not isinstance(entry, dict):
|
||||
violations.append("governance.regexes entries must be objects.")
|
||||
continue
|
||||
pattern = validate_metadata_entry(
|
||||
kind="regex",
|
||||
entry=entry,
|
||||
warnings=warnings,
|
||||
violations=violations,
|
||||
today=today,
|
||||
warn_days=args.warn_days,
|
||||
)
|
||||
if pattern:
|
||||
governed_regex_patterns.add(pattern)
|
||||
|
||||
unmanaged_paths = sorted(set(configured_paths) - governed_path_patterns)
|
||||
unmanaged_regexes = sorted(set(configured_regexes) - governed_regex_patterns)
|
||||
stale_path_governance = sorted(governed_path_patterns - set(configured_paths))
|
||||
stale_regex_governance = sorted(governed_regex_patterns - set(configured_regexes))
|
||||
|
||||
for pattern in unmanaged_paths:
|
||||
violations.append(
|
||||
f"path: `{pattern}` exists in .gitleaks.toml allowlist but has no governance metadata."
|
||||
)
|
||||
for pattern in unmanaged_regexes:
|
||||
violations.append(
|
||||
f"regex: `{pattern}` exists in .gitleaks.toml allowlist but has no governance metadata."
|
||||
)
|
||||
for pattern in stale_path_governance:
|
||||
warnings.append(
|
||||
f"path: `{pattern}` exists in governance metadata but not in current .gitleaks.toml allowlist."
|
||||
)
|
||||
for pattern in stale_regex_governance:
|
||||
warnings.append(
|
||||
f"regex: `{pattern}` exists in governance metadata but not in current .gitleaks.toml allowlist."
|
||||
)
|
||||
|
||||
report = {
|
||||
"schema_version": "zeroclaw.audit.v1",
|
||||
"event_type": "secrets_governance_guard",
|
||||
"generated_at": dt.datetime.now(dt.timezone.utc).isoformat(),
|
||||
"gitleaks_file": str(gitleaks_path),
|
||||
"governance_file": str(governance_path),
|
||||
"allowlist_paths": len(configured_paths),
|
||||
"allowlist_regexes": len(configured_regexes),
|
||||
"governance_entries": len(governed_path_patterns) + len(governed_regex_patterns),
|
||||
"unmanaged_paths": unmanaged_paths,
|
||||
"unmanaged_regexes": unmanaged_regexes,
|
||||
"stale_path_governance": stale_path_governance,
|
||||
"stale_regex_governance": stale_regex_governance,
|
||||
"warnings": warnings,
|
||||
"violations": violations,
|
||||
}
|
||||
|
||||
output_json = Path(args.output_json)
|
||||
output_md = Path(args.output_md)
|
||||
output_json.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(report, indent=2, ensure_ascii=False) + "\n", encoding="utf-8")
|
||||
output_md.write_text(build_markdown(report), encoding="utf-8")
|
||||
|
||||
if args.fail_on_violation and violations:
|
||||
print("secrets governance violations found:", file=sys.stderr)
|
||||
for item in violations:
|
||||
print(f"- {item}", file=sys.stderr)
|
||||
return 3
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user