package main

import (
        "context"
        "log/slog"
        "os"
        "os/signal"
        "syscall"
        "time"

        "github.com/joho/godotenv"

        "github.com/gerrowadat/nomad-gitops/internal/config"
        "github.com/gerrowadat/nomad-gitops/internal/gitwatch"
        "github.com/gerrowadat/nomad-gitops/internal/nomad"
        "github.com/gerrowadat/nomad-gitops/internal/server"
)

// Injected at build time via -ldflags.
var (
        version   = "dev"
        commit    = "unknown"
        buildDate = "unknown"
)

func main() {
        // Load .env for local development. Non-fatal if the file is absent.
        if err := godotenv.Load(); err != nil && !os.IsNotExist(err) {
                slog.Warn("Error loading .env file", "err", err)
        }

        cfg, err := config.Load()
        if err != nil {
                slog.Error("Loading config", "err", err)
                os.Exit(1)
        }

        setupLogging(cfg.LogLevel)
        slog.Info("Starting nomad-gitops", "version", version, "commit", commit, "buildDate", buildDate)

        ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
        defer cancel()

        differ, err := nomad.NewDiffer(cfg)
        if err != nil {
                slog.Error("Creating Nomad differ", "err", err)
                os.Exit(1)
        }

        // onChange is called by the watcher whenever the branch HEAD advances.
        // We close over watcher, which is set below before Run is called.
        var watcher *gitwatch.Watcher
        onChange := func(newCommit string) {
                hclFiles, err := watcher.ReadHCLFiles()
                if err != nil {
                        slog.Error("Reading HCL files from repo", "err", err)
                        return
                }
                if err := differ.Check(hclFiles, newCommit); err != nil {
                        slog.Error("Running diff check", "err", err)
                }
        }

        watcher = gitwatch.New(cfg, onChange)
        // The differ reads prior git state (via the watcher) to tell whether drift
        // pre-dates a job's opt-in.
        differ.SetHistorySource(watcher)

        if err := watcher.Clone(ctx); err != nil {
                slog.Error("Cloning repository", "err", err)
                os.Exit(1)
        }

        // Run an initial diff check immediately after clone.
        onChange(watcher.LastCommit())

        // Periodic diff checks independent of git changes (catches Nomad-side drift).
        go func() {
                ticker := time.NewTicker(cfg.DiffInterval)
                defer ticker.Stop()
                for {
                        select {
                        case <-ctx.Done():
                                return
                        case <-ticker.C:
                                commit, _ := watcher.Status()
                                hclFiles, err := watcher.ReadHCLFiles()
                                if err != nil {
                                        slog.Error("Reading HCL files for periodic check", "err", err)
                                        continue
                                }
                                if err := differ.Check(hclFiles, commit); err != nil {
                                        slog.Error("Periodic diff check failed", "err", err)
                                }
                        }
                }
        }()

        // Git staleness checker: triggers a fetch when the repo has not been
        // successfully fetched within MaxGitStaleness. Disabled when zero.
        if cfg.MaxGitStaleness > 0 {
                go func() {
                        checkInterval := cfg.MaxGitStaleness / 2
                        if checkInterval < 10*time.Second {
                                checkInterval = 10 * time.Second
                        }
                        ticker := time.NewTicker(checkInterval)
                        defer ticker.Stop()
                        for {
                                select {
                                case <-ctx.Done():
                                        return
                                case <-ticker.C:
                                        _, lastGitUpdate := watcher.Status()
                                        if !lastGitUpdate.IsZero() && time.Since(lastGitUpdate) > cfg.MaxGitStaleness {
                                                slog.Info("Git repo is stale, triggering refresh", "age", time.Since(lastGitUpdate), "max", cfg.MaxGitStaleness)
                                                watcher.TriggerStale()
                                        }
                                }
                        }
                }()
        }

        // Nomad staleness checker: forces a diff check when Nomad state has not
        // been checked within MaxNomadStaleness. Disabled when zero.
        if cfg.MaxNomadStaleness > 0 {
                go func() {
                        checkInterval := cfg.MaxNomadStaleness / 2
                        if checkInterval < 10*time.Second {
                                checkInterval = 10 * time.Second
                        }
                        ticker := time.NewTicker(checkInterval)
                        defer ticker.Stop()
                        for {
                                select {
                                case <-ctx.Done():
                                        return
                                case <-ticker.C:
                                        _, lastNomadCheck, _ := differ.Diffs()
                                        if !lastNomadCheck.IsZero() && time.Since(lastNomadCheck) > cfg.MaxNomadStaleness {
                                                slog.Info("Nomad state is stale, forcing diff check", "age", time.Since(lastNomadCheck), "max", cfg.MaxNomadStaleness)
                                                commit, _ := watcher.Status()
                                                hclFiles, err := watcher.ReadHCLFiles()
                                                if err != nil {
                                                        slog.Error("Reading HCL files for staleness check", "err", err)
                                                        continue
                                                }
                                                if err := differ.ForceCheck(hclFiles, commit); err != nil {
                                                        slog.Error("Staleness diff check failed", "err", err)
                                                }
                                        }
                                }
                        }
                }()
        }

        // Watcher polls git and triggers onChange on new commits.
        go watcher.Run(ctx)

        // Applier drains the GitOps update queue. With the default
        // --default-update-policy=none nothing is ever enqueued, so this loop
        // idles unless jobs opt in via meta or the default policy is raised.
        go differ.RunApplier(ctx)

        // Token refresher keeps a file-sourced Nomad token (workload identity)
        // current. No-op for a static token or no token.
        go differ.RunTokenRefresher(ctx)

        srv := server.New(cfg, differ, watcher, server.BuildInfo{
                Version:   version,
                Commit:    commit,
                BuildDate: buildDate,
        })
        if err := srv.Run(ctx); err != nil {
                slog.Error("HTTP server error", "err", err)
                os.Exit(1)
        }
}

func setupLogging(level string) {
        var l slog.Level
        switch level {
        case "debug":
                l = slog.LevelDebug
        case "warn":
                l = slog.LevelWarn
        case "error":
                l = slog.LevelError
        default:
                l = slog.LevelInfo
        }
        slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stderr, &slog.HandlerOptions{Level: l})))
}

package config

import (
        "flag"
        "fmt"
        "os"
        "strings"
        "time"
)

type Config struct {
        // Git
        RepoURL              string
        Branch               string
        PollInterval         time.Duration
        HCLDir               string
        GitToken             string
        GitSSHKeyPath        string
        GitSSHKeyPass        string
        GitSSHKnownHostsFile string

        // Nomad
        NomadAddr      string
        NomadToken     string
        NomadNamespace string

        // NomadTokenFile is a path to a file containing a Nomad ACL token SecretID,
        // re-read periodically so a rotating token stays current. Use it for a real
        // SecretID written to a file (e.g. by a sidecar). Note: this must be a
        // 36-char ACL SecretID, not a workload-identity JWT — a raw WI JWT is
        // rejected by Nomad's Job.Plan RPC (see NomadLoginAuthMethod).
        NomadTokenFile string
        // NomadTokenPollInterval is how often the token file is re-read for changes.
        NomadTokenPollInterval time.Duration

        // NomadLoginAuthMethod, when set, enables Nomad workload-identity login: the
        // identity JWT (NomadLoginJWTFile) is exchanged for a real ACL token via
        // POST /v1/acl/login against this JWT auth method, and re-exchanged before
        // it expires. This is the working way to use workload identity — a raw WI
        // JWT authenticates read RPCs but is rejected by Job.Plan, which
        // nomad-gitops needs for every drift check (issue #74).
        NomadLoginAuthMethod string
        // NomadLoginJWTFile is the path to the workload-identity JWT to exchange.
        // Defaults to ${NOMAD_SECRETS_DIR}/nomad_token; point it at a named
        // identity's file (nomad_<name>.jwt) when the auth method's audience does
        // not match the default identity.
        NomadLoginJWTFile string

        // Server
        ListenAddr    string
        WebhookSecret string
        WebhookPath   string
        APIKey        string // PSK for /api/ endpoints; empty disables the API

        // Diff
        DiffInterval    time.Duration
        IncludeDeadJobs bool
        RedactSecrets   bool

        // Apply (GitOps mutation)
        DefaultUpdatePolicy string
        EnableJobCreation   bool
        ApplyInterval       time.Duration

        // Managed-meta-only changes: a diff confined to nomad-gitops's own
        // meta keys (e.g. gitops_managed). By default these neither trigger an
        // update nor count as drift; the keys converge opportunistically on the
        // next real update.
        ApplyMetaOnlyChanges bool
        CountMetaOnlyChanges bool

        // ApplyExistingDrift controls whether drift that already existed when a
        // change widened a job's scope is applied. Scope widens two ways, treated
        // the same: a job gains the managed meta tag (enablement), or its update
        // policy is widened to cover drift it was deferring (e.g. image-only → full).
        // Off by default: a scope change does not retroactively mutate the job; only
        // changes committed after it apply.
        ApplyExistingDrift bool

        // Deregistration of jobs removed from the repo (file deleted or job
        // renamed). Off by default; the one destructive write nomad-gitops can
        // make, so heavily gated.
        EnableDeregister bool
        DeregisterPurge  bool
        DeregisterGrace  time.Duration

        // FlapGuard controls how nomad-gitops avoids re-applying a job spec that
        // a recent Nomad job version already failed to deploy (the
        // apply→fail→revert→re-apply loop). One of: history (Approach A: compare
        // spec fingerprints against Nomad's in-cluster version history, ephemeral
        // and GC-bounded), tag (Approach B: additionally tag the failed version so
        // the block survives version GC), or off (disabled). Per-job overridable
        // via the <prefix>_flap_guard meta key. Only applies to deployment-producing
        // jobs (service jobs with an update stanza and health checks).
        FlapGuard string

        // AllowRollback enables active rollback: for managed deployment-producing
        // jobs whose update stanza does not set auto_revert, nomad-gitops reverts
        // the job to its last stable version when a deployment fails. Off by
        // default. Per-job overridable via the <prefix>_rollback meta key. Where a
        // job's update stanza sets auto_revert=true, Nomad's own rollback always
        // wins and nomad-gitops stands down.
        AllowRollback bool

        // Job selection. Git is always the source of truth for nomad-gitops's
        // own meta keys: when a job has an HCL file in the repo, that file alone
        // decides selection and policy. There is deliberately no flag to invert
        // this.
        JobSelectorGlob   string
        ManagedMetaPrefix string

        // Staleness
        MaxGitStaleness   time.Duration
        MaxNomadStaleness time.Duration

        // RecloneInterval is how often to discard the in-memory git clone and
        // fetch a fresh one, reclaiming the git object store that grows as pulls
        // accumulate history over a long-running process. 0 disables reclones.
        RecloneInterval time.Duration

        // Logging
        LogLevel string
}

// Load parses flags from os.Args and falls back to environment variables.
func Load() (*Config, error) {
        return LoadFromArgs(flag.CommandLine, os.Args[1:])
}

// LoadFromArgs registers flags on fs and parses args.
// Tests pass a fresh flag.NewFlagSet to avoid touching flag.CommandLine.
func LoadFromArgs(fs *flag.FlagSet, args []string) (*Config, error) {
        c := &Config{}

        fs.StringVar(&c.RepoURL, "repo-url", envOrDefault("GIT_REPO_URL", ""), "Remote git repo URL (required)")
        fs.StringVar(&c.Branch, "branch", envOrDefault("GIT_BRANCH", "main"), "Branch to watch")
        fs.DurationVar(&c.PollInterval, "poll-interval", envDurationOrDefault("POLL_INTERVAL", 5*time.Minute), "Git poll interval (e.g. 5m, 30s)")
        fs.StringVar(&c.HCLDir, "hcl-dir", envOrDefault("HCL_DIR", ""), "Directory within repo containing HCL job files (empty = repo root)")
        fs.StringVar(&c.GitToken, "git-token", envOrDefault("GIT_TOKEN", ""), "Git HTTP token for private repos (e.g. GitHub PAT)")
        fs.StringVar(&c.GitSSHKeyPath, "git-ssh-key", envOrDefault("GIT_SSH_KEY", ""), "Path to SSH private key for git auth")
        fs.StringVar(&c.GitSSHKeyPass, "git-ssh-key-password", envOrDefault("GIT_SSH_KEY_PASSWORD", ""), "SSH private key passphrase")
        fs.StringVar(&c.GitSSHKnownHostsFile, "git-ssh-known-hosts", envOrDefault("GIT_SSH_KNOWN_HOSTS", ""), "Path to known_hosts file for SSH host key verification (defaults to ~/.ssh/known_hosts; set to empty string to use system defaults)")

        fs.StringVar(&c.NomadAddr, "nomad-addr", envOrDefault("NOMAD_ADDR", "http://127.0.0.1:4646"), "Nomad API address")
        fs.StringVar(&c.NomadToken, "nomad-token", envOrDefault("NOMAD_TOKEN", ""), "Nomad ACL token (static SecretID). Intended for manual running and testing; for a deployment under Nomad, use workload identity (see --nomad-login-auth-method).")
        fs.StringVar(&c.NomadTokenFile, "nomad-token-file", envOrDefault("NOMAD_TOKEN_FILE", ""), "Path to a file containing a Nomad ACL token SecretID, re-read periodically so a rotating token stays current. Must be a 36-char SecretID, not a workload-identity JWT. Takes precedence over --nomad-token.")
        fs.DurationVar(&c.NomadTokenPollInterval, "nomad-token-poll-interval", envDurationOrDefault("NOMAD_TOKEN_POLL_INTERVAL", 30*time.Second), "How often to re-read the Nomad token file (--nomad-token-file) for a rotated token.")
        fs.StringVar(&c.NomadLoginAuthMethod, "nomad-login-auth-method", envOrDefault("NOMAD_LOGIN_AUTH_METHOD", ""), "Enable Nomad workload-identity login: name of the JWT ACL auth method to exchange the identity JWT (--nomad-login-jwt-file) for an ACL token via /v1/acl/login, re-exchanged before it expires. This is the working way to use workload identity — a raw WI JWT is rejected by Nomad's Job.Plan RPC.")
        fs.StringVar(&c.NomadLoginJWTFile, "nomad-login-jwt-file", envOrDefault("NOMAD_LOGIN_JWT_FILE", ""), "Path to the workload-identity JWT to exchange (login mode). Defaults to ${NOMAD_SECRETS_DIR}/nomad_token; point it at a named identity's file (nomad_<name>.jwt) when the auth method audience does not match the default identity.")
        fs.StringVar(&c.NomadNamespace, "nomad-namespace", envOrDefault("NOMAD_NAMESPACE", "default"), "Nomad namespace")

        fs.StringVar(&c.ListenAddr, "listen-addr", envOrDefault("LISTEN_ADDR", ":8080"), "HTTP listen address")
        fs.StringVar(&c.WebhookSecret, "webhook-secret", envOrDefault("WEBHOOK_SECRET", ""), "GitHub webhook HMAC secret")
        fs.StringVar(&c.WebhookPath, "webhook-path", envOrDefault("WEBHOOK_PATH", "/webhook"), "HTTP path for webhook endpoint")
        fs.StringVar(&c.APIKey, "api-key", envOrDefault("API_KEY", ""), "Pre-shared key for /api/ endpoints (Bearer token). Empty disables the JSON API.")

        fs.DurationVar(&c.DiffInterval, "diff-interval", envDurationOrDefault("DIFF_INTERVAL", time.Minute), "How often to run a diff check regardless of git changes")
        fs.BoolVar(&c.IncludeDeadJobs, "include-dead-jobs", envBoolOrDefault("INCLUDE_DEAD_JOBS", false), "Treat dead Nomad jobs like running ones (by default dead jobs are treated as missing)")
        fs.BoolVar(&c.RedactSecrets, "redact-secrets", envBoolOrDefault("REDACT_SECRETS", true), "Redact potentially sensitive plan-diff values (env vars, template bodies, fields with secret-like names) before storing and rendering diffs")
        fs.StringVar(&c.DefaultUpdatePolicy, "default-update-policy", envOrDefault("DEFAULT_UPDATE_POLICY", "none"), "Update policy for managed jobs without an explicit <prefix>_update_policy meta key: none (detect only), image-only (apply drift confined to Docker image fields), full (apply any drift)")
        fs.BoolVar(&c.EnableJobCreation, "enable-job-creation", envBoolOrDefault("ENABLE_JOB_CREATION", false), "Allow registering jobs that exist in Git but not in Nomad (first-time registration). Off by default; requires an effective update policy of full for the job.")
        fs.DurationVar(&c.ApplyInterval, "apply-interval", envDurationOrDefault("APPLY_INTERVAL", 10*time.Second), "Fallback cadence of the apply loop; enqueued updates are also applied immediately")
        fs.BoolVar(&c.ApplyMetaOnlyChanges, "apply-meta-only-changes", envBoolOrDefault("APPLY_META_ONLY_CHANGES", false), "Apply a diff whose only change is to nomad-gitops's own meta keys (e.g. gitops_managed). Off by default: re-registering a running job just to push these keys is disruptive and unnecessary (the HCL is already authoritative), so they ride along the next real update instead.")
        fs.BoolVar(&c.CountMetaOnlyChanges, "count-meta-only-changes", envBoolOrDefault("COUNT_META_ONLY_CHANGES", false), "Count a managed-meta-only diff as drift (surface it on /diffs, /healthz, and the drift metrics). Off by default so these expected differences do not trigger drift alerts.")
        fs.BoolVar(&c.ApplyExistingDrift, "apply-existing-drift", envBoolOrDefault("APPLY_EXISTING_DRIFT", false), "When a change widens a job's scope, apply drift that already existed at that moment. Scope widens two ways, treated the same: a job gains the managed meta tag (enablement), or its update policy is widened to cover drift it was deferring (e.g. image-only → full applying a non-image change committed earlier). Off by default (conservative): a scope change does not retroactively mutate the job; only changes committed after it apply. Drift reconciles normally when scope is unchanged.")
        fs.BoolVar(&c.EnableDeregister, "enable-deregister", envBoolOrDefault("ENABLE_DEREGISTER", false), "Deregister jobs that were removed from the repo entirely (HCL file deleted or job renamed) while still running in Nomad. Off by default. Only ever acts on a job carrying gitops_managed=true in its live meta whose effective update policy is full, and only after it has been continuously orphaned for --deregister-grace. Removing only the gitops_managed tag (with the job still in the repo) never deregisters — it just stops management.")
        fs.BoolVar(&c.DeregisterPurge, "deregister-purge", envBoolOrDefault("DEREGISTER_PURGE", false), "When deregistering, purge the job from Nomad's state immediately instead of a graceful stop (which leaves it queryable and garbage-collected later). Off by default.")
        fs.DurationVar(&c.DeregisterGrace, "deregister-grace", envDurationOrDefault("DEREGISTER_GRACE", 5*time.Minute), "How long a job must be continuously orphaned (running in Nomad, removed from the repo) before it is deregistered. Absorbs transient renames and mid-edit commits.")
        fs.StringVar(&c.FlapGuard, "flap-guard", envOrDefault("FLAP_GUARD", "history"), "How to avoid re-applying a spec a recent Nomad job version already failed to deploy (the apply/fail/revert/re-apply loop): history (compare spec fingerprints against Nomad's version history; ephemeral, lost when Nomad GCs old versions), tag (additionally tag the failed version so the block survives GC), or off (disabled). Per-job overridable via the <prefix>_flap_guard meta key. Only applies to deployment-producing jobs.")
        fs.BoolVar(&c.AllowRollback, "allow-rollback", envBoolOrDefault("ALLOW_ROLLBACK", false), "Enable active rollback: for managed deployment-producing jobs whose update stanza does not set auto_revert, revert to the last stable version when a deployment fails. Off by default. Per-job overridable via the <prefix>_rollback meta key. Where the job's update stanza sets auto_revert=true, Nomad's own rollback wins and nomad-gitops stands down.")
        fs.StringVar(&c.JobSelectorGlob, "job-selector-glob", envOrDefault("JOB_SELECTOR_GLOB", ""), "Glob pattern selecting jobs by name (e.g. 'myprefix-*', '*' for all). Jobs matching either this or --managed-meta-prefix are watched. Empty means no glob selection.")
        fs.StringVar(&c.ManagedMetaPrefix, "managed-meta-prefix", envOrDefault("MANAGED_META_PREFIX", "gitops"), "Prefix for job meta keys used by nomad-gitops (e.g. 'gitops' means 'gitops_managed = true' in a job's HCL opts it in). Git is always the source of truth for these keys: when a job has an HCL file, the live job's keys are ignored for selection. Empty disables meta-based selection.")
        fs.DurationVar(&c.MaxGitStaleness, "max-git-staleness", envDurationOrDefault("MAX_GIT_STALENESS", 0), "Maximum time since last successful git fetch before forcing a refresh (0 disables)")
        fs.DurationVar(&c.MaxNomadStaleness, "max-nomad-staleness", envDurationOrDefault("MAX_NOMAD_STALENESS", 0), "Maximum time since last successful Nomad diff check before forcing a refresh (0 disables)")
        fs.DurationVar(&c.RecloneInterval, "reclone-interval", envDurationOrDefault("RECLONE_INTERVAL", 24*time.Hour), "How often to discard and re-fetch the in-memory git clone to reclaim memory that grows as pulls accumulate history (0 disables)")

        fs.StringVar(&c.LogLevel, "log-level", envOrDefault("LOG_LEVEL", "info"), "Log level: debug, info, warn, error")

        if err := fs.Parse(args); err != nil {
                return nil, fmt.Errorf("parsing flags: %w", err)
        }

        if c.RepoURL == "" {
                return nil, fmt.Errorf("--repo-url / GIT_REPO_URL is required")
        }

        // A git token over plain HTTP is sent in cleartext to the remote.
        if c.GitToken != "" && strings.HasPrefix(strings.ToLower(c.RepoURL), "http://") {
                return nil, fmt.Errorf("--git-token / GIT_TOKEN cannot be used with a plain http:// repo URL: the token would be sent in cleartext; use https:// or SSH instead")
        }

        switch c.DefaultUpdatePolicy {
        case "none", "image-only", "full":
        default:
                return nil, fmt.Errorf("--default-update-policy / DEFAULT_UPDATE_POLICY must be one of none, image-only, full; got %q", c.DefaultUpdatePolicy)
        }

        switch c.FlapGuard {
        case "history", "tag", "off":
        default:
                return nil, fmt.Errorf("--flap-guard / FLAP_GUARD must be one of history, tag, off; got %q", c.FlapGuard)
        }

        // Tag mode builds failed-version tag names from the managed-meta prefix
        // (<prefix>-failed-<fingerprint>) and recognises them by that prefix. With
        // an empty prefix the tag name would start with "-failed-" and could never
        // be recognised again, so durable blocking would silently not work.
        if c.FlapGuard == "tag" && c.ManagedMetaPrefix == "" {
                return nil, fmt.Errorf("--flap-guard=tag requires --managed-meta-prefix / MANAGED_META_PREFIX to be non-empty: failed-version tag names are derived from the prefix")
        }

        return c, nil
}

func envOrDefault(key, def string) string {
        if v := os.Getenv(key); v != "" {
                return v
        }
        return def
}

func envDurationOrDefault(key string, def time.Duration) time.Duration {
        if v := os.Getenv(key); v != "" {
                d, err := time.ParseDuration(v)
                if err == nil {
                        return d
                }
        }
        return def
}

func envBoolOrDefault(key string, def bool) bool {
        if v := os.Getenv(key); v != "" {
                switch strings.ToLower(v) {
                case "true", "1", "yes":
                        return true
                case "false", "0", "no":
                        return false
                }
        }
        return def
}

// Package gitwatch clones a remote git repo into memory and watches it for
// changes, triggering a callback whenever the watched branch advances.
package gitwatch

import (
        "context"
        "fmt"
        "log/slog"
        "strings"
        "sync"
        "time"

        "github.com/go-git/go-billy/v5/memfs"
        "github.com/go-git/go-git/v5"
        "github.com/go-git/go-git/v5/plumbing"
        "github.com/go-git/go-git/v5/plumbing/object"
        "github.com/go-git/go-git/v5/plumbing/transport"
        githttp "github.com/go-git/go-git/v5/plumbing/transport/http"
        gitssh "github.com/go-git/go-git/v5/plumbing/transport/ssh"
        "github.com/go-git/go-git/v5/storage/memory"
        "github.com/prometheus/client_golang/prometheus"
        "github.com/prometheus/client_golang/prometheus/promauto"

        "github.com/gerrowadat/nomad-gitops/internal/config"
)

// Watcher holds a live in-memory clone of a git repo and keeps it up to date.
type Watcher struct {
        cfg *config.Config

        mu         sync.RWMutex
        repo       *git.Repository
        lastCommit string
        lastUpdate time.Time

        triggerCh chan struct{}
        onChange  func(commit string)

        gitFetches     prometheus.Counter
        gitFetchErrors prometheus.Counter
        gitLastUpdate  prometheus.Gauge
        staleRefreshes prometheus.Counter
}

// New creates a Watcher that registers metrics into the default Prometheus registry.
func New(cfg *config.Config, onChange func(commit string)) *Watcher {
        return NewWithRegistry(cfg, onChange, prometheus.DefaultRegisterer)
}

// NewWithRegistry creates a Watcher that registers metrics into reg.
// Use this in tests to avoid duplicate-registration panics.
func NewWithRegistry(cfg *config.Config, onChange func(commit string), reg prometheus.Registerer) *Watcher {
        f := promauto.With(reg)
        return &Watcher{
                cfg:       cfg,
                triggerCh: make(chan struct{}, 1),
                onChange:  onChange,
                gitFetches: f.NewCounter(prometheus.CounterOpts{
                        Name: "nomad_gitops_git_fetches_total",
                        Help: "Total number of remote git fetch/clone attempts.",
                }),
                gitFetchErrors: f.NewCounter(prometheus.CounterOpts{
                        Name: "nomad_gitops_git_fetch_errors_total",
                        Help: "Total number of remote git fetch/clone failures.",
                }),
                gitLastUpdate: f.NewGauge(prometheus.GaugeOpts{
                        Name: "nomad_gitops_git_last_update_timestamp_seconds",
                        Help: "Unix timestamp of the most recent successful git fetch.",
                }),
                staleRefreshes: f.NewCounter(prometheus.CounterOpts{
                        Name: "nomad_gitops_git_staleness_refreshes_total",
                        Help: "Total number of git fetches triggered by the staleness check.",
                }),
        }
}

// cloneInto performs a fresh clone into a new in-memory store and returns the
// repository and its HEAD commit. It touches no Watcher state beyond the fetch
// metrics, so it is safe to call while the current w.repo is still in use by a
// concurrent reader; the caller decides when to swap the result in.
func (w *Watcher) cloneInto(ctx context.Context) (*git.Repository, string, error) {
        auth, err := w.buildAuth()
        if err != nil {
                return nil, "", fmt.Errorf("building git auth: %w", err)
        }

        w.gitFetches.Inc()

        storer := memory.NewStorage()
        fs := memfs.New()

        repo, err := git.CloneContext(ctx, storer, fs, &git.CloneOptions{
                URL:           w.cfg.RepoURL,
                ReferenceName: plumbing.NewBranchReferenceName(w.cfg.Branch),
                SingleBranch:  true,
                Auth:          auth,
                Progress:      nil,
        })
        if err != nil {
                w.gitFetchErrors.Inc()
                return nil, "", fmt.Errorf("cloning %s: %w", w.cfg.RepoURL, err)
        }

        commit, err := headCommit(repo)
        if err != nil {
                w.gitFetchErrors.Inc()
                return nil, "", err
        }
        return repo, commit, nil
}

// Clone performs the initial clone into memory. Must be called before Run.
func (w *Watcher) Clone(ctx context.Context) error {
        slog.Info("Cloning repository", "url", w.cfg.RepoURL, "branch", w.cfg.Branch)

        repo, commit, err := w.cloneInto(ctx)
        if err != nil {
                return err
        }

        now := time.Now()
        w.mu.Lock()
        w.repo = repo
        w.lastCommit = commit
        w.lastUpdate = now
        w.mu.Unlock()

        w.gitLastUpdate.Set(float64(now.Unix()))
        slog.Info("Repository cloned", "commit", commit)
        return nil
}

// Run polls for updates on the configured interval and also reacts to Trigger
// calls. It optionally re-clones on a slow cadence to reclaim memory. All git
// operations it drives (pull, reclone) run from this single goroutine, so they
// are serialised with each other and never overlap. Blocks until ctx is
// cancelled.
func (w *Watcher) Run(ctx context.Context) {
        ticker := time.NewTicker(w.cfg.PollInterval)
        defer ticker.Stop()

        // A nil channel blocks forever in select, disabling the reclone case when
        // the interval is 0.
        var recloneC <-chan time.Time
        if w.cfg.RecloneInterval > 0 {
                recloneTicker := time.NewTicker(w.cfg.RecloneInterval)
                defer recloneTicker.Stop()
                recloneC = recloneTicker.C
        }

        for {
                select {
                case <-ctx.Done():
                        return
                case <-ticker.C:
                        w.pull(ctx)
                case <-w.triggerCh:
                        slog.Info("Webhook trigger received, pulling")
                        w.pull(ctx)
                case <-recloneC:
                        w.reclone(ctx)
                }
        }
}

// reclone replaces the in-memory clone with a fresh one, discarding the git
// object store that grows as pulls accumulate history. The fetch runs before
// any lock is taken, so concurrent readers keep using the old repo; the swap is
// atomic under w.mu, and readers that captured the old *git.Repository continue
// against it until they finish (the old store is freed once unreferenced). A
// failed reclone leaves the existing clone untouched. Called only from Run, so
// it never overlaps a pull.
func (w *Watcher) reclone(ctx context.Context) {
        slog.Info("Re-cloning repository to reclaim memory", "url", w.cfg.RepoURL, "branch", w.cfg.Branch)

        repo, commit, err := w.cloneInto(ctx)
        if err != nil {
                slog.Warn("Reclone failed, keeping existing clone", "err", err)
                return
        }

        now := time.Now()
        w.mu.Lock()
        prev := w.lastCommit
        w.repo = repo
        w.lastCommit = commit
        w.lastUpdate = now
        w.mu.Unlock()

        w.gitLastUpdate.Set(float64(now.Unix()))

        if commit != prev {
                // A reclone normally lands on the same HEAD, but a commit may have
                // arrived since the last pull; treat it like any other new commit.
                slog.Info("New commit observed during reclone", "branch", w.cfg.Branch, "commit", commit, "prev", prev)
                if w.onChange != nil {
                        w.onChange(commit)
                }
        }
}

// Trigger schedules an immediate fetch, e.g. when a webhook fires.
// Non-blocking: if a trigger is already pending it is coalesced.
func (w *Watcher) Trigger() {
        select {
        case w.triggerCh <- struct{}{}:
        default:
        }
}

// TriggerStale schedules an immediate fetch because the repo has exceeded the
// configured maximum staleness. Increments the staleness counter and delegates
// to Trigger.
func (w *Watcher) TriggerStale() {
        w.staleRefreshes.Inc()
        w.Trigger()
}

// Ready reports whether the initial clone has completed successfully.
// Before Clone returns, Status and ReadHCLFiles return zero/nil values.
func (w *Watcher) Ready() bool {
        w.mu.RLock()
        defer w.mu.RUnlock()
        return w.repo != nil
}

// Status returns the last seen commit hash and the time it was seen.
func (w *Watcher) Status() (lastCommit string, lastUpdate time.Time) {
        w.mu.RLock()
        defer w.mu.RUnlock()
        return w.lastCommit, w.lastUpdate
}

// LastCommit returns just the last commit hash.
func (w *Watcher) LastCommit() string {
        w.mu.RLock()
        defer w.mu.RUnlock()
        return w.lastCommit
}

// ReadHCLFiles returns a map of repo-relative path → file content for every
// .hcl file under the configured HCLDir.
func (w *Watcher) ReadHCLFiles() (map[string]string, error) {
        w.mu.RLock()
        repo := w.repo
        w.mu.RUnlock()

        if repo == nil {
                return nil, fmt.Errorf("repository not cloned yet")
        }

        ref, err := repo.Head()
        if err != nil {
                return nil, fmt.Errorf("getting HEAD: %w", err)
        }

        commit, err := repo.CommitObject(ref.Hash())
        if err != nil {
                return nil, fmt.Errorf("getting commit object: %w", err)
        }

        tree, err := commit.Tree()
        if err != nil {
                return nil, fmt.Errorf("getting commit tree: %w", err)
        }

        // Build prefix filter for HCLDir.
        hclPrefix := normalizeHCLDir(w.cfg.HCLDir)

        result := make(map[string]string)
        err = tree.Files().ForEach(func(f *object.File) error {
                if !strings.HasSuffix(f.Name, ".hcl") {
                        return nil
                }
                if hclPrefix != "" && !strings.HasPrefix(f.Name, hclPrefix) {
                        return nil
                }
                content, err := f.Contents()
                if err != nil {
                        slog.Warn("Could not read HCL file from git tree", "file", f.Name, "err", err)
                        return nil // skip bad files, don't abort the walk
                }
                result[f.Name] = content
                return nil
        })
        if err != nil {
                return nil, fmt.Errorf("walking commit tree: %w", err)
        }

        slog.Debug("Read HCL files from repo", "count", len(result), "hcl_dir", w.cfg.HCLDir)
        return result, nil
}

// FileAtParentOf returns the content of path as it was at the first parent of
// the named commit. ok is false when the repo is not cloned, the commit is
// unknown or has no parent (the root commit), or the file did not exist at the
// parent. Keying off an explicit commit — rather than the repo's current HEAD —
// keeps the answer consistent with the HCL snapshot being evaluated even if a
// concurrent pull has advanced HEAD. It is used to decide whether a change at
// that commit (such as adding the managed meta tag) is new relative to the
// previous commit.
func (w *Watcher) FileAtParentOf(commit, path string) (content string, ok bool) {
        w.mu.RLock()
        repo := w.repo
        w.mu.RUnlock()
        if repo == nil || commit == "" {
                return "", false
        }

        c, err := repo.CommitObject(plumbing.NewHash(commit))
        if err != nil || c.NumParents() == 0 {
                return "", false
        }
        parent, err := c.Parent(0)
        if err != nil {
                return "", false
        }
        f, err := parent.File(path)
        if err != nil {
                // Includes object.ErrFileNotFound: the file did not exist at the parent.
                return "", false
        }
        content, err = f.Contents()
        if err != nil {
                return "", false
        }
        return content, true
}

// pull fetches the latest changes and calls onChange if the HEAD moved.
func (w *Watcher) pull(ctx context.Context) {
        auth, err := w.buildAuth()
        if err != nil {
                slog.Error("Building git auth for pull", "err", err)
                return
        }

        w.mu.RLock()
        repo := w.repo
        w.mu.RUnlock()

        wt, err := repo.Worktree()
        if err != nil {
                slog.Error("Getting worktree", "err", err)
                return
        }

        w.gitFetches.Inc()
        err = wt.PullContext(ctx, &git.PullOptions{
                RemoteName:    "origin",
                ReferenceName: plumbing.NewBranchReferenceName(w.cfg.Branch),
                SingleBranch:  true,
                Force:         true,
                Auth:          auth,
        })
        if err != nil && err != git.NoErrAlreadyUpToDate {
                w.gitFetchErrors.Inc()
                slog.Warn("Pull failed, attempting re-clone", "err", err)
                if err2 := w.Clone(ctx); err2 != nil {
                        slog.Error("Re-clone failed", "err", err2)
                        return
                }
                // Clone already updated state and gauges; check if commit changed.
        }

        commit, err := headCommit(repo)
        if err != nil {
                slog.Error("Getting HEAD after pull", "err", err)
                return
        }

        now := time.Now()
        w.mu.Lock()
        prev := w.lastCommit
        w.lastCommit = commit
        w.lastUpdate = now
        w.mu.Unlock()

        w.gitLastUpdate.Set(float64(now.Unix()))

        if commit != prev {
                slog.Info("New commit on branch", "branch", w.cfg.Branch, "commit", commit, "prev", prev)
                if w.onChange != nil {
                        w.onChange(commit)
                }
        }
}

func (w *Watcher) buildAuth() (transport.AuthMethod, error) {
        if w.cfg.GitSSHKeyPath != "" {
                auth, err := gitssh.NewPublicKeysFromFile("git", w.cfg.GitSSHKeyPath, w.cfg.GitSSHKeyPass)
                if err != nil {
                        return nil, fmt.Errorf("loading SSH key from %s: %w", w.cfg.GitSSHKeyPath, err)
                }
                if err := w.setSSHHostKeyCallback(auth); err != nil {
                        return nil, err
                }
                return auth, nil
        }
        if w.cfg.GitToken != "" {
                return &githttp.BasicAuth{
                        Username: "x-token", // username is ignored by GitHub for token auth
                        Password: w.cfg.GitToken,
                }, nil
        }
        return nil, nil // anonymous / SSH agent
}

// setSSHHostKeyCallback configures host key verification on auth.
// When --git-ssh-known-hosts is set the named file is required; if it cannot
// be opened, an error is returned. Without the flag go-git's default known_hosts
// locations (~/..ssh/known_hosts, /etc/ssh/ssh_known_hosts) are tried. If none
// are found, verification is skipped and a warning is logged.
func (w *Watcher) setSSHHostKeyCallback(auth *gitssh.PublicKeys) error {
        if w.cfg.GitSSHKnownHostsFile != "" {
                cb, err := gitssh.NewKnownHostsCallback(w.cfg.GitSSHKnownHostsFile)
                if err != nil {
                        return fmt.Errorf("loading known_hosts from %s: %w", w.cfg.GitSSHKnownHostsFile, err)
                }
                auth.HostKeyCallback = cb
                return nil
        }
        cb, err := gitssh.NewKnownHostsCallback()
        if err != nil {
                slog.Warn("SSH host key verification disabled: no known_hosts file found; "+
                        "set --git-ssh-known-hosts / GIT_SSH_KNOWN_HOSTS to enable verification", "err", err)
                return nil
        }
        auth.HostKeyCallback = cb
        return nil
}

func headCommit(repo *git.Repository) (string, error) {
        ref, err := repo.Head()
        if err != nil {
                return "", fmt.Errorf("getting HEAD: %w", err)
        }
        return ref.Hash().String(), nil
}

// normalizeHCLDir converts a user-supplied HCLDir value to the prefix used
// when filtering tree file paths (e.g. "jobs" → "jobs/", "" → "").
func normalizeHCLDir(dir string) string {
        dir = strings.Trim(dir, "/")
        if dir == "" || dir == "." {
                return ""
        }
        return dir + "/"
}

package nomad

import (
        "context"
        "fmt"
        "log/slog"
        "time"

        nomadapi "github.com/hashicorp/nomad/api"
)

// Updates returns a snapshot of the update queue for the JSON API.
func (d *Differ) Updates() []JobUpdate {
        return d.updateQueue.Snapshot()
}

// notifyApplier wakes the applier loop. Non-blocking; multiple rapid
// notifications coalesce, mirroring the git watcher's trigger channel.
func (d *Differ) notifyApplier() {
        select {
        case d.applyCh <- struct{}{}:
        default:
        }
}

// invalidateSkip clears the cached Raft index so the next Check cannot take
// the skip-optimisation shortcut. Called after a failed apply: the failure is
// retried by letting the next full diff cycle re-detect the drift and
// re-enqueue the same UpdateID, rather than by a bespoke retry loop.
func (d *Differ) invalidateSkip() {
        d.mu.Lock()
        d.lastNomadIndex = 0
        d.mu.Unlock()
}

// RunApplier drains the update queue until ctx is cancelled. It wakes on
// every enqueue and on a fallback ticker (--apply-interval). Detection and
// application are deliberately decoupled: a slow or failing apply never
// delays the next diff check.
func (d *Differ) RunApplier(ctx context.Context) {
        ticker := time.NewTicker(d.applyInterval)
        defer ticker.Stop()

        for {
                select {
                case <-ctx.Done():
                        return
                case <-ticker.C:
                case <-d.applyCh:
                }
                d.drainUpdates()
        }
}

// drainUpdates applies every PENDING update in queue order.
func (d *Differ) drainUpdates() {
        for {
                u := d.updateQueue.NextPending()
                if u == nil {
                        d.pendingUpdates.Set(0)
                        return
                }
                d.applyUpdate(u)
                d.pendingUpdates.Set(float64(d.updateQueue.PendingCount()))
        }
}

// completeUpdate records an apply outcome in the queue and metrics.
func (d *Differ) completeUpdate(u *JobUpdate, status JobUpdateStatus, appliedIndex uint64, errMsg string) {
        d.updateQueue.Complete(u.UpdateID, status, appliedIndex, errMsg)
        d.jobUpdatesTotal.WithLabelValues(string(u.Operation), string(status)).Inc()
}

// applyUpdate executes one update against Nomad.
func (d *Differ) applyUpdate(u *JobUpdate) {
        if u.Operation == JobUpdateOperationDeregister {
                d.applyDeregister(u)
                return
        }
        if u.Operation == JobUpdateOperationRevert {
                d.applyRevert(u)
                return
        }
        if u.Operation != JobUpdateOperationRegister {
                d.completeUpdate(u, JobUpdateStatusFailed, 0, fmt.Sprintf("unsupported operation %q", u.Operation))
                return
        }
        if u.job == nil {
                d.completeUpdate(u, JobUpdateStatusFailed, 0, "internal error: update has no parsed job")
                return
        }

        wq := &nomadapi.WriteOptions{Namespace: d.namespace}

        // Plan first — never register without a plan. If the plan no longer
        // shows any Git-owned change, the drift resolved between detection and
        // apply (or was only autoscaler churn); the update completes as a no-op.
        plan, _, err := d.jobs.Plan(u.job, true, wq)
        if err != nil {
                d.nomadAPIErrors.WithLabelValues("plan").Inc()
                slog.Warn("Apply: plan failed", "job", u.JobID, "update_id", u.UpdateID, "err", err)
                d.completeUpdate(u, JobUpdateStatusFailed, 0, fmt.Sprintf("plan: %v", err))
                d.invalidateSkip()
                return
        }
        if classifyDiff(plan.Diff, autoscaledGroups(u.job), d.managedMetaPrefix) == DiffClassNone {
                slog.Info("Apply: plan shows no change, nothing to do", "job", u.JobID, "update_id", u.UpdateID)
                d.completeUpdate(u, JobUpdateStatusSucceeded, 0, "")
                return
        }

        // CAS register: EnforceIndex with the ModifyIndex captured at detection
        // time. Nomad rejects the write if the job changed in between; for new
        // jobs the index is 0, which Nomad reads as "must not already exist".
        // PreserveCounts keeps autoscaler-owned group counts out of the write.
        resp, _, err := d.jobs.RegisterOpts(u.job, &nomadapi.RegisterOptions{
                EnforceIndex:   true,
                ModifyIndex:    u.NomadJobModifyIndex,
                PreserveCounts: u.preserveCounts,
        }, wq)
        if err != nil {
                d.nomadAPIErrors.WithLabelValues("register").Inc()
                slog.Warn("Apply: register failed", "job", u.JobID, "update_id", u.UpdateID,
                        "enforce_index", u.NomadJobModifyIndex, "err", err)
                d.completeUpdate(u, JobUpdateStatusFailed, 0, fmt.Sprintf("register: %v", err))
                // Whether a CAS conflict or a transient error, the recovery is the
                // same: force the next diff cycle to run in full so it re-detects
                // current state and enqueues a fresh update with a current token.
                d.invalidateSkip()
                return
        }

        slog.Info("Apply: job registered", "job", u.JobID, "update_id", u.UpdateID,
                "eval_id", resp.EvalID, "new_modify_index", resp.JobModifyIndex)
        d.completeUpdate(u, JobUpdateStatusSucceeded, resp.JobModifyIndex, "")
}

// applyDeregister removes a job that was deleted from the repo. It rechecks
// live state immediately before the call rather than trusting the stored
// intent ("recheck, don't remember"): a deregister only proceeds if the job
// still exists and still carries the managed tag. If it is already gone the
// update succeeds as a no-op; if it exists but is no longer tagged (someone
// re-registered it, or took it out of management) the deregister is abandoned.
func (d *Differ) applyDeregister(u *JobUpdate) {
        q := &nomadapi.QueryOptions{Namespace: d.namespace}
        wq := &nomadapi.WriteOptions{Namespace: d.namespace}

        live, _, err := d.jobs.Info(u.JobID, q)
        if err != nil {
                if isNotFound(err) {
                        slog.Info("Deregister: job already gone, nothing to do", "job", u.JobID, "update_id", u.UpdateID)
                        d.completeUpdate(u, JobUpdateStatusSucceeded, 0, "")
                        return
                }
                d.nomadAPIErrors.WithLabelValues("info").Inc()
                slog.Warn("Deregister: recheck failed", "job", u.JobID, "update_id", u.UpdateID, "err", err)
                d.completeUpdate(u, JobUpdateStatusFailed, 0, fmt.Sprintf("recheck: %v", err))
                d.invalidateSkip()
                return
        }
        if live == nil || !d.metaKeyPresent(live.Meta) {
                // The job no longer carries the managed tag: precondition gone, do not
                // touch it. The next cycle re-evaluates against current state.
                slog.Info("Deregister: live job no longer carries the managed tag; not deregistering",
                        "job", u.JobID, "update_id", u.UpdateID)
                d.completeUpdate(u, JobUpdateStatusFailed, 0, "live job no longer carries the managed tag")
                d.invalidateSkip()
                return
        }

        slog.Info("Deregister: removing job that was deleted from the repo",
                "job", u.JobID, "update_id", u.UpdateID, "purge", d.deregisterPurge)
        evalID, _, err := d.jobs.Deregister(u.JobID, d.deregisterPurge, wq)
        if err != nil {
                d.nomadAPIErrors.WithLabelValues("deregister").Inc()
                slog.Warn("Deregister failed", "job", u.JobID, "update_id", u.UpdateID, "err", err)
                d.completeUpdate(u, JobUpdateStatusFailed, 0, fmt.Sprintf("deregister: %v", err))
                d.invalidateSkip()
                return
        }
        slog.Info("Deregister: job deregistered", "job", u.JobID, "update_id", u.UpdateID, "eval_id", evalID, "purge", d.deregisterPurge)
        d.completeUpdate(u, JobUpdateStatusSucceeded, 0, "")
}

// applyRevert rolls a job back to its last stable version after a failed
// deployment. The revert is CAS-guarded by enforcePriorVersion (the failed
// version): if the job has moved on since detection — a human change, or
// Nomad's own auto_revert beating us to it — Nomad rejects the revert and the
// update fails, leaving the next cycle to recompute against current state. This
// is why a double-revert with auto_revert cannot happen: even if the
// stand-down check were bypassed, the CAS guard would reject the redundant
// write.
func (d *Differ) applyRevert(u *JobUpdate) {
        wq := &nomadapi.WriteOptions{Namespace: d.namespace}
        enforce := u.RevertFromVersion

        slog.Info("Revert: rolling failed deployment back to last stable version",
                "job", u.JobID, "update_id", u.UpdateID,
                "from_version", u.RevertFromVersion, "to_version", u.RevertToVersion)
        resp, _, err := d.jobs.Revert(u.JobID, u.RevertToVersion, &enforce, wq, "", "")
        if err != nil {
                d.nomadAPIErrors.WithLabelValues("revert").Inc()
                slog.Warn("Revert failed", "job", u.JobID, "update_id", u.UpdateID,
                        "from_version", u.RevertFromVersion, "to_version", u.RevertToVersion, "err", err)
                d.completeUpdate(u, JobUpdateStatusFailed, 0, fmt.Sprintf("revert: %v", err))
                d.invalidateSkip()
                return
        }
        slog.Info("Revert: job reverted", "job", u.JobID, "update_id", u.UpdateID,
                "eval_id", resp.EvalID, "new_modify_index", resp.JobModifyIndex)
        d.completeUpdate(u, JobUpdateStatusSucceeded, resp.JobModifyIndex, "")
}

package nomad

import (
        "log/slog"
        "strings"

        nomadapi "github.com/hashicorp/nomad/api"
)

// DiffClass categorises a plan diff for update-policy decisions.
type DiffClass int

const (
        // DiffClassNone means the diff contains no changes that Git owns —
        // either it is empty, or everything in it is autoscaler-owned
        // Count/Scaling churn. Nothing to apply.
        DiffClassNone DiffClass = iota
        // DiffClassImageOnly means every Git-owned change is a Docker image
        // reference (the "image" field inside a task's Config object), possibly
        // alongside nomad-gitops's own managed-meta keys.
        DiffClassImageOnly
        // DiffClassManagedMetaOnly means every Git-owned change is to one of
        // nomad-gitops's own managed-prefix meta keys (e.g. gitops_managed,
        // gitops_update_policy). These are not applied on their own by default:
        // re-registering a running job purely to push our keys onto it is
        // disruptive and unnecessary, since the HCL is already the source of
        // truth for them. They ride along the next real update.
        DiffClassManagedMetaOnly
        // DiffClassOther means the diff contains at least one Git-owned change
        // that is not an image reference or a managed-meta key.
        DiffClassOther
)

func (c DiffClass) String() string {
        switch c {
        case DiffClassNone:
                return "none"
        case DiffClassImageOnly:
                return "image-only"
        case DiffClassManagedMetaOnly:
                return "managed-meta-only"
        default:
                return "other"
        }
}

// changeCounts tallies the changed leaves in a plan diff by kind.
type changeCounts struct {
        image       int
        managedMeta int
        other       int
}

// classifyDiff walks a plan diff and classifies it. autoscaled names the task
// groups that carry a scaling policy: changes to their Count field and
// Scaling object are owned by the autoscaler, not by Git, and are ignored
// (per the "do not fight the autoscaler" design rule). metaPrefix is the
// managed meta prefix (e.g. "gitops"); changes to keys under it are bucketed
// separately. Container nodes (job/group/task marked Edited because a child
// changed) are not leaves and do not affect the result.
func classifyDiff(d *nomadapi.JobDiff, autoscaled map[string]bool, metaPrefix string) DiffClass {
        if d == nil || d.Type == "" || d.Type == "None" {
                return DiffClassNone
        }

        var c changeCounts
        c.addFields(d.Fields, nil, false, false, metaPrefix)
        c.addObjects(d.Objects, metaPrefix, 1)

        for _, tg := range d.TaskGroups {
                if tg == nil {
                        continue
                }
                isAuto := autoscaled[tg.Name]

                // Whole-group additions or removals are structural changes.
                if tg.Type == "Added" || tg.Type == "Deleted" {
                        c.other++
                }

                var skip map[string]bool
                if isAuto {
                        skip = map[string]bool{"Count": true}
                }
                c.addFields(tg.Fields, skip, false, false, metaPrefix)
                for _, o := range tg.Objects {
                        if o == nil {
                                continue
                        }
                        if isAuto && o.Name == "Scaling" {
                                continue
                        }
                        c.addObject(o, metaPrefix, 1)
                }

                for _, task := range tg.Tasks {
                        if task == nil {
                                continue
                        }
                        if task.Type == "Added" || task.Type == "Deleted" {
                                c.other++
                        }
                        c.addFields(task.Fields, nil, false, false, metaPrefix)
                        for _, o := range task.Objects {
                                if o == nil {
                                        continue
                                }
                                c.addObject(o, metaPrefix, 1)
                        }
                }
        }

        switch {
        case c.other > 0:
                return DiffClassOther
        case c.image > 0:
                return DiffClassImageOnly
        case c.managedMeta > 0:
                return DiffClassManagedMetaOnly
        default:
                return DiffClassNone
        }
}

// addObject counts the changed leaves under one object, aware of two special
// objects: "Config" (whose "image" field is a Docker image reference) and
// "Meta" (whose fields are meta keys, so managed-prefix ones are tracked
// separately). Image/meta semantics do not propagate into nested sub-objects.
// depth is the object's nesting level (1 at the top); beyond
// MaxPlanDiffObjectDepth, recursion stops and the unexamined subtree counts as
// a non-image, non-meta change — the conservative reading, since an
// update-policy of image-only or none must not wave through a change nobody
// actually looked at.
func (c *changeCounts) addObject(o *nomadapi.ObjectDiff, metaPrefix string, depth int) {
        if depth > MaxPlanDiffObjectDepth {
                slog.Warn("Plan diff exceeds maximum nesting depth; treating unexamined nesting as a non-trivial change",
                        "depth", depth, "max_depth", MaxPlanDiffObjectDepth)
                c.other++
                return
        }
        c.addFields(o.Fields, nil, o.Name == "Meta", o.Name == "Config", metaPrefix)
        c.addObjects(o.Objects, metaPrefix, depth+1)
}

func (c *changeCounts) addObjects(objs []*nomadapi.ObjectDiff, metaPrefix string, depth int) {
        for _, o := range objs {
                if o == nil {
                        continue
                }
                c.addObject(o, metaPrefix, depth)
        }
}

// addFields counts changed fields into the right bucket. inMeta marks fields
// inside a Meta object (their names are bare meta keys); inConfig marks
// fields inside a Config object (where "image" is the Docker image). skip
// names fields to ignore entirely (autoscaler-owned Count).
func (c *changeCounts) addFields(fields []*nomadapi.FieldDiff, skip map[string]bool, inMeta, inConfig bool, metaPrefix string) {
        for _, f := range fields {
                if f == nil || !fieldChanged(f.Type) {
                        continue
                }
                if skip != nil && skip[f.Name] {
                        continue
                }
                switch {
                case inConfig && strings.EqualFold(f.Name, "image"):
                        c.image++
                case isManagedMetaName(f.Name, inMeta, metaPrefix):
                        c.managedMeta++
                default:
                        c.other++
                }
        }
}

// isManagedMetaName reports whether a changed field refers to one of
// nomad-gitops's own meta keys. It accepts both a bare key name when the
// field is inside a Meta object (inMeta) and the wrapped "Meta[key]" form
// anywhere, so it is robust to how different Nomad versions render meta diffs.
func isManagedMetaName(name string, inMeta bool, metaPrefix string) bool {
        if metaPrefix == "" {
                return false
        }
        key := name
        if strings.HasPrefix(name, "Meta[") && strings.HasSuffix(name, "]") {
                key = name[len("Meta[") : len(name)-1]
        } else if !inMeta {
                return false
        }
        return strings.HasPrefix(key, metaPrefix+"_") || strings.HasPrefix(key, metaPrefix+".")
}

func fieldChanged(t string) bool {
        return t == "Added" || t == "Deleted" || t == "Edited"
}

// autoscaledGroups returns the names of task groups in the parsed job that
// carry an enabled scaling policy.
func autoscaledGroups(job *nomadapi.Job) map[string]bool {
        if job == nil {
                return nil
        }
        var out map[string]bool
        for _, tg := range job.TaskGroups {
                if tg == nil || tg.Name == nil || tg.Scaling == nil {
                        continue
                }
                if tg.Scaling.Enabled != nil && !*tg.Scaling.Enabled {
                        continue
                }
                if out == nil {
                        out = make(map[string]bool)
                }
                out[*tg.Name] = true
        }
        return out
}

// Package nomad compares HCL job definitions against a live Nomad cluster and
// reports any diffs it finds.
package nomad

import (
        "context"
        "fmt"
        "log/slog"
        "path"
        "regexp"
        "sort"
        "strings"
        "sync"
        "time"

        nomadapi "github.com/hashicorp/nomad/api"
        "github.com/prometheus/client_golang/prometheus"
        "github.com/prometheus/client_golang/prometheus/promauto"

        "github.com/gerrowadat/nomad-gitops/internal/config"
)

// jobBlockRe matches a top-level Nomad job stanza in HCL.
// Files without this pattern are silently skipped (e.g. ACL policies, volumes, namespaces).
var jobBlockRe = regexp.MustCompile(`(?m)^\s*job\s+"`)

// DiffType describes the relationship between a job in HCL and in Nomad.
type DiffType string

const (
        // DiffTypeModified means the job exists in both HCL and Nomad but the
        // definitions differ (Nomad plan shows changes).
        DiffTypeModified DiffType = "modified"

        // DiffTypeMissingFromNomad means the job is defined in HCL but not
        // currently registered in Nomad.
        DiffTypeMissingFromNomad DiffType = "missing_from_nomad"

        // DiffTypeMissingFromHCL means the job is running in Nomad but there is
        // no corresponding HCL file in the repo.
        DiffTypeMissingFromHCL DiffType = "missing_from_hcl"
)

// SelectionReason describes why a job was included in the watched set.
type SelectionReason string

const (
        // SelectionReasonGlob means the job was selected by the job-selector-glob pattern.
        SelectionReasonGlob SelectionReason = "glob"
        // SelectionReasonMeta means the job was selected by the managed-meta-prefix key.
        SelectionReasonMeta SelectionReason = "meta"
        // SelectionReasonBoth means the job matched both the glob and the meta key.
        SelectionReasonBoth SelectionReason = "both"
)

// SelectedJob records a job that matched the configured selection criteria
// and the reason it was included.
type SelectedJob struct {
        JobID  string          `json:"job_id"`
        Reason SelectionReason `json:"selection_reason"`
}

// JobDiff describes a single divergence between the git repo and Nomad.
type JobDiff struct {
        JobID    string   `json:"job_id"`
        HCLFile  string   `json:"hcl_file,omitempty"` // empty for MissingFromHCL
        DiffType DiffType `json:"diff_type"`
        Detail   string   `json:"detail"`

        // ApplyAction records what nomad-gitops will do about this diff and,
        // when it will not apply it, why. Lets the API and web console explain
        // non-application without log scraping.
        ApplyAction ApplyAction `json:"apply_action,omitempty"`

        // ApplyDetail is an optional, more specific human-readable explanation
        // that refines ApplyAction with the actual values involved — for example,
        // which update policy blocked the change, where that policy came from, and
        // what would need to change. Empty when ApplyAction.Describe() already says
        // everything there is to say.
        ApplyDetail string `json:"apply_detail,omitempty"`

        // PlanDiff holds the structured diff from the Nomad plan API.
        // Only populated for DiffTypeModified entries.
        PlanDiff *nomadapi.JobDiff `json:"-"`
}

// ApplyAction is the disposition of a detected diff: whether it will be
// applied and, if not, the reason.
type ApplyAction string

const (
        // ApplyActionQueued means an update was enqueued and will be applied.
        ApplyActionQueued ApplyAction = "queued"
        // ApplyActionPolicyBlocked means the effective update policy disallows it.
        ApplyActionPolicyBlocked ApplyAction = "blocked_by_policy"
        // ApplyActionPreExisting means the drift pre-dated the scope change that
        // brought it into scope — the job's opt-in, or a policy widening.
        ApplyActionPreExisting ApplyAction = "blocked_preexisting_drift"
        // ApplyActionCreationBlocked means first-time registration is disabled.
        ApplyActionCreationBlocked ApplyAction = "blocked_creation_disabled"
        // ApplyActionMetaOnly means the diff is confined to our own meta keys.
        ApplyActionMetaOnly ApplyAction = "skipped_meta_only"
        // ApplyActionObservationOnly means a job is running in Nomad with no HCL,
        // and deregistration is disabled (or the job is not deregister-eligible):
        // it is left running, observation-only.
        ApplyActionObservationOnly ApplyAction = "observation_only"
        // ApplyActionDeregisterQueued means an orphaned job will be deregistered.
        ApplyActionDeregisterQueued ApplyAction = "queued_deregister"
        // ApplyActionDeregisterGrace means an orphaned job is deregister-eligible
        // but its grace period has not yet elapsed.
        ApplyActionDeregisterGrace ApplyAction = "deregister_pending_grace"
        // ApplyActionNoChange means the only diff is autoscaler-owned churn.
        ApplyActionNoChange ApplyAction = "no_actionable_change"
        // ApplyActionKnownFailed means the flap-loop guard is holding the apply:
        // the HCL spec matches a recent Nomad job version whose deployment failed,
        // so re-applying it would re-enter a known failure. Released when Git moves
        // to a spec that has not failed.
        ApplyActionKnownFailed ApplyAction = "blocked_known_failed"
)

// Describe returns a human-readable explanation for display.
func (a ApplyAction) Describe() string {
        switch a {
        case ApplyActionQueued:
                return "queued for apply"
        case ApplyActionPolicyBlocked:
                return "not applied: blocked by update policy"
        case ApplyActionPreExisting:
                return "not applied: drift pre-dates the scope change that brought it in — opt-in or policy widening (set --apply-existing-drift)"
        case ApplyActionCreationBlocked:
                return "not applied: job creation disabled (set --enable-job-creation)"
        case ApplyActionMetaOnly:
                return "not applied: change is confined to managed meta keys"
        case ApplyActionObservationOnly:
                return "not applied: running but absent from the repo; left untouched (set --enable-deregister to remove)"
        case ApplyActionDeregisterQueued:
                return "queued for deregistration (removed from the repo)"
        case ApplyActionDeregisterGrace:
                return "will deregister after the grace period (removed from the repo)"
        case ApplyActionNoChange:
                return "no actionable change (autoscaler-owned)"
        case ApplyActionKnownFailed:
                return "not applied: this spec matches a recent failed deployment (flap-loop guard); waiting for a fix in Git"
        default:
                return string(a)
        }
}

// hclEntry is a parsed HCL job that has passed the preliminary selection filter.
// Final selection (using live Nomad meta) is applied in checkHCLCandidate.
type hclEntry struct {
        job     *nomadapi.Job
        file    string
        globSel bool // matched by job-selector-glob
        metaHCL bool // managed meta key present in parsed HCL
}

// HistorySource provides read-only access to prior git state, used to decide
// whether drift pre-dates a job entering management scope. *gitwatch.Watcher
// satisfies it. When nil, pre-existing-drift detection is disabled and drift
// reconciles normally.
type HistorySource interface {
        // FileAtParentOf returns the content of path at the first parent of the
        // named commit. ok is false when the commit is unknown, has no parent
        // (root commit), or the file is absent there. The lookup is keyed off the
        // commit being evaluated — not the repo's current HEAD — so the decision
        // stays consistent with the HCL snapshot passed to Check even if the
        // watcher pulls a newer commit concurrently.
        FileAtParentOf(commit, path string) (content string, ok bool)
}

// NomadJobsClient is the subset of the Nomad API jobs client we use.
// The concrete *nomadapi.Jobs satisfies this interface; tests inject a mock.
type NomadJobsClient interface {
        ParseHCL(jobHCL string, canonicalize bool) (*nomadapi.Job, error)
        Plan(job *nomadapi.Job, diff bool, q *nomadapi.WriteOptions) (*nomadapi.JobPlanResponse, *nomadapi.WriteMeta, error)
        Info(jobID string, q *nomadapi.QueryOptions) (*nomadapi.Job, *nomadapi.QueryMeta, error)
        List(q *nomadapi.QueryOptions) ([]*nomadapi.JobListStub, *nomadapi.QueryMeta, error)
        RegisterOpts(job *nomadapi.Job, opts *nomadapi.RegisterOptions, q *nomadapi.WriteOptions) (*nomadapi.JobRegisterResponse, *nomadapi.WriteMeta, error)
        Deregister(jobID string, purge bool, q *nomadapi.WriteOptions) (string, *nomadapi.WriteMeta, error)
        // Versions returns a job's retained version history (most recent first).
        Versions(jobID string, diffs bool, q *nomadapi.QueryOptions) ([]*nomadapi.Job, []*nomadapi.JobDiff, *nomadapi.QueryMeta, error)
        // Deployments returns a job's deployments (most recent first).
        Deployments(jobID string, all bool, q *nomadapi.QueryOptions) ([]*nomadapi.Deployment, *nomadapi.QueryMeta, error)
        // LatestDeployment returns the job's most recent deployment, or nil.
        LatestDeployment(jobID string, q *nomadapi.QueryOptions) (*nomadapi.Deployment, *nomadapi.QueryMeta, error)
        // Revert rolls a job back to a prior version. enforcePriorVersion, when
        // non-nil, is a CAS guard: the revert only lands if the job is still at that
        // version.
        Revert(jobID string, version uint64, enforcePriorVersion *uint64, q *nomadapi.WriteOptions, consulToken, vaultToken string) (*nomadapi.JobRegisterResponse, *nomadapi.WriteMeta, error)
        // TagVersion attaches a durable name to a job version so it survives GC.
        TagVersion(jobID string, version uint64, name, description string, q *nomadapi.WriteOptions) (*nomadapi.WriteMeta, error)
}

// Differ runs periodic diff checks and stores the latest results.
type Differ struct {
        jobs NomadJobsClient
        // nomadClient is the concrete client behind jobs, retained so the token can
        // be rotated via SetSecretID. nil in tests (which inject a mock jobs client).
        nomadClient *nomadapi.Client
        // tokenFilePath, when non-empty, is the Nomad token file the refresher
        // re-reads; tokenPollInterval is how often; initialToken is the value read
        // at startup, used as the refresher's baseline.
        tokenFilePath     string
        tokenPollInterval time.Duration
        initialToken      string
        // Workload-identity login mode: loginAuthMethod (non-empty enables it) is the
        // JWT auth method; loginJWTFile is the identity JWT to exchange. loginExpiry
        // is the startup token's expiry (nil if login not used or startup login
        // failed), loginFailed records a failed startup login so the refresher
        // retries promptly.
        loginAuthMethod   string
        loginJWTFile      string
        loginExpiry       *time.Time
        loginFailed       bool
        loginNoTTLWarned  sync.Once
        namespace         string
        includeDeadJobs   bool
        jobSelectorGlob   string
        managedMetaPrefix string
        // redactSecrets replaces potentially sensitive plan-diff values (env vars,
        // templates, secret-like keys) with RedactedValue before the diff is
        // stored, so no downstream consumer can expose them.
        redactSecrets bool

        // defaultPolicy is the update policy applied to managed jobs whose HCL
        // meta carries no <prefix>_update_policy key. Defaults to "none": detect
        // and surface drift, never apply it.
        defaultPolicy UpdatePolicy
        // enableJobCreation gates first-time registration of jobs that exist in
        // Git but not in Nomad. Off by default.
        enableJobCreation bool
        // applyMetaOnlyChanges allows a managed-meta-only diff to trigger an
        // update on its own. countMetaOnlyChanges allows it to count as drift.
        // Both off by default.
        applyMetaOnlyChanges bool
        countMetaOnlyChanges bool
        // applyExistingDrift allows drift that pre-existed a job's scope entry
        // (the managed tag added in the HEAD commit) to be applied. Off by default.
        applyExistingDrift bool
        // Deregistration of jobs removed from the repo. enableDeregister gates it;
        // deregisterPurge selects purge vs graceful stop; deregisterGrace is how
        // long a job must stay orphaned first. All off/conservative by default.
        enableDeregister bool
        deregisterPurge  bool
        deregisterGrace  time.Duration
        // flapGuard is the default flap-loop guard mode (history, tag, or off),
        // from --flap-guard. Per-job overridable via the <prefix>_flap_guard meta
        // key. allowRollback is the default for active rollback, from
        // --allow-rollback, overridable via <prefix>_rollback. Both only apply to
        // deployment-producing jobs.
        flapGuard     string
        allowRollback bool
        // history answers whether the managed tag was present before HEAD, used to
        // detect pre-existing drift. nil disables the check.
        history HistorySource
        // managedKeyRe matches the opt-in key set to "true" in raw HCL text, for
        // the cheap parent-content check. policyKeyRe captures the value of the
        // <prefix>_update_policy key in raw HCL text, used to read a job's policy at
        // the parent commit without re-parsing.
        managedKeyRe *regexp.Regexp
        policyKeyRe  *regexp.Regexp
        // applyInterval is the fallback cadence of the applier loop; enqueues
        // also wake it immediately via applyCh.
        applyInterval time.Duration
        updateQueue   *UpdateQueue
        applyCh       chan struct{}

        // metaIssuesLogged dedups meta-key issue log lines: each unique
        // (job, key, value, issue) is logged once per process. The counter
        // metric is not deduped.
        metaIssuesLogged sync.Map

        // rollbackLogged dedups the auto_revert-clash WARN: each job is logged at
        // most once per process when active rollback stands down in favour of
        // Nomad's own auto_revert. The metric is not deduped.
        rollbackLogged sync.Map

        // prevMeta holds the previous cycle's prefix-key snapshots per
        // (source, job), used to notice and log meta-key transitions. prevManaged
        // is the set of job IDs actively managed via HCL last cycle, used to log
        // a job leaving GitOps management exactly once. Both protected by metaMu.
        metaMu      sync.Mutex
        prevMeta    map[string]metaState
        prevManaged map[string]bool

        mu             sync.RWMutex
        diffs          []JobDiff
        selectedJobs   []SelectedJob
        lastCheckTime  time.Time
        lastCommit     string
        lastNomadIndex uint64               // Raft index from the last successful List(); protected by mu
        driftFirstSeen map[string]time.Time // key: driftKey(jobID, diffType); protected by mu

        hclParseErrors    prometheus.Counter
        hclFilesSkipped   prometheus.Counter
        diffChecks        prometheus.Counter
        diffChecksSkipped prometheus.Counter
        staleChecks       prometheus.Counter
        redactedFields    prometheus.Counter
        jobsSkippedBySel  *prometheus.CounterVec
        nomadAPIErrors    *prometheus.CounterVec
        lastCheck         prometheus.Gauge
        jobDiffs          *prometheus.GaugeVec
        driftedJobs       *prometheus.GaugeVec
        jobDriftSince     *prometheus.GaugeVec

        updatesBlockedByPolicy         *prometheus.CounterVec
        updatesBlockedCreationDisabled *prometheus.CounterVec
        jobUpdatesTotal                *prometheus.CounterVec
        pendingUpdates                 prometheus.Gauge
        metaKeyIssues                  *prometheus.CounterVec
        metaKeyChanges                 *prometheus.CounterVec
        metaOnlyDiffs                  *prometheus.CounterVec
        updatesBlockedExistingDrift    *prometheus.CounterVec
        jobsLeftManagement             *prometheus.CounterVec
        updatesBlockedKnownFailed      *prometheus.CounterVec
        rollbacks                      *prometheus.CounterVec
        failedVersionsTagged           *prometheus.CounterVec
        nomadTokenRefreshes            *prometheus.CounterVec
        nomadLogins                    *prometheus.CounterVec
}

// newDifferBase constructs a Differ from config with metrics registered into reg.
func newDifferBase(jobs NomadJobsClient, cfg *config.Config, reg prometheus.Registerer) *Differ {
        defaultPolicy := UpdatePolicy(cfg.DefaultUpdatePolicy)
        if !ValidUpdatePolicy(cfg.DefaultUpdatePolicy) {
                defaultPolicy = UpdatePolicyNone
        }
        applyInterval := cfg.ApplyInterval
        if applyInterval <= 0 {
                applyInterval = 10 * time.Second
        }
        flapGuard := cfg.FlapGuard
        if !validFlapGuardValue(flapGuard) {
                // Empty or invalid (e.g. a Config built directly in a test): fall back
                // to the documented default rather than silently disabling the guard.
                flapGuard = "history"
        }

        var managedKeyRe, policyKeyRe *regexp.Regexp
        if cfg.ManagedMetaPrefix != "" {
                // Matches <prefix>_managed = "true" in raw HCL, in both the block form
                // (bare key) and the object-expression form (quoted key).
                managedKeyRe = regexp.MustCompile(`(?m)"?` + regexp.QuoteMeta(cfg.ManagedMetaPrefix) + `_managed"?\s*=\s*"true"`)
                // Captures the value of <prefix>_update_policy in either HCL form.
                policyKeyRe = regexp.MustCompile(`(?m)"?` + regexp.QuoteMeta(cfg.ManagedMetaPrefix) + `_update_policy"?\s*=\s*"([^"]*)"`)
        }

        f := promauto.With(reg)
        d := &Differ{
                jobs:                 jobs,
                namespace:            cfg.NomadNamespace,
                includeDeadJobs:      cfg.IncludeDeadJobs,
                jobSelectorGlob:      cfg.JobSelectorGlob,
                managedMetaPrefix:    cfg.ManagedMetaPrefix,
                redactSecrets:        cfg.RedactSecrets,
                defaultPolicy:        defaultPolicy,
                enableJobCreation:    cfg.EnableJobCreation,
                applyMetaOnlyChanges: cfg.ApplyMetaOnlyChanges,
                countMetaOnlyChanges: cfg.CountMetaOnlyChanges,
                applyExistingDrift:   cfg.ApplyExistingDrift,
                enableDeregister:     cfg.EnableDeregister,
                deregisterPurge:      cfg.DeregisterPurge,
                deregisterGrace:      cfg.DeregisterGrace,
                managedKeyRe:         managedKeyRe,
                policyKeyRe:          policyKeyRe,
                flapGuard:            flapGuard,
                allowRollback:        cfg.AllowRollback,
                applyInterval:        applyInterval,
                updateQueue:          NewUpdateQueue(),
                applyCh:              make(chan struct{}, 1),
                driftFirstSeen:       make(map[string]time.Time),
                hclParseErrors: f.NewCounter(prometheus.CounterOpts{
                        Name: "nomad_gitops_hcl_parse_errors_total",
                        Help: "Total number of HCL files that failed to parse as Nomad job definitions.",
                }),
                hclFilesSkipped: f.NewCounter(prometheus.CounterOpts{
                        Name: "nomad_gitops_hcl_non_job_files_skipped_total",
                        Help: "Total number of HCL files skipped because they lack a top-level job stanza (e.g. ACL policies, volumes).",
                }),
                diffChecks: f.NewCounter(prometheus.CounterOpts{
                        Name: "nomad_gitops_diff_checks_total",
                        Help: "Total number of diff checks run against the Nomad cluster.",
                }),
                diffChecksSkipped: f.NewCounter(prometheus.CounterOpts{
                        Name: "nomad_gitops_diff_checks_skipped_total",
                        Help: "Total number of diff checks skipped because neither the Nomad index nor the git commit changed.",
                }),
                staleChecks: f.NewCounter(prometheus.CounterOpts{
                        Name: "nomad_gitops_staleness_checks_total",
                        Help: "Total number of Nomad diff checks triggered by the staleness check.",
                }),
                redactedFields: f.NewCounter(prometheus.CounterOpts{
                        Name: "nomad_gitops_diff_fields_redacted_total",
                        Help: "Total number of potentially sensitive plan-diff field values redacted before storage.",
                }),
                jobsSkippedBySel: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_jobs_skipped_by_selector_total",
                        Help: "Total number of jobs skipped because they did not match the configured selection criteria, by source (hcl or nomad).",
                }, []string{"source"}),
                nomadAPIErrors: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_api_errors_total",
                        Help: "Total number of Nomad API errors by operation.",
                }, []string{"op"}),
                lastCheck: f.NewGauge(prometheus.GaugeOpts{
                        Name: "nomad_gitops_last_check_timestamp_seconds",
                        Help: "Unix timestamp of the most recent diff check.",
                }),
                jobDiffs: f.NewGaugeVec(prometheus.GaugeOpts{
                        Name: "nomad_gitops_job_diffs",
                        Help: "1 for each job/diff-type combination currently detected.",
                }, []string{"nomad_job", "diff_type"}),
                driftedJobs: f.NewGaugeVec(prometheus.GaugeOpts{
                        Name: "nomad_gitops_drifted_jobs",
                        Help: "Number of jobs currently in each drift state.",
                }, []string{"diff_type"}),
                jobDriftSince: f.NewGaugeVec(prometheus.GaugeOpts{
                        Name: "nomad_gitops_job_drift_first_seen_timestamp_seconds",
                        Help: "Unix timestamp when drift was first detected for each job. Cleared when drift resolves. Use time()-metric to get seconds in drift state.",
                }, []string{"nomad_job", "diff_type"}),
                updatesBlockedByPolicy: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_updates_blocked_by_policy_total",
                        Help: "Detected diffs that would have produced a JobUpdate but were filtered out by the effective update policy.",
                }, []string{"nomad_job", "policy"}),
                updatesBlockedCreationDisabled: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_updates_blocked_creation_disabled_total",
                        Help: "First-time registrations blocked because --enable-job-creation is off.",
                }, []string{"nomad_job"}),
                jobUpdatesTotal: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_job_updates_total",
                        Help: "JobUpdates reaching a terminal state, by operation and status.",
                }, []string{"operation", "status"}),
                pendingUpdates: f.NewGauge(prometheus.GaugeOpts{
                        Name: "nomad_gitops_job_updates_pending",
                        Help: "Number of JobUpdates currently waiting to be applied.",
                }),
                metaKeyIssues: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_meta_key_issues_total",
                        Help: "Job meta keys under the managed prefix that nomad-gitops cannot act on, by issue (unknown_key, invalid_value). Counted every check cycle the issue persists.",
                }, []string{"nomad_job", "issue"}),
                metaKeyChanges: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_meta_key_changes_total",
                        Help: "Transitions of managed-prefix meta keys (added, removed, changed) noticed between check cycles, by source (hcl or nomad).",
                }, []string{"nomad_job", "source"}),
                metaOnlyDiffs: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_meta_only_diffs_total",
                        Help: "Diffs confined to nomad-gitops's own meta keys, detected per check cycle. By default these are neither counted as drift nor applied (see --count-meta-only-changes, --apply-meta-only-changes); they converge on the next real update.",
                }, []string{"nomad_job"}),
                updatesBlockedExistingDrift: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_updates_blocked_preexisting_total",
                        Help: "Updates not enqueued because the drift pre-dated a scope change that brought it in: the job's opt-in (managed tag added) or a policy widening (e.g. image-only to full). Enable with --apply-existing-drift.",
                }, []string{"nomad_job"}),
                jobsLeftManagement: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_jobs_left_management_total",
                        Help: "Managed jobs that left GitOps management, by reason: tag_removed (gitops_managed dropped from HCL) or removed_from_repo (HCL file deleted or job renamed). Logged once per transition.",
                }, []string{"nomad_job", "reason"}),
                updatesBlockedKnownFailed: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_updates_blocked_known_failed_total",
                        Help: "Registrations withheld by the flap-loop guard because the HCL spec matches a recent Nomad job version whose deployment failed. The signal that a job is stuck on a known-bad commit awaiting a fix in Git.",
                }, []string{"nomad_job"}),
                rollbacks: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_rollbacks_total",
                        Help: "Active rollback outcomes for deployment-producing jobs without auto_revert, by result: queued (a revert was enqueued), deferred_auto_revert (stood down because the job's update stanza sets auto_revert), no_stable_version (no stable version to revert to).",
                }, []string{"nomad_job", "result"}),
                failedVersionsTagged: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_failed_versions_tagged_total",
                        Help: "Failed job versions tagged in Nomad by the flap-guard tag mode (--flap-guard=tag) so the block survives version GC.",
                }, []string{"nomad_job"}),
                nomadTokenRefreshes: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_token_refreshes_total",
                        Help: "Re-reads of the Nomad token file (--nomad-token-file), by result: rotated (the token changed and was applied), error (the file could not be read; previous token kept).",
                }, []string{"result"}),
                nomadLogins: f.NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_logins_total",
                        Help: "Workload-identity token exchanges via /v1/acl/login (--nomad-login-auth-method), by result: success (a fresh ACL token was obtained and applied) or error (the exchange failed; previous token kept).",
                }, []string{"result"}),
        }

        // Pre-populate Vec metrics for all finite label values so they appear in
        // Gather() output (with value 0) even before the first Check call.
        for _, src := range []string{"hcl", "nomad"} {
                d.jobsSkippedBySel.WithLabelValues(src)
        }
        for _, op := range []string{"list", "info", "plan", "register", "deregister", "versions", "deployments", "deployment", "revert", "tag"} {
                d.nomadAPIErrors.WithLabelValues(op)
        }
        for _, st := range []JobUpdateStatus{JobUpdateStatusSucceeded, JobUpdateStatusFailed, JobUpdateStatusSuperseded} {
                d.jobUpdatesTotal.WithLabelValues(string(JobUpdateOperationRegister), string(st))
                d.jobUpdatesTotal.WithLabelValues(string(JobUpdateOperationDeregister), string(st))
                d.jobUpdatesTotal.WithLabelValues(string(JobUpdateOperationRevert), string(st))
        }
        for _, dt := range []string{string(DiffTypeModified), string(DiffTypeMissingFromNomad), string(DiffTypeMissingFromHCL)} {
                d.driftedJobs.WithLabelValues(dt)
        }
        for _, r := range []string{"rotated", "error"} {
                d.nomadTokenRefreshes.WithLabelValues(r)
        }
        for _, r := range []string{"success", "error"} {
                d.nomadLogins.WithLabelValues(r)
        }

        return d
}

// NewDiffer creates a Differ backed by a real Nomad API client, registering
// metrics into the default Prometheus registry.
func NewDiffer(cfg *config.Config) (*Differ, error) {
        return NewDifferWithRegistry(cfg, prometheus.DefaultRegisterer)
}

// NewDifferWithRegistry is like NewDiffer but registers metrics into reg rather
// than the default registry, so more than one real-client Differ can be built
// in a single process (e.g. tests exercising token resolution, or embedding)
// without a duplicate-registration panic.
func NewDifferWithRegistry(cfg *config.Config, reg prometheus.Registerer) (*Differ, error) {
        nomadCfg := nomadapi.DefaultConfig()
        nomadCfg.Address = cfg.NomadAddr

        loginMode := cfg.NomadLoginAuthMethod != ""

        var token, watchPath string
        if !loginMode {
                var err error
                token, watchPath, err = resolveNomadToken(cfg)
                if err != nil {
                        return nil, err
                }
        }
        // Set the token explicitly (possibly to "") so it always reflects our
        // resolution and overrides whatever DefaultConfig read from NOMAD_TOKEN.
        nomadCfg.SecretID = token

        client, err := nomadapi.NewClient(nomadCfg)
        if err != nil {
                return nil, fmt.Errorf("creating nomad client: %w", err)
        }

        d := newDifferBase(client.Jobs(), cfg, reg)
        d.nomadClient = client
        d.tokenPollInterval = cfg.NomadTokenPollInterval

        if loginMode {
                d.loginAuthMethod = cfg.NomadLoginAuthMethod
                d.loginJWTFile = loginJWTPath(cfg)
                if d.loginJWTFile == "" {
                        return nil, fmt.Errorf("--nomad-login-auth-method is set but no JWT file is available: set --nomad-login-jwt-file or run where NOMAD_SECRETS_DIR is set")
                }
                slog.Info("Authenticating to Nomad with workload-identity login (JWT exchange)",
                        "auth_method", d.loginAuthMethod, "jwt_file", d.loginJWTFile)
                // Exchange once at startup so the first diff check has a token. A failure
                // here is not fatal — the refresher retries — but is logged clearly.
                secretID, expiry, lerr := d.login()
                if lerr != nil {
                        d.loginFailed = true
                        d.nomadLogins.WithLabelValues("error").Inc()
                        slog.Error("Initial Nomad workload-identity login failed; will retry. Check the auth method name, that the JWT's audience matches the method, and that a binding rule maps this job to a policy",
                                "auth_method", d.loginAuthMethod, "jwt_file", d.loginJWTFile, "err", lerr)
                } else {
                        client.SetSecretID(secretID)
                        d.loginExpiry = expiry
                        d.nomadLogins.WithLabelValues("success").Inc()
                        slog.Info("Obtained a Nomad ACL token via workload-identity login", "expires", fmtExpiry(expiry))
                }
                return d, nil
        }

        d.tokenFilePath = watchPath
        d.initialToken = token
        switch {
        case watchPath != "":
                if looksLikeJWT(token) {
                        slog.Warn("The token file looks like a workload-identity JWT, not an ACL SecretID; a raw WI JWT is rejected by Nomad's Job.Plan RPC. Use --nomad-login-auth-method to exchange it (see docs/setup/nomad-access.md)",
                                "token_file", watchPath)
                }
                slog.Info("Authenticating to Nomad with a token file", "token_file", watchPath, "refresh_interval", cfg.NomadTokenPollInterval)
        case token != "":
                if looksLikeJWT(token) {
                        slog.Warn("The static Nomad token looks like a workload-identity JWT, not an ACL SecretID; use --nomad-login-auth-method to exchange it (see docs/setup/nomad-access.md)")
                }
                slog.Info("Authenticating to Nomad with a static token")
        default:
                // Anonymous. If a workload-identity token file is sitting there unused,
                // the operator almost certainly meant to authenticate — point them at
                // login exchange rather than letting every plan fail silently.
                if p := defaultWorkloadTokenPath(); p != "" {
                        slog.Warn("A workload-identity token file is present but no Nomad authentication is configured. Raw WI tokens are rejected by Nomad's Job.Plan RPC (issue #74); set --nomad-login-auth-method to exchange the identity JWT for an ACL token (see docs/setup/nomad-access.md)",
                                "token_file", p)
                } else {
                        slog.Info("No Nomad token configured; using anonymous access (works only when ACLs are disabled)")
                }
        }
        return d, nil
}

// login exchanges the workload-identity JWT for a real ACL token via
// /v1/acl/login, returning the SecretID and its expiry.
func (d *Differ) login() (secretID string, expiry *time.Time, err error) {
        jwt, err := readTokenFile(d.loginJWTFile)
        if err != nil {
                return "", nil, err
        }
        if jwt == "" {
                return "", nil, fmt.Errorf("workload-identity JWT file %q is empty", d.loginJWTFile)
        }
        // A WI JWT with no expiry means the identity block has no ttl: Nomad issues a
        // non-expiring token and never rewrites the file, so login works now but
        // fails once the exchanged ACL token expires (issue #76). Warn once, at the
        // first login (startup), so this silent, delayed failure is caught early.
        if jwtLacksExpiry(jwt) {
                d.loginNoTTLWarned.Do(func() {
                        slog.Warn("Workload-identity JWT has no expiry (exp) claim: the identity block is missing a ttl, so Nomad will not renew the token file. Login will start failing once the current exchanged token expires (after the auth method's max_token_ttl). Set ttl on the identity block, e.g. `ttl = \"1h\"` with `change_mode = \"noop\"` (see docs/setup/nomad-access.md).",
                                "jwt_file", d.loginJWTFile)
                })
        }
        tok, _, err := d.nomadClient.ACLAuth().Login(&nomadapi.ACLLoginRequest{
                AuthMethodName: d.loginAuthMethod,
                LoginToken:     jwt,
        }, &nomadapi.WriteOptions{Namespace: d.namespace})
        if err != nil {
                return "", nil, err
        }
        return tok.SecretID, tok.ExpirationTime, nil
}

// fmtExpiry renders a token expiry for logging.
func fmtExpiry(t *time.Time) string {
        if t == nil {
                return "never"
        }
        return t.UTC().Format(time.RFC3339)
}

// RunTokenRefresher keeps the Nomad token current: in login mode it re-exchanges
// the workload-identity JWT before the ACL token expires; in file mode it
// re-reads the token file. It is a no-op for a static token or no token, and
// blocks until ctx is cancelled.
func (d *Differ) RunTokenRefresher(ctx context.Context) {
        if d.nomadClient == nil {
                return
        }
        if d.loginAuthMethod != "" {
                firstDelay := nextLoginDelay(d.loginExpiry)
                if d.loginFailed {
                        firstDelay = loginRetryBackoff
                }
                runLoginRefresher(ctx, firstDelay,
                        d.login,
                        func(secretID string) {
                                d.nomadClient.SetSecretID(secretID)
                                d.nomadLogins.WithLabelValues("success").Inc()
                                slog.Info("Re-exchanged the Nomad workload-identity token via login")
                        },
                        func(err error) {
                                d.nomadLogins.WithLabelValues("error").Inc()
                                slog.Warn("Nomad workload-identity login failed; keeping the previous token and retrying", "auth_method", d.loginAuthMethod, "err", err)
                        },
                )
                return
        }
        if d.tokenFilePath == "" {
                return
        }
        interval := d.tokenPollInterval
        if interval <= 0 {
                interval = 30 * time.Second
        }
        refreshTokenFile(ctx, d.tokenFilePath, interval, d.initialToken,
                func(tok string) {
                        d.nomadClient.SetSecretID(tok)
                        d.nomadTokenRefreshes.WithLabelValues("rotated").Inc()
                        slog.Info("Applied a rotated Nomad token from the token file", "token_file", d.tokenFilePath)
                },
                func(err error) {
                        d.nomadTokenRefreshes.WithLabelValues("error").Inc()
                        slog.Warn("Could not re-read the Nomad token file; keeping the previous token", "token_file", d.tokenFilePath, "err", err)
                },
        )
}

// NewWithClient creates a Differ with a custom jobs client, intended for tests.
func NewWithClient(cfg *config.Config, jobs NomadJobsClient) *Differ {
        return newDifferBase(jobs, cfg, prometheus.NewRegistry())
}

// NewWithClientAndRegistry creates a Differ with a custom jobs client and Prometheus
// registry. Use this in tests that need to inspect metric values.
func NewWithClientAndRegistry(cfg *config.Config, jobs NomadJobsClient, reg prometheus.Registerer) *Differ {
        return newDifferBase(jobs, cfg, reg)
}

// metaKeyPresent reports whether the managed meta key is set in meta.
func (d *Differ) metaKeyPresent(meta map[string]string) bool {
        return d.managedMetaPrefix != "" && meta[d.managedMetaPrefix+"_managed"] == "true"
}

// selectionReasonFor maps a (glob match, meta match) pair to a SelectionReason.
// Returns (false, "") when neither criterion is met.
func selectionReasonFor(glob, meta bool) (bool, SelectionReason) {
        switch {
        case glob && meta:
                return true, SelectionReasonBoth
        case glob:
                return true, SelectionReasonGlob
        case meta:
                return true, SelectionReasonMeta
        default:
                return false, ""
        }
}

// jobSelectionReason reports whether a job should be watched and, if so, why.
func (d *Differ) jobSelectionReason(jobID string, meta map[string]string) (bool, SelectionReason) {
        glob := d.jobSelectorGlob != ""
        if glob {
                matched, _ := path.Match(d.jobSelectorGlob, jobID)
                glob = matched
        }
        return selectionReasonFor(glob, d.metaKeyPresent(meta))
}

// mergeSelectionReason returns the combined reason when a job is seen from
// multiple sources (e.g. HCL phase and Nomad phase). Any combination of two
// different non-empty reasons upgrades to SelectionReasonBoth.
func mergeSelectionReason(existing, incoming SelectionReason) SelectionReason {
        if existing == "" {
                return incoming
        }
        if existing == incoming {
                return existing
        }
        return SelectionReasonBoth
}

// SelectedJobs returns a snapshot of the jobs that matched the configured
// selection criteria during the last check, together with the reason each
// matched. The second and third return values are the same last-check time and
// commit as Diffs().
func (d *Differ) SelectedJobs() ([]SelectedJob, time.Time, string) {
        d.mu.RLock()
        defer d.mu.RUnlock()
        result := make([]SelectedJob, len(d.selectedJobs))
        copy(result, d.selectedJobs)
        return result, d.lastCheckTime, d.lastCommit
}

// parseHCLCandidates parses all HCL files and returns those that pass the
// selection filter (glob match or managed meta key in HCL), plus the set of
// every successfully parsed job ID — selected or not — so the Nomad-side
// phase knows which jobs Git has an opinion about.
// metaSeen collects each parsed job's prefix-key snapshot for change tracking.
func (d *Differ) parseHCLCandidates(hclFiles map[string]string, metaSeen map[string]metaState) (map[string]hclEntry, map[string]struct{}) {
        entries := make(map[string]hclEntry)
        parsedIDs := make(map[string]struct{})
        for filename, content := range hclFiles {
                if !jobBlockRe.MatchString(content) {
                        slog.Debug("Skipping HCL file with no job stanza", "file", filename)
                        d.hclFilesSkipped.Inc()
                        continue
                }
                job, err := d.jobs.ParseHCL(content, true)
                if err != nil {
                        slog.Warn("Failed to parse HCL file, skipping", "file", filename, "err", err)
                        d.hclParseErrors.Inc()
                        continue
                }
                if job == nil || job.ID == nil || *job.ID == "" {
                        slog.Warn("HCL file yielded no job ID, skipping", "file", filename)
                        continue
                }
                jobID := *job.ID

                // Flag prefix-addressed meta keys we cannot act on before the
                // selection filter: a typo'd opt-in key is exactly what makes a job
                // silently unselected.
                d.validateManagedMeta(jobID, "hcl:"+filename, job.Meta)
                recordMetaSeen(metaSeen, d, "hcl", jobID, job.Meta)
                parsedIDs[jobID] = struct{}{}

                globSel := d.jobSelectorGlob != ""
                if globSel {
                        globSel, _ = path.Match(d.jobSelectorGlob, jobID)
                }
                metaHCL := d.metaKeyPresent(job.Meta)

                if !globSel && !metaHCL {
                        slog.Debug("Skipping job not matching selection criteria", "job", jobID, "file", filename)
                        d.jobsSkippedBySel.WithLabelValues("hcl").Inc()
                        continue
                }
                entries[jobID] = hclEntry{job: job, file: filename, globSel: globSel, metaHCL: metaHCL}
                slog.Debug("Parsed HCL file", "file", filename, "job_id", jobID)
        }
        return entries, parsedIDs
}

// updateCandidate carries the context needed to turn a detected diff into a
// JobUpdate: the parsed job, the CAS token, and the diff classification.
// Policy gating happens later, in maybeEnqueueUpdate.
type updateCandidate struct {
        jobID        string
        hclFile      string
        job          *nomadapi.Job
        modifyIndex  uint64
        class        DiffClass
        isCreation   bool               // job absent (or dead) in Nomad: first-time registration
        globSel      bool               // selected by job-selector-glob (no opt-in moment)
        operation    JobUpdateOperation // REGISTER (default) or DEREGISTER
        policy       UpdatePolicy       // effective policy, for DEREGISTER candidates (from live meta)
        action       ApplyAction        // disposition, decided in decideApplyAction
        actionDetail string             // optional specific explanation refining action
}

// jobModifyIndex safely extracts a job's ModifyIndex.
func jobModifyIndex(j *nomadapi.Job) uint64 {
        if j == nil || j.JobModifyIndex == nil {
                return 0
        }
        return *j.JobModifyIndex
}

// checkHCLCandidate applies final selection against the live Nomad job, then
// runs Info + Plan to produce a diff. Returns (false, "", nil, nil) when the
// job should be skipped; (true, reason, nil, nil) when selected but no diff
// was found; (true, reason, &diff, cand) when a diff was detected. cand is
// non-nil only when the diff is actionable (a REGISTER could resolve it).
func (d *Differ) checkHCLCandidate(jobID string, entry hclEntry, q *nomadapi.QueryOptions, wq *nomadapi.WriteOptions) (bool, SelectionReason, *JobDiff, *updateCandidate) {
        nomadJob, _, infoErr := d.jobs.Info(jobID, q)
        notFound := infoErr != nil && isNotFound(infoErr)

        // Git is always the source of truth for nomad-gitops's own behaviour:
        // when a job has an HCL file, its keys alone decide selection. The opt-in
        // key in HCL selects the job even when the live copy does not carry it
        // yet — the key's absence on the live job is itself drift, and applying
        // it (policy permitting) is how the live meta converges. A live key with
        // no HCL counterpart never selects; it is surfaced as a meta-change
        // notice instead.
        _, reason := selectionReasonFor(entry.globSel, entry.metaHCL)

        if notFound {
                diff := &JobDiff{
                        JobID:    jobID,
                        HCLFile:  entry.file,
                        DiffType: DiffTypeMissingFromNomad,
                        Detail:   "job is defined in HCL but not registered in Nomad",
                }
                // ModifyIndex 0 with EnforceIndex means "job must not exist", which
                // is exactly the guard a first registration wants.
                cand := &updateCandidate{
                        jobID: jobID, hclFile: entry.file, job: entry.job,
                        modifyIndex: 0, class: DiffClassOther, isCreation: true, globSel: entry.globSel,
                }
                return true, reason, diff, cand
        }
        if infoErr != nil {
                d.nomadAPIErrors.WithLabelValues("info").Inc()
                slog.Warn("Failed to query job from Nomad", "job", jobID, "err", infoErr)
                return true, reason, nil, nil
        }

        // Unless the caller explicitly wants dead jobs included, treat a dead
        // job the same as a missing one.
        if !d.includeDeadJobs && nomadJob != nil && nomadJob.Status != nil && *nomadJob.Status == "dead" {
                slog.Debug("Job is dead in Nomad, treating as missing", "job", jobID)
                diff := &JobDiff{
                        JobID:    jobID,
                        HCLFile:  entry.file,
                        DiffType: DiffTypeMissingFromNomad,
                        Detail:   "job is defined in HCL but is in 'dead' state in Nomad",
                }
                // The dead job still exists in Nomad's state store, so the CAS
                // token is its current ModifyIndex, not 0.
                cand := &updateCandidate{
                        jobID: jobID, hclFile: entry.file, job: entry.job,
                        modifyIndex: jobModifyIndex(nomadJob), class: DiffClassOther, isCreation: true, globSel: entry.globSel,
                }
                return true, reason, diff, cand
        }

        // Job exists and is live — run a plan to detect config drift.
        // Nomad's deregister call sets Stop=true on the job record. Copy it
        // onto the HCL job so the plan does not report a Stop field diff.
        job := entry.job
        if nomadJob.Stop != nil && *nomadJob.Stop {
                stop := true
                job.Stop = &stop
        }
        plan, _, err := d.jobs.Plan(job, true, wq)
        if err != nil {
                d.nomadAPIErrors.WithLabelValues("plan").Inc()
                slog.Warn("Failed to plan job", "job", jobID, "err", err)
                return true, reason, nil, nil
        }

        if plan.Diff != nil && plan.Diff.Type != "" && plan.Diff.Type != "None" {
                // For dead jobs, Nomad may return Type="Edited" with only task-group
                // bookkeeping entries (Type="None" task groups). hasContentDiff filters
                // those out so we don't report spurious drift.
                isDead := nomadJob != nil && nomadJob.Status != nil && *nomadJob.Status == "dead"
                if isDead && !hasContentDiff(plan.Diff) {
                        return true, reason, nil, nil
                }
                // Classify before redaction; classification reads only structure
                // (names and change types), but doing it first keeps the order
                // obviously safe.
                class := classifyDiff(plan.Diff, autoscaledGroups(entry.job), d.managedMetaPrefix)
                // Redact before the diff is stored so potentially sensitive values
                // never reach /diffs or any other consumer of the stored state.
                if d.redactSecrets {
                        if n := RedactJobDiff(plan.Diff); n > 0 {
                                d.redactedFields.Add(float64(n))
                        }
                }
                diff := &JobDiff{
                        JobID:    jobID,
                        HCLFile:  entry.file,
                        DiffType: DiffTypeModified,
                        Detail:   fmt.Sprintf("Nomad plan shows diff type %q", plan.Diff.Type),
                        PlanDiff: plan.Diff,
                }
                cand := &updateCandidate{
                        jobID: jobID, hclFile: entry.file, job: entry.job,
                        modifyIndex: jobModifyIndex(nomadJob), class: class, globSel: entry.globSel,
                }
                return true, reason, diff, cand
        }
        return true, reason, nil, nil
}

// commitResults stores the computed diffs and updates drift tracking and metrics.
// It is called at the end of each Check to atomically publish new state.
func (d *Differ) commitResults(diffs []JobDiff, selReasons map[string]SelectionReason, commit string, listMeta *nomadapi.QueryMeta, now time.Time) {
        currentKeys := make(map[string]struct{}, len(diffs))
        for _, diff := range diffs {
                currentKeys[driftKey(diff.JobID, string(diff.DiffType))] = struct{}{}
        }

        selectedJobs := make([]SelectedJob, 0, len(selReasons))
        for jobID, reason := range selReasons {
                selectedJobs = append(selectedJobs, SelectedJob{JobID: jobID, Reason: reason})
        }
        sort.Slice(selectedJobs, func(i, j int) bool { return selectedJobs[i].JobID < selectedJobs[j].JobID })

        d.mu.Lock()
        d.diffs = diffs
        d.selectedJobs = selectedJobs
        d.lastCheckTime = now
        d.lastCommit = commit
        if listMeta != nil {
                d.lastNomadIndex = listMeta.LastIndex
        }
        for k := range d.driftFirstSeen {
                if _, ok := currentKeys[k]; !ok {
                        delete(d.driftFirstSeen, k)
                }
        }
        for k := range currentKeys {
                if _, ok := d.driftFirstSeen[k]; !ok {
                        d.driftFirstSeen[k] = now
                }
        }
        firstSeenSnapshot := make(map[string]time.Time, len(d.driftFirstSeen))
        for k, v := range d.driftFirstSeen {
                firstSeenSnapshot[k] = v
        }
        d.mu.Unlock()

        d.lastCheck.Set(float64(now.Unix()))
        d.jobDiffs.Reset()
        d.driftedJobs.Reset()
        d.jobDriftSince.Reset()
        typeCounts := make(map[string]int)
        for _, diff := range diffs {
                d.jobDiffs.WithLabelValues(diff.JobID, string(diff.DiffType)).Set(1)
                typeCounts[string(diff.DiffType)]++
        }
        for typ, count := range typeCounts {
                d.driftedJobs.WithLabelValues(typ).Set(float64(count))
        }
        for _, diff := range diffs {
                k := driftKey(diff.JobID, string(diff.DiffType))
                if t, ok := firstSeenSnapshot[k]; ok {
                        d.jobDriftSince.WithLabelValues(diff.JobID, string(diff.DiffType)).Set(float64(t.Unix()))
                }
        }
}

// Check compares the given HCL files (path → content) against the live Nomad
// cluster and stores the results. commit is recorded for informational purposes.
func (d *Differ) Check(hclFiles map[string]string, commit string) error {
        // ?meta=true is documented in the Nomad HTTP API (GET /v1/jobs) and
        // causes the list response to include each job's Meta map. Without it,
        // Meta is omitted from the stub and meta-prefix selection cannot work.
        q := &nomadapi.QueryOptions{
                Namespace: d.namespace,
                Params:    map[string]string{"meta": "true"},
        }
        wq := &nomadapi.WriteOptions{Namespace: d.namespace}

        // List all Nomad jobs first. The returned Raft index lets us skip the
        // expensive per-job work when neither Nomad state nor the git commit has
        // changed since the last check.
        allJobs, listMeta, err := d.jobs.List(q)
        if err != nil {
                d.nomadAPIErrors.WithLabelValues("list").Inc()
                slog.Warn("Failed to list Nomad jobs", "err", err)
                allJobs = nil
                listMeta = nil
        }

        d.mu.RLock()
        prevCommit := d.lastCommit
        prevIndex := d.lastNomadIndex
        d.mu.RUnlock()

        if listMeta != nil && listMeta.LastIndex == prevIndex && commit == prevCommit {
                slog.Debug("Skipping diff: Nomad index and commit unchanged", "index", listMeta.LastIndex, "commit", commit)
                d.diffChecksSkipped.Inc()
                d.lastCheck.Set(float64(time.Now().Unix()))
                return nil
        }

        slog.Info("Running diff check", "commit", commit, "hcl_files", len(hclFiles))
        d.diffChecks.Inc()

        metaSeen := make(map[string]metaState)
        hclEntries, parsedIDs := d.parseHCLCandidates(hclFiles, metaSeen)
        // hclJobSet tracks jobs that passed HCL-phase selection; used below to
        // detect jobs running in Nomad that have no corresponding HCL file.
        hclJobSet := make(map[string]struct{}, len(hclEntries))
        selReasons := make(map[string]SelectionReason)
        // metaByJob holds the HCL meta of each managed job that has an HCL file in
        // the repo, used by the rollback poll. Active rollback is deliberately
        // scoped to HCL-defined jobs: a job running in Nomad with no HCL is either
        // unmanaged or an orphan leaving management, not something nomad-gitops
        // drives applies for, so it is not a rollback candidate.
        metaByJob := make(map[string]map[string]string)
        var diffs []JobDiff
        var candidates []*updateCandidate

        for jobID, entry := range hclEntries {
                selected, reason, diff, cand := d.checkHCLCandidate(jobID, entry, q, wq)
                if !selected {
                        continue
                }
                selReasons[jobID] = mergeSelectionReason(selReasons[jobID], reason)
                hclJobSet[jobID] = struct{}{}
                metaByJob[jobID] = entry.job.Meta
                if cand != nil {
                        // Decide the disposition once; it is recorded on the diff (for the
                        // API/console) and drives whether the update is enqueued below.
                        cand.action = d.decideApplyAction(cand, commit, q)
                }
                if diff != nil {
                        metaOnly := cand != nil && cand.class == DiffClassManagedMetaOnly
                        if metaOnly {
                                d.metaOnlyDiffs.WithLabelValues(jobID).Inc()
                        }
                        if cand != nil {
                                diff.ApplyAction = cand.action
                                diff.ApplyDetail = cand.actionDetail
                        }
                        // A diff confined to our own meta keys is an expected,
                        // non-disruptive difference. By default it is not counted as
                        // drift (so it does not trigger alerts) — it is surfaced via its
                        // own counter and the meta-change logs instead, and converges on
                        // the next real update.
                        if !metaOnly || d.countMetaOnlyChanges {
                                diffs = append(diffs, *diff)
                        }
                }
                if cand != nil {
                        candidates = append(candidates, cand)
                }
        }

        // Find jobs in Nomad that have no corresponding HCL file.
        // Dead jobs are skipped unless --include-dead-jobs is set, since a dead
        // job without HCL is expected (it was stopped intentionally).
        // Only jobs that match the configured selection criteria are considered managed.
        for _, j := range allJobs {
                // Validate before any skip: a live job with a malformed opt-in key
                // is silently out of scope, which is the failure worth surfacing.
                if _, inHCL := hclJobSet[j.ID]; !inHCL {
                        d.validateManagedMeta(j.ID, "nomad", j.Meta)
                }
                // Track the live side for every job, including managed ones: a
                // manual `nomad job run` that drops the keys is exactly the
                // meta-drift event worth noticing.
                recordMetaSeen(metaSeen, d, "nomad", j.ID, j.Meta)
                if !d.includeDeadJobs && j.Status == "dead" {
                        continue
                }
                // Git is always the source of truth for our own keys: when the job
                // has an HCL file, that file alone already decided selection in the
                // HCL phase. A live key on a job whose HCL does not opt in never
                // overrides Git; it is only surfaced via meta validation and
                // change notices.
                if _, parsed := parsedIDs[j.ID]; parsed {
                        if _, ok := hclJobSet[j.ID]; !ok {
                                d.jobsSkippedBySel.WithLabelValues("nomad").Inc()
                        }
                        continue
                }
                // Meta is populated because the List call includes ?meta=true.
                selected, reason := d.jobSelectionReason(j.ID, j.Meta)
                if !selected {
                        d.jobsSkippedBySel.WithLabelValues("nomad").Inc()
                        continue
                }
                selReasons[j.ID] = mergeSelectionReason(selReasons[j.ID], reason)
                if _, ok := hclJobSet[j.ID]; !ok {
                        diff := JobDiff{
                                JobID:       j.ID,
                                DiffType:    DiffTypeMissingFromHCL,
                                Detail:      fmt.Sprintf("job is running in Nomad (status: %s) but has no HCL definition in the repo", j.Status),
                                ApplyAction: ApplyActionObservationOnly,
                        }
                        // A job carrying our tag in its live meta with no HCL declaring it
                        // is an orphan — removed from the repo (file deleted or renamed),
                        // since tag-removal-with-file-present is excluded by the parsedIDs
                        // skip above. Such a job is a deregister candidate. A glob-only
                        // orphan (no tag) is never deregistered; it stays observation-only.
                        if d.metaKeyPresent(j.Meta) {
                                cand := &updateCandidate{
                                        jobID:     j.ID,
                                        operation: JobUpdateOperationDeregister,
                                        policy:    d.effectivePolicy(j.Meta),
                                }
                                cand.action = d.decideDeregisterAction(cand)
                                diff.ApplyAction = cand.action
                                diff.ApplyDetail = cand.actionDetail
                                candidates = append(candidates, cand)
                        }
                        diffs = append(diffs, diff)
                }
        }

        d.commitResults(diffs, selReasons, commit, listMeta, time.Now())
        d.logMetaChanges(metaSeen)
        liveJobs := make(map[string]string, len(allJobs))
        for _, j := range allJobs {
                liveJobs[j.ID] = j.Status
        }
        d.logScopeExits(hclJobSet, parsedIDs, liveJobs)

        var raftIndex uint64
        if listMeta != nil {
                raftIndex = listMeta.LastIndex
        }
        enqueued := 0
        for _, cand := range candidates {
                if cand.action == ApplyActionQueued || cand.action == ApplyActionDeregisterQueued {
                        d.enqueueUpdate(cand, commit, raftIndex)
                        enqueued++
                }
        }
        if enqueued > 0 {
                d.notifyApplier()
        }

        // Active rollback poll: for managed deployment-producing jobs that have
        // rollback enabled, revert a failed deployment to the last stable version
        // (unless the job uses auto_revert, where Nomad wins). Cheap and skipped
        // entirely when no managed job opts in.
        d.checkRollbacks(metaByJob, q, raftIndex)

        slog.Info("Diff check complete", "diffs", len(diffs), "updates_enqueued", enqueued, "commit", commit)
        return nil
}

// effectivePolicy resolves the update policy for a job: the HCL meta key
// <prefix>_update_policy wins (Git is intent); otherwise the configured
// default applies. An unrecognised meta value is treated as "none" — the
// conservative reading — and logged.
func (d *Differ) effectivePolicy(meta map[string]string) UpdatePolicy {
        if d.managedMetaPrefix != "" {
                if v, ok := meta[d.managedMetaPrefix+"_update_policy"]; ok {
                        if ValidUpdatePolicy(v) {
                                return UpdatePolicy(v)
                        }
                        // Already logged at ERROR by validateManagedMeta during parsing.
                        return UpdatePolicyNone
                }
        }
        return d.defaultPolicy
}

// policyMetaKey is the meta key that overrides the update policy per job.
func (d *Differ) policyMetaKey() string {
        return d.managedMetaPrefix + "_update_policy"
}

// policySource describes where a job's effective update policy comes from, for
// operator-facing messages: the per-job HCL meta key (only when its value is a
// recognised policy) or the configured default. A present-but-invalid meta
// value is handled separately by policyBlockedDetail, since effectivePolicy
// coerces it to "none" and the raw value is what the operator needs to see.
func (d *Differ) policySource(meta map[string]string) string {
        if d.managedMetaPrefix != "" {
                if v, ok := meta[d.policyMetaKey()]; ok && ValidUpdatePolicy(v) {
                        return "set by " + d.policyMetaKey() + " in the job's HCL meta"
                }
        }
        return "the --default-update-policy default"
}

// policyBlockedDetail explains, for the /diffs view and JSON API, exactly why a
// candidate's effective update policy stops this change from being applied: the
// policy value, where it came from, and what would need to change to apply it.
func (d *Differ) policyBlockedDetail(c *updateCandidate, policy UpdatePolicy) string {
        // A present-but-unrecognised meta value is coerced to "none" by
        // effectivePolicy. Report the actual value and the coercion rather than
        // claiming the operator asked for "none".
        if d.managedMetaPrefix != "" {
                if v, ok := c.job.Meta[d.policyMetaKey()]; ok && !ValidUpdatePolicy(v) {
                        return fmt.Sprintf("not applied: %s is set to %q in the job's HCL meta, which is not a valid update policy, so it is treated as %q and never applies drift for this job. Set %s to \"image-only\" or \"full\" to enable applies.", d.policyMetaKey(), v, UpdatePolicyNone, d.policyMetaKey())
                }
        }
        src := d.policySource(c.job.Meta)
        switch policy {
        case UpdatePolicyNone:
                return fmt.Sprintf("not applied: update policy is %q (%s), which never applies drift for this job. Set the policy to \"image-only\" or \"full\" to enable applies.", policy, src)
        case UpdatePolicyImageOnly:
                if c.isCreation {
                        return fmt.Sprintf("not applied: update policy is %q (%s), and a first-time registration is not an image-only change. Set the policy to \"full\" to allow creating this job.", policy, src)
                }
                return fmt.Sprintf("not applied: update policy is %q (%s), but this change modifies more than the container image. Set the policy to \"full\" to apply it.", policy, src)
        }
        return ""
}

// SetHistorySource wires the git-history accessor used to detect pre-existing
// drift. Called once at startup after the watcher exists.
func (d *Differ) SetHistorySource(h HistorySource) {
        d.history = h
}

// isPreExistingDrift reports whether a candidate's drift pre-dates the change,
// at the commit being evaluated, that brought it into scope to apply. Two such
// scope-widening changes are treated the same way (issue #69):
//
//   - Enablement: the managed opt-in tag (<prefix>_managed) was added at this
//     commit (absent in the parent version of the file). The whole job entered
//     management; its existing drift pre-dates the opt-in.
//   - Policy promotion: the job was managed at the parent, but its effective
//     update policy there would not have applied this diff's class, while the
//     policy at this commit does (e.g. image-only → full applying a memory
//     change that image-only had been deferring). The drift was live the whole
//     time, merely held back by the stricter policy.
//
// In both cases the conservative default is not to retroactively apply the
// accumulated drift: changing scope expresses intent about future
// reconciliation, not "deploy the backlog now". --apply-existing-drift opts in
// to applying it.
//
// Derived from git history, so it holds identically whether the change landed
// while the process was running or before it started. Glob-selected jobs are
// always in scope and have no opt-in moment, so they are never pre-existing.
// Creations have no live job to pre-date. When history is unavailable the check
// is skipped (not pre-existing) so reconciliation is not broken.
func (d *Differ) isPreExistingDrift(c *updateCandidate, commit string) bool {
        if c.isCreation || c.globSel || d.history == nil || d.managedKeyRe == nil || c.hclFile == "" {
                return false
        }
        parent, ok := d.history.FileAtParentOf(commit, c.hclFile)
        if !ok {
                // No parent version: the file was created at this commit (or it is the
                // root commit). The tag, policy, and spec were introduced together, so
                // nothing was deferred under an earlier scope — not retroactive.
                return false
        }
        if !d.managedKeyRe.MatchString(parent) {
                // The opt-in tag was added at this commit: the job entered management
                // here, so its drift pre-dates the opt-in.
                return true
        }
        // Managed at the parent too. A managed-meta-only diff is governed by the
        // meta-only gate, not this one.
        if c.class == DiffClassManagedMetaOnly {
                return false
        }
        // Pre-existing iff the parent's effective policy would not have applied this
        // diff's class but the current one does — a scope-widening policy change at
        // this commit.
        return !policyPermits(d.effectivePolicyFromText(parent), c.class) &&
                policyPermits(d.effectivePolicy(c.job.Meta), c.class)
}

// policyPermits reports whether an update policy would apply a diff of the given
// class. Mirrors the policy gate in decideApplyAction (creation, which is
// full-only, is handled separately and never reaches the pre-existing check).
func policyPermits(policy UpdatePolicy, class DiffClass) bool {
        switch policy {
        case UpdatePolicyFull:
                return true
        case UpdatePolicyImageOnly:
                return class == DiffClassImageOnly
        default: // none, or an unrecognised value treated as none
                return false
        }
}

// effectivePolicyFromText resolves a job's update policy from a raw HCL snapshot
// (e.g. a parent commit's version of the file), without re-parsing: the
// <prefix>_update_policy value wins, an unrecognised value is treated as none
// (matching effectivePolicy), and an absent key falls back to the default.
func (d *Differ) effectivePolicyFromText(hclText string) UpdatePolicy {
        if d.policyKeyRe != nil {
                if m := d.policyKeyRe.FindStringSubmatch(hclText); m != nil {
                        if ValidUpdatePolicy(m[1]) {
                                return UpdatePolicy(m[1])
                        }
                        return UpdatePolicyNone
                }
        }
        return d.defaultPolicy
}

// logScopeExits logs, once per transition, when a job leaves active GitOps
// management. managed is this cycle's HCL-managed set, parsedIDs the jobs
// present in the repo at all, and liveJobs maps job ID to Nomad status. A job
// that was managed last cycle and is not now either had its tag removed (still
// in the repo — already reported by the meta-change log) or was removed from
// the repo entirely (file deleted or job renamed). prevManaged is nil on the
// first cycle, so a restart logs nothing.
func (d *Differ) logScopeExits(managed, parsedIDs map[string]struct{}, liveJobs map[string]string) {
        d.metaMu.Lock()
        defer d.metaMu.Unlock()

        if d.prevManaged != nil {
                for jobID := range d.prevManaged {
                        if _, still := managed[jobID]; still {
                                continue
                        }
                        if _, inRepo := parsedIDs[jobID]; inRepo {
                                // Tag removed but the job is still in the repo: the meta-change
                                // log already carries the human message; just count it.
                                d.jobsLeftManagement.WithLabelValues(jobID, "tag_removed").Inc()
                                continue
                        }
                        d.jobsLeftManagement.WithLabelValues(jobID, "removed_from_repo").Inc()
                        if status, running := liveJobs[jobID]; running {
                                slog.Info("Job left GitOps management: removed from the repo (file deleted or job renamed); the running job is left untouched",
                                        "job", jobID, "nomad_status", status, "deregister_enabled", d.enableDeregister)
                        } else {
                                slog.Info("Job left GitOps management: removed from the repo and no longer present in Nomad",
                                        "job", jobID)
                        }
                }
        }

        next := make(map[string]bool, len(managed))
        for jobID := range managed {
                next[jobID] = true
        }
        d.prevManaged = next
}

// decideDeregisterAction decides what to do with an orphaned managed job (a
// tagged live job removed from the repo). Gated, most-conservative first:
// deregistration must be enabled, the job's effective policy must be full, and
// it must have been orphaned for the grace period. Anything short of that
// leaves the job running (observation, policy-blocked, or grace-pending).
func (d *Differ) decideDeregisterAction(c *updateCandidate) ApplyAction {
        if !d.enableDeregister {
                return ApplyActionObservationOnly
        }
        if c.policy != UpdatePolicyFull {
                d.updatesBlockedByPolicy.WithLabelValues(c.jobID, string(c.policy)).Inc()
                c.actionDetail = fmt.Sprintf("not applied: deregistration requires update policy \"full\", but this job's effective policy is %q. A job removed from the repo is only deregistered under a full policy.", c.policy)
                return ApplyActionPolicyBlocked
        }
        if !d.orphanGraceElapsed(c.jobID) {
                return ApplyActionDeregisterGrace
        }
        return ApplyActionDeregisterQueued
}

// orphanGraceElapsed reports whether a job has been continuously orphaned
// (missing_from_hcl) for at least the configured grace period. It reads the
// first-seen time recorded by the previous cycle's commitResults; a job
// orphaned for the first time this cycle has no entry yet and is not eligible.
func (d *Differ) orphanGraceElapsed(jobID string) bool {
        d.mu.RLock()
        t, ok := d.driftFirstSeen[driftKey(jobID, string(DiffTypeMissingFromHCL))]
        d.mu.RUnlock()
        return ok && time.Since(t) >= d.deregisterGrace
}

// decideApplyAction determines a candidate's disposition and records the
// reason via metrics/logs. It does not enqueue anything; the caller enqueues
// when the action is ApplyActionQueued. The gates are ordered most-conservative
// first so the surfaced reason is the primary one.
func (d *Differ) decideApplyAction(c *updateCandidate, commit string, q *nomadapi.QueryOptions) ApplyAction {
        if c.class == DiffClassNone && !c.isCreation {
                // Everything in the diff is autoscaler-owned Count/Scaling churn;
                // Git has nothing to apply. The diff stays visible as an observation.
                return ApplyActionNoChange
        }

        if !c.isCreation && d.isPreExistingDrift(c, commit) && !d.applyExistingDrift {
                // The drift was already there when the change that brought it into scope
                // landed at the HEAD commit — the job's opt-in tag was added, or its
                // policy was widened to cover this diff. Conservative default: a scope
                // change does not retroactively mutate the job; only changes committed
                // after it apply. Enable with --apply-existing-drift.
                slog.Info("Pre-existing drift not applied on scope change (opt-in or policy widening); set --apply-existing-drift to apply", "job", c.jobID)
                d.updatesBlockedExistingDrift.WithLabelValues(c.jobID).Inc()
                return ApplyActionPreExisting
        }

        if c.class == DiffClassManagedMetaOnly && !c.isCreation && !d.applyMetaOnlyChanges {
                // A change confined to our own meta keys: leave the running job
                // alone. Re-registering just to push gitops_* keys is disruptive and
                // unnecessary — the HCL is already authoritative for them, and they
                // converge on the next real update (registering from HCL carries them
                // along, even under an image-only policy). Enable with
                // --apply-meta-only-changes.
                return ApplyActionMetaOnly
        }

        policy := d.effectivePolicy(c.job.Meta)
        switch policy {
        case UpdatePolicyNone:
                d.updatesBlockedByPolicy.WithLabelValues(c.jobID, string(policy)).Inc()
                c.actionDetail = d.policyBlockedDetail(c, policy)
                return ApplyActionPolicyBlocked
        case UpdatePolicyImageOnly:
                // Initial registration is full-only by design: registering a job
                // for the first time is not an image-only change.
                if c.isCreation || c.class != DiffClassImageOnly {
                        d.updatesBlockedByPolicy.WithLabelValues(c.jobID, string(policy)).Inc()
                        c.actionDetail = d.policyBlockedDetail(c, policy)
                        return ApplyActionPolicyBlocked
                }
        }

        if c.isCreation && !d.enableJobCreation {
                slog.Info("Job creation blocked: --enable-job-creation is off", "job", c.jobID)
                d.updatesBlockedCreationDisabled.WithLabelValues(c.jobID).Inc()
                return ApplyActionCreationBlocked
        }

        // Flap-loop guard: hold a re-apply of a spec that a recent deployment
        // already failed (apply→fail→revert→re-apply). Only meaningful for an
        // existing job; a first registration has no prior failed version to match.
        // Released automatically when Git moves to a spec that has not failed.
        if !c.isCreation {
                if mode := d.effectiveFlapGuard(c.job.Meta); mode != "off" && d.flapGuardBlocks(c, mode, q) {
                        slog.Info("Flap-guard: holding re-apply of a spec a recent deployment already failed; waiting for a fix in Git", "job", c.jobID)
                        d.updatesBlockedKnownFailed.WithLabelValues(c.jobID).Inc()
                        return ApplyActionKnownFailed
                }
        }

        return ApplyActionQueued
}

// enqueueUpdate places an approved candidate's update on the queue. A
// DEREGISTER candidate carries no parsed job (the job is removed by ID); a
// REGISTER candidate carries the HCL job to write.
func (d *Differ) enqueueUpdate(c *updateCandidate, commit string, raftIndex uint64) {
        op := c.operation
        if op == "" {
                op = JobUpdateOperationRegister
        }
        u := JobUpdate{
                UpdateID:       updateID(c.jobID, commit),
                JobID:          c.jobID,
                HCLFile:        c.hclFile,
                GitCommit:      commit,
                Operation:      op,
                Status:         JobUpdateStatusPending,
                NomadRaftIndex: raftIndex,
                DetectedAt:     time.Now().UTC().Format(time.RFC3339),
        }
        if op == JobUpdateOperationRegister {
                u.Policy = d.effectivePolicy(c.job.Meta)
                u.NomadJobModifyIndex = c.modifyIndex
                u.job = c.job
                u.preserveCounts = len(autoscaledGroups(c.job)) > 0
        } else {
                u.Policy = c.policy
        }
        superseded := d.updateQueue.Enqueue(u)
        if superseded > 0 {
                d.jobUpdatesTotal.WithLabelValues(string(op), string(JobUpdateStatusSuperseded)).Add(float64(superseded))
        }
        d.pendingUpdates.Set(float64(d.updateQueue.PendingCount()))
        slog.Info("Enqueued job update", "job", c.jobID, "update_id", u.UpdateID, "operation", op, "policy", u.Policy)
}

// ForceCheck runs a diff check unconditionally because the Nomad state has
// exceeded the configured maximum staleness. Increments the staleness counter
// and delegates to Check.
func (d *Differ) ForceCheck(hclFiles map[string]string, commit string) error {
        d.staleChecks.Inc()
        return d.Check(hclFiles, commit)
}

// driftKey returns a map key for a (jobID, diffType) pair.
func driftKey(jobID, diffType string) string {
        return jobID + "\x00" + diffType
}

// Ready reports whether at least one diff check has completed successfully.
// Before the first check finishes, callers cannot distinguish "no drift" from
// "haven't checked yet", so they should treat the Differ as unavailable.
func (d *Differ) Ready() bool {
        d.mu.RLock()
        defer d.mu.RUnlock()
        return !d.lastCheckTime.IsZero()
}

// Diffs returns a snapshot of the latest diffs, the time they were computed,
// and the git commit they were computed against.
func (d *Differ) Diffs() ([]JobDiff, time.Time, string) {
        d.mu.RLock()
        defer d.mu.RUnlock()
        result := make([]JobDiff, len(d.diffs))
        copy(result, d.diffs)
        return result, d.lastCheckTime, d.lastCommit
}

// hasContentDiff reports whether d contains spec changes beyond allocation
// bookkeeping entries. Used to suppress spurious diffs when planning HCL
// against a dead job where task groups appear with Type="None" (no spec
// change, only allocation count bookkeeping).
//
// Type="None" is defined in Nomad source (nomad/structs/diff.go, DiffTypeNone)
// and is returned in plan responses when a task group has no spec changes.
func hasContentDiff(d *nomadapi.JobDiff) bool {
        if d == nil || d.Type == "" || d.Type == "None" {
                return false
        }
        if len(d.Fields) > 0 || len(d.Objects) > 0 {
                return true
        }
        for _, tg := range d.TaskGroups {
                if tg.Type == "Added" || tg.Type == "Deleted" {
                        return true
                }
                if len(tg.Fields) > 0 || len(tg.Objects) > 0 {
                        return true
                }
                for _, task := range tg.Tasks {
                        if task.Type != "None" || len(task.Fields) > 0 || len(task.Objects) > 0 {
                                return true
                        }
                }
        }
        return false
}

func isNotFound(err error) bool {
        if err == nil {
                return false
        }
        s := err.Error()
        return strings.Contains(s, "404") || strings.Contains(strings.ToLower(s), "not found")
}

package nomad

import (
        "fmt"
        "log/slog"
        "path"
        "strings"
)

// Meta-key change tracking: the gitops_* keys are behavioural switches, so a
// job gaining, losing, or editing one changes what nomad-gitops does with
// it. Every transition is logged with the consequence — what the tool will
// do differently to honour it. Both sources are watched: the HCL side (a
// commit changed the keys) and the live side (someone re-registered the job
// manually, which is how the meta-drift problem manifests).
//
// Snapshots live in memory only: the first cycle after startup is a
// baseline and logs nothing.

// metaState holds a job's prefix-addressed meta keys from one source.
type metaState map[string]string

// prefixMetaKeys extracts the keys addressed to nomad-gitops (underscore
// and dotted forms) from meta.
func (d *Differ) prefixMetaKeys(meta map[string]string) metaState {
        out := metaState{}
        if d.managedMetaPrefix == "" {
                return out
        }
        underscored := d.managedMetaPrefix + "_"
        dotted := d.managedMetaPrefix + "."
        for k, v := range meta {
                if strings.HasPrefix(k, underscored) || strings.HasPrefix(k, dotted) {
                        out[k] = v
                }
        }
        return out
}

// metaStateKey identifies one (source, job) snapshot.
func metaStateKey(source, jobID string) string {
        return source + "\x00" + jobID
}

// recordMetaSeen stores a job's current prefix keys for change detection at
// the end of the check cycle.
func recordMetaSeen(seen map[string]metaState, d *Differ, source, jobID string, meta map[string]string) {
        if d.managedMetaPrefix == "" {
                return
        }
        seen[metaStateKey(source, jobID)] = d.prefixMetaKeys(meta)
}

// logMetaChanges compares this cycle's snapshots against the previous
// cycle's and logs every key transition. Jobs seen for the first time are
// baselined silently; jobs that disappeared are dropped silently (their
// absence is drift, which is reported elsewhere).
func (d *Differ) logMetaChanges(seen map[string]metaState) {
        if d.managedMetaPrefix == "" {
                return
        }
        d.metaMu.Lock()
        defer d.metaMu.Unlock()

        if d.prevMeta != nil {
                for k, cur := range seen {
                        prev, ok := d.prevMeta[k]
                        if !ok {
                                continue
                        }
                        parts := strings.SplitN(k, "\x00", 2)
                        d.diffMetaState(parts[0], parts[1], prev, cur)
                }
        }
        d.prevMeta = seen
}

// diffMetaState logs every added, removed, or changed prefix key for one
// (source, job) pair.
func (d *Differ) diffMetaState(source, jobID string, prev, cur metaState) {
        keys := make(map[string]struct{}, len(prev)+len(cur))
        for k := range prev {
                keys[k] = struct{}{}
        }
        for k := range cur {
                keys[k] = struct{}{}
        }
        for k := range keys {
                oldV, hadOld := prev[k]
                newV, hasNew := cur[k]
                if hadOld && hasNew && oldV == newV {
                        continue
                }
                change := "changed"
                switch {
                case !hadOld:
                        change = "added"
                case !hasNew:
                        change = "removed"
                }
                d.metaKeyChanges.WithLabelValues(jobID, source).Inc()
                slog.Info("Job meta key under the managed prefix changed",
                        "job", jobID, "source", source, "key", k, "change", change,
                        "old", oldV, "new", newV,
                        "action", d.metaChangeAction(jobID, source, k, oldV, newV, hasNew))
        }
}

// metaChangeAction describes what nomad-gitops will do to honour a key
// transition.
func (d *Differ) metaChangeAction(jobID, source, key, oldV, newV string, hasNew bool) string {
        switch key {
        case d.managedMetaPrefix + "_managed":
                return d.managedTransitionAction(jobID, source, newV, hasNew)
        case d.managedMetaPrefix + "_update_policy":
                return d.policyTransitionAction(source, newV, hasNew)
        case d.managedMetaPrefix + "_flap_guard":
                return d.flapGuardTransitionAction(source, newV, hasNew)
        case d.managedMetaPrefix + "_rollback":
                return d.rollbackTransitionAction(source, newV, hasNew)
        default:
                return "key is not one nomad-gitops understands; no behaviour change (see meta-key issue warnings)"
        }
}

// managedTransitionAction explains the consequence of an opt-in change.
func (d *Differ) managedTransitionAction(jobID, source, newV string, hasNew bool) string {
        if source == "nomad" {
                // Git is always the source of truth for our keys: a live-side change
                // only matters for jobs Git knows nothing about.
                return "noticed on the live job only; when the job has an HCL file in the repo, Git is the source of truth and the live value does not drive behaviour (for jobs without HCL, the live key controls missing_from_hcl detection)"
        }
        if hasNew && newV == "true" {
                base := "job is now opted in to GitOps management: it will be diffed against its HCL and applied per its effective update policy"
                if d.applyMetaOnlyChanges {
                        return base + "; the live job does not carry the key yet, and with --apply-meta-only-changes that difference will be applied on its own"
                }
                return base + "; the live job does not need to carry the key — that managed-meta-only difference is not applied or counted as drift on its own, and converges on the next real update"
        }
        // Removed, "false", or an invalid value: the opt-in check no longer passes.
        suffix := "nomad-gitops stops diffing it and will never register or deregister it"
        if hasNew && !validManagedValue(newV) {
                suffix = "the value is invalid, so the opt-in check fails; " + suffix
        }
        if d.jobSelectorGlob != "" {
                if matched, _ := path.Match(d.jobSelectorGlob, jobID); matched {
                        return fmt.Sprintf("opt-in no longer effective, but the job still matches --job-selector-glob %q and remains watched", d.jobSelectorGlob)
                }
        }
        return "job is no longer managed: " + suffix
}

// policyTransitionAction explains the consequence of an update-policy change.
func (d *Differ) policyTransitionAction(source, newV string, hasNew bool) string {
        effective := d.defaultPolicy
        qualifier := ""
        switch {
        case !hasNew:
                qualifier = fmt.Sprintf("key removed; falling back to the default policy %q: ", d.defaultPolicy)
        case ValidUpdatePolicy(newV):
                effective = UpdatePolicy(newV)
        default:
                effective = UpdatePolicyNone
                qualifier = fmt.Sprintf("value %q is invalid; treating as %q: ", newV, UpdatePolicyNone)
        }

        var behaviour string
        switch effective {
        case UpdatePolicyFull:
                behaviour = "any detected drift will now be applied automatically"
        case UpdatePolicyImageOnly:
                behaviour = "only drift confined to Docker image changes will be applied; anything else is surfaced as a diff"
        default:
                behaviour = "drift will be surfaced but no longer applied"
        }

        note := ""
        if source == "nomad" {
                note = " (note: policy is read from the HCL side; the live job's value does not drive behaviour)"
        }
        return qualifier + behaviour + note
}

// flapGuardTransitionAction explains the consequence of a flap-guard change.
func (d *Differ) flapGuardTransitionAction(source, newV string, hasNew bool) string {
        mode := d.flapGuard
        qualifier := ""
        switch {
        case !hasNew:
                qualifier = fmt.Sprintf("key removed; falling back to the --flap-guard default %q: ", d.flapGuard)
        case validFlapGuardValue(newV):
                mode = newV
        default:
                qualifier = fmt.Sprintf("value %q is invalid; falling back to the --flap-guard default %q: ", newV, d.flapGuard)
        }

        var behaviour string
        switch mode {
        case "off":
                behaviour = "the flap-loop guard is disabled for this job: a spec a recent Nomad version already failed to deploy may be re-applied"
        case "tag":
                behaviour = "a failed deployment's version will be tagged so the guard survives version GC; a re-applied spec matching a known-failed version is held"
        default:
                behaviour = "a re-applied spec matching a recent failed deployment version is held until Git moves on (version history is ephemeral, lost when Nomad GCs old versions)"
        }

        note := ""
        if source == "nomad" {
                note = " (note: flap-guard is read from the HCL side; the live job's value does not drive behaviour)"
        }
        return qualifier + behaviour + note
}

// rollbackTransitionAction explains the consequence of a rollback override change.
func (d *Differ) rollbackTransitionAction(source, newV string, hasNew bool) string {
        enabled := d.allowRollback
        qualifier := ""
        switch {
        case !hasNew:
                qualifier = fmt.Sprintf("key removed; falling back to the --allow-rollback default (%v): ", d.allowRollback)
        case validManagedValue(newV):
                enabled = newV == "true"
        default:
                qualifier = fmt.Sprintf("value %q is invalid; falling back to the --allow-rollback default (%v): ", newV, d.allowRollback)
        }

        var behaviour string
        if enabled {
                behaviour = "active rollback is enabled: a failed deployment on this job reverts to the last stable version (unless the update stanza sets auto_revert, in which case Nomad's own rollback wins)"
        } else {
                behaviour = "active rollback is disabled: a failed deployment is surfaced but not reverted by nomad-gitops (Nomad's auto_revert, if set, still applies)"
        }

        note := ""
        if source == "nomad" {
                note = " (note: rollback override is read from the HCL side; the live job's value does not drive behaviour)"
        }
        return qualifier + behaviour + note
}

package nomad

import (
        "log/slog"
        "strings"
)

// Meta-key validation: anything in a job's meta that starts with the managed
// prefix is addressed to nomad-gitops, so a key we don't recognise — or a
// recognised key with a value we can't act on — is almost certainly a typo
// that is silently changing behaviour (e.g. `gitops.managed` instead of
// `gitops_managed` drops the job out of scope without a trace).
//
// Each unique (job, key, value, issue) is logged once per process; the
// nomad_gitops_meta_key_issues_total counter keeps incrementing every
// cycle the issue persists, so dashboards can see it without log spam.

const (
        metaIssueUnknownKey   = "unknown_key"
        metaIssueInvalidValue = "invalid_value"
)

// validManagedValues are the accepted values for <prefix>_managed: "true"
// opts in, "false" is an explicit opt-out. Anything else ("True", "yes",
// "1") silently fails the opt-in check and is flagged.
func validManagedValue(v string) bool {
        return v == "true" || v == "false"
}

// validFlapGuardValue accepts the per-job <prefix>_flap_guard override values,
// the same set as the --flap-guard flag.
func validFlapGuardValue(v string) bool {
        return v == "history" || v == "tag" || v == "off"
}

// validateManagedMeta scans meta for keys addressed to nomad-gitops and
// records an issue for any it cannot act on. source says where the meta was
// seen ("hcl:<file>" or "nomad") for the log line.
func (d *Differ) validateManagedMeta(jobID, source string, meta map[string]string) {
        if d.managedMetaPrefix == "" || len(meta) == 0 {
                return
        }
        underscored := d.managedMetaPrefix + "_"
        dotted := d.managedMetaPrefix + "."
        for k, v := range meta {
                if !strings.HasPrefix(k, underscored) && !strings.HasPrefix(k, dotted) {
                        continue
                }
                switch k {
                case d.managedMetaPrefix + "_managed":
                        if !validManagedValue(v) {
                                d.recordMetaIssue(jobID, source, k, v, metaIssueInvalidValue,
                                        `accepted values are "true" and "false"`)
                        }
                case d.managedMetaPrefix + "_update_policy":
                        if !ValidUpdatePolicy(v) {
                                d.recordMetaIssue(jobID, source, k, v, metaIssueInvalidValue,
                                        `accepted values are "full", "image-only" and "none"; treated as "none"`)
                        }
                case d.managedMetaPrefix + "_flap_guard":
                        if !validFlapGuardValue(v) {
                                d.recordMetaIssue(jobID, source, k, v, metaIssueInvalidValue,
                                        `accepted values are "history", "tag" and "off"; falling back to the --flap-guard default`)
                        }
                case d.managedMetaPrefix + "_rollback":
                        if !validManagedValue(v) {
                                d.recordMetaIssue(jobID, source, k, v, metaIssueInvalidValue,
                                        `accepted values are "true" and "false"; falling back to the --allow-rollback default`)
                        }
                default:
                        d.recordMetaIssue(jobID, source, k, v, metaIssueUnknownKey,
                                "not a key nomad-gitops understands — possible typo")
                }
        }
}

// recordMetaIssue counts the issue and logs it the first time it is seen.
// Known keys with bad values log at ERROR — the author clearly intended to
// configure nomad-gitops and the value is being ignored or downgraded.
// Unknown keys under the prefix log at WARN.
func (d *Differ) recordMetaIssue(jobID, source, key, value, issue, hint string) {
        d.metaKeyIssues.WithLabelValues(jobID, issue).Inc()

        dedup := strings.Join([]string{jobID, key, value, issue}, "\x00")
        if _, seen := d.metaIssuesLogged.LoadOrStore(dedup, struct{}{}); seen {
                return
        }
        if issue == metaIssueInvalidValue {
                slog.Error("Job meta has a recognised nomad-gitops key with an invalid value",
                        "job", jobID, "source", source, "key", key, "value", value, "hint", hint)
        } else {
                slog.Warn("Job meta has an unrecognised key under the managed prefix",
                        "job", jobID, "source", source, "key", key, "value", value, "hint", hint)
        }
}

package nomad

import (
        "log/slog"
        "strings"

        nomadapi "github.com/hashicorp/nomad/api"
)

// RedactedValue replaces potentially sensitive values in plan diffs when
// secret redaction is enabled (--redact-secrets, on by default).
const RedactedValue = "[REDACTED]"

// redactedAnnotation is appended to each redacted field's annotations so the
// rendered diff states explicitly that the value was withheld.
const redactedAnnotation = "value redacted"

// secretKeywords are matched case-insensitively as substrings of field names.
// A field whose name contains any of these has its values redacted.
var secretKeywords = []string{
        "secret", "password", "passwd", "token", "credential",
        "api_key", "apikey", "private_key", "access_key",
}

// isSensitiveFieldName reports whether a plan-diff field's values should be
// redacted. All env vars are treated as potentially sensitive (Nomad renders
// them as fields named "Env[KEY]"), as are template bodies (EmbeddedTmpl) and
// any field whose name contains a secret-like keyword (e.g. Meta[db_password],
// driver Config[registry_token]).
func isSensitiveFieldName(name string) bool {
        n := strings.ToLower(name)
        if strings.HasPrefix(n, "env[") || n == "embeddedtmpl" {
                return true
        }
        for _, kw := range secretKeywords {
                if strings.Contains(n, kw) {
                        return true
                }
        }
        return false
}

// RedactJobDiff replaces potentially sensitive field values throughout d with
// RedactedValue, in place, and annotates each redacted field. The diff
// structure (field names, added/deleted/edited types, nesting) is preserved so
// the rendered output still reads like a plan diff. Returns the number of
// fields redacted.
func RedactJobDiff(d *nomadapi.JobDiff) int {
        if d == nil {
                return 0
        }
        n := redactFields(d.Fields)
        n += redactObjects(d.Objects, 1)
        for _, tg := range d.TaskGroups {
                if tg == nil {
                        continue
                }
                n += redactFields(tg.Fields)
                n += redactObjects(tg.Objects, 1)
                for _, t := range tg.Tasks {
                        if t == nil {
                                continue
                        }
                        n += redactFields(t.Fields)
                        n += redactObjects(t.Objects, 1)
                }
        }
        return n
}

// redactObjects walks a diff's Objects tree redacting sensitive field values.
// depth is the nesting level (1 at the top); beyond MaxPlanDiffObjectDepth,
// recursion stops rather than continuing without bound — see
// diffdepth.go. A diff pathological enough to hit this cap is not a shape any
// legitimate job spec produces.
func redactObjects(objs []*nomadapi.ObjectDiff, depth int) int {
        if depth > MaxPlanDiffObjectDepth {
                slog.Warn("Plan diff exceeds maximum nesting depth; stopped redacting beyond this point",
                        "depth", depth, "max_depth", MaxPlanDiffObjectDepth)
                return 0
        }
        n := 0
        for _, o := range objs {
                if o == nil {
                        continue
                }
                n += redactFields(o.Fields)
                n += redactObjects(o.Objects, depth+1)
        }
        return n
}

// redactFields redacts the non-empty values of sensitive fields. Empty sides
// are left empty so Added fields still render as additions and Deleted fields
// as deletions.
func redactFields(fields []*nomadapi.FieldDiff) int {
        n := 0
        for _, f := range fields {
                if f == nil || !isSensitiveFieldName(f.Name) {
                        continue
                }
                if f.Old == "" && f.New == "" {
                        continue
                }
                if f.Old != "" {
                        f.Old = RedactedValue
                }
                if f.New != "" {
                        f.New = RedactedValue
                }
                f.Annotations = append(f.Annotations, redactedAnnotation)
                n++
        }
        return n
}

package nomad

import (
        "crypto/sha256"
        "encoding/hex"
        "encoding/json"
        "log/slog"
        "strings"

        nomadapi "github.com/hashicorp/nomad/api"
)

// Rollback and the flap-loop guard. Both lean on Nomad's own state — version
// history and deployment outcomes — so nomad-gitops holds no durable record
// of "what failed". See docs/design/automatic-rollback.md.
//
// Scope: only deployment-producing jobs (service jobs with an update stanza and
// health checks) ever participate. A job that produces no deployment has no
// failed-deployment signal, so the flap-guard never matches and active rollback
// never fires — both fall through naturally without special-casing.

// volatileJobFields are server-injected or per-registration fields that must be
// stripped before fingerprinting a job spec, so the same intent registered at
// different times fingerprints identically.
var volatileJobFields = []string{
        "Version", "Stable", "SubmitTime", "ModifyIndex", "JobModifyIndex",
        "CreateIndex", "Status", "StatusDescription", "VersionTag", "Namespace",
}

// specFingerprint returns a stable hash of a job's spec, ignoring exactly what
// the diff classifier ignores (nomad-gitops's own managed-prefix meta keys
// and autoscaler-owned Count/Scaling) plus Nomad-injected version bookkeeping.
// Comparing an HCL-parsed job against a stored Nomad version is best-effort:
// server-side defaulting can make a legitimately-identical spec differ, in
// which case the guard misses and the bad spec is retried once more and caught
// again. That degradation is one-way and safe; a false block (two distinct
// specs colliding) needs a sha256 collision.
func specFingerprint(job *nomadapi.Job, metaPrefix string) (string, error) {
        raw, err := json.Marshal(job)
        if err != nil {
                return "", err
        }
        var m map[string]interface{}
        if err := json.Unmarshal(raw, &m); err != nil {
                return "", err
        }
        for _, k := range volatileJobFields {
                delete(m, k)
        }
        if meta, ok := m["Meta"].(map[string]interface{}); ok {
                stripPrefixedKeys(meta, metaPrefix)
                if len(meta) == 0 {
                        delete(m, "Meta")
                }
        }
        // Drop autoscaler-owned Count/Scaling from autoscaled groups so an
        // autoscaler nudge does not change the fingerprint.
        autoscaled := autoscaledGroups(job)
        if len(autoscaled) > 0 {
                if tgs, ok := m["TaskGroups"].([]interface{}); ok {
                        for _, raw := range tgs {
                                g, ok := raw.(map[string]interface{})
                                if !ok {
                                        continue
                                }
                                name, _ := g["Name"].(string)
                                if autoscaled[name] {
                                        delete(g, "Count")
                                        delete(g, "Scaling")
                                }
                        }
                }
        }
        norm, err := json.Marshal(m) // map keys are marshalled in sorted order
        if err != nil {
                return "", err
        }
        sum := sha256.Sum256(norm)
        return hex.EncodeToString(sum[:]), nil
}

// stripPrefixedKeys removes nomad-gitops's own managed meta keys (both the
// underscore and dotted forms) from a meta map.
func stripPrefixedKeys(meta map[string]interface{}, prefix string) {
        if prefix == "" {
                return
        }
        for k := range meta {
                if strings.HasPrefix(k, prefix+"_") || strings.HasPrefix(k, prefix+".") {
                        delete(meta, k)
                }
        }
}

// effectiveFlapGuard resolves the flap-guard mode for a job: the HCL meta key
// <prefix>_flap_guard wins (Git is intent), otherwise the --flap-guard default.
// An invalid meta value falls back to the default (already logged at ERROR by
// validateManagedMeta).
func (d *Differ) effectiveFlapGuard(meta map[string]string) string {
        if d.managedMetaPrefix != "" {
                if v, ok := meta[d.managedMetaPrefix+"_flap_guard"]; ok && validFlapGuardValue(v) {
                        return v
                }
        }
        return d.flapGuard
}

// effectiveRollback resolves whether active rollback is enabled for a job: the
// HCL meta key <prefix>_rollback wins, otherwise the --allow-rollback default.
func (d *Differ) effectiveRollback(meta map[string]string) bool {
        if d.managedMetaPrefix != "" {
                if v, ok := meta[d.managedMetaPrefix+"_rollback"]; ok && validManagedValue(v) {
                        return v == "true"
                }
        }
        return d.allowRollback
}

// jobHasAutoRevert reports whether a job's update stanza opts into Nomad's
// native auto_revert, at the job level or on any task group. When true,
// nomad-gitops stands down: Nomad's own rollback always wins.
func jobHasAutoRevert(job *nomadapi.Job) bool {
        if job == nil {
                return false
        }
        if job.Update != nil && job.Update.AutoRevert != nil && *job.Update.AutoRevert {
                return true
        }
        for _, tg := range job.TaskGroups {
                if tg == nil || tg.Update == nil {
                        continue
                }
                if tg.Update.AutoRevert != nil && *tg.Update.AutoRevert {
                        return true
                }
        }
        return false
}

// failedTagPrefix is the version-tag name prefix used by flap-guard tag mode.
func (d *Differ) failedTagPrefix() string {
        return d.managedMetaPrefix + "-failed-"
}

// failedTagName builds the durable version-tag name for a failed spec.
func (d *Differ) failedTagName(fingerprint string) string {
        return d.failedTagPrefix() + fingerprint
}

// parseFailedFingerprint recovers a spec fingerprint from a failed-version tag
// name, or ("", false) if the tag is not one of ours.
func (d *Differ) parseFailedFingerprint(tagName string) (string, bool) {
        p := d.failedTagPrefix()
        if d.managedMetaPrefix == "" || !strings.HasPrefix(tagName, p) {
                return "", false
        }
        return strings.TrimPrefix(tagName, p), true
}

// flapGuardBlocks reports whether re-applying the candidate's HCL spec would
// re-enter a deployment that already failed. mode is the effective guard mode
// (history or tag; "off" is handled by the caller). On any Nomad API error it
// fails open (returns false): a missed block costs at most one more failed
// attempt, whereas a spurious block could freeze a legitimate apply.
func (d *Differ) flapGuardBlocks(c *updateCandidate, mode string, q *nomadapi.QueryOptions) bool {
        want, err := specFingerprint(c.job, d.managedMetaPrefix)
        if err != nil {
                slog.Warn("Flap-guard: could not fingerprint HCL job; not blocking", "job", c.jobID, "err", err)
                return false
        }
        failed, err := d.failedVersionFingerprints(c.jobID, mode, q)
        if err != nil {
                slog.Warn("Flap-guard: could not read Nomad version history; not blocking", "job", c.jobID, "err", err)
                return false
        }
        _, blocked := failed[want]
        return blocked
}

// failedVersionFingerprints returns the set of spec fingerprints for job
// versions whose deployment failed. In history mode it derives them from the
// current failed deployments and the retained version specs (ephemeral, lost
// when Nomad GCs the version). In tag mode it additionally reads fingerprints
// recovered from durable version tags, and tags any newly-observed failed
// version so the block survives GC.
func (d *Differ) failedVersionFingerprints(jobID, mode string, q *nomadapi.QueryOptions) (map[string]struct{}, error) {
        deps, _, err := d.jobs.Deployments(jobID, false, q)
        if err != nil {
                d.nomadAPIErrors.WithLabelValues("deployments").Inc()
                return nil, err
        }
        failedVersions := make(map[uint64]struct{})
        for _, dep := range deps {
                if dep != nil && dep.Status == nomadapi.DeploymentStatusFailed {
                        failedVersions[dep.JobVersion] = struct{}{}
                }
        }
        // History mode with nothing currently failed: no Versions call needed.
        if len(failedVersions) == 0 && mode != "tag" {
                return nil, nil
        }

        versions, _, _, err := d.jobs.Versions(jobID, false, q)
        if err != nil {
                d.nomadAPIErrors.WithLabelValues("versions").Inc()
                return nil, err
        }
        byVersion := make(map[uint64]*nomadapi.Job, len(versions))
        for _, v := range versions {
                if v != nil && v.Version != nil {
                        byVersion[*v.Version] = v
                }
        }

        out := make(map[string]struct{})
        // Durable tags survive version GC; recover their fingerprints first.
        if mode == "tag" {
                for _, v := range versions {
                        if v == nil || v.VersionTag == nil {
                                continue
                        }
                        if fp, ok := d.parseFailedFingerprint(v.VersionTag.Name); ok {
                                out[fp] = struct{}{}
                        }
                }
        }
        for ver := range failedVersions {
                v := byVersion[ver]
                if v == nil {
                        continue
                }
                fp, err := specFingerprint(v, d.managedMetaPrefix)
                if err != nil {
                        continue
                }
                out[fp] = struct{}{}
                if mode == "tag" {
                        d.tagFailedVersion(jobID, ver, v, fp, q)
                }
        }
        return out, nil
}

// tagFailedVersion durably tags a failed version so the flap-guard survives
// version GC. A version carries at most one tag, so a version already tagged
// (by us on a prior cycle, or by anything else) is left alone.
func (d *Differ) tagFailedVersion(jobID string, version uint64, v *nomadapi.Job, fingerprint string, q *nomadapi.QueryOptions) {
        if v.VersionTag != nil {
                return
        }
        wq := &nomadapi.WriteOptions{Namespace: d.namespace}
        if _, err := d.jobs.TagVersion(jobID, version, d.failedTagName(fingerprint),
                "nomad-gitops: deployment failed; held by the flap-loop guard", wq); err != nil {
                d.nomadAPIErrors.WithLabelValues("tag").Inc()
                slog.Warn("Flap-guard: could not tag failed version", "job", jobID, "version", version, "err", err)
                return
        }
        d.failedVersionsTagged.WithLabelValues(jobID).Inc()
        slog.Info("Flap-guard: tagged failed version so the block survives GC", "job", jobID, "version", version)
}

// lastStableVersion returns the highest job version strictly below failed that
// is marked Stable — the version to roll back to.
func lastStableVersion(versions []*nomadapi.Job, failed uint64) (uint64, bool) {
        var best uint64
        found := false
        for _, v := range versions {
                if v == nil || v.Version == nil || v.Stable == nil || !*v.Stable {
                        continue
                }
                if *v.Version >= failed {
                        continue
                }
                if !found || *v.Version > best {
                        best = *v.Version
                        found = true
                }
        }
        return best, found
}

// checkRollbacks runs the active-rollback poll over the managed jobs. For each
// job that has rollback enabled and whose latest deployment has failed, it
// enqueues a REVERT to the last stable version — unless the job's update stanza
// sets auto_revert, in which case Nomad's own rollback wins and nomad-gitops
// stands down (logged once). Jobs without a deployment are skipped naturally.
// metaByJob carries each managed job's meta (HCL where present, else live).
func (d *Differ) checkRollbacks(metaByJob map[string]map[string]string, q *nomadapi.QueryOptions, raftIndex uint64) {
        for jobID, meta := range metaByJob {
                if !d.effectiveRollback(meta) {
                        continue
                }
                dep, _, err := d.jobs.LatestDeployment(jobID, q)
                if err != nil {
                        d.nomadAPIErrors.WithLabelValues("deployment").Inc()
                        slog.Warn("Rollback: could not read latest deployment", "job", jobID, "err", err)
                        continue
                }
                if dep == nil || dep.Status != nomadapi.DeploymentStatusFailed {
                        continue
                }

                // The live job decides auto_revert (what Nomad will actually do) and
                // gives us the current version for the CAS guard.
                liveJob, _, err := d.jobs.Info(jobID, q)
                if err != nil {
                        d.nomadAPIErrors.WithLabelValues("info").Inc()
                        slog.Warn("Rollback: could not read live job", "job", jobID, "err", err)
                        continue
                }
                if jobHasAutoRevert(liveJob) {
                        // auto_revert always wins. Log the clash once per job so an operator
                        // who set both knows nomad-gitops is deliberately standing down.
                        if _, seen := d.rollbackLogged.LoadOrStore(jobID, struct{}{}); !seen {
                                slog.Warn("Rollback: job has a failed deployment but its update stanza sets auto_revert; standing down and letting Nomad revert",
                                        "job", jobID)
                        }
                        d.rollbacks.WithLabelValues(jobID, "deferred_auto_revert").Inc()
                        continue
                }

                versions, _, _, err := d.jobs.Versions(jobID, false, q)
                if err != nil {
                        d.nomadAPIErrors.WithLabelValues("versions").Inc()
                        slog.Warn("Rollback: could not read version history", "job", jobID, "err", err)
                        continue
                }
                failedVersion := dep.JobVersion
                target, ok := lastStableVersion(versions, failedVersion)
                if !ok {
                        slog.Warn("Rollback: deployment failed but no earlier stable version to revert to; leaving as is",
                                "job", jobID, "failed_version", failedVersion)
                        d.rollbacks.WithLabelValues(jobID, "no_stable_version").Inc()
                        continue
                }

                u := JobUpdate{
                        UpdateID:          revertUpdateID(jobID, failedVersion),
                        JobID:             jobID,
                        Operation:         JobUpdateOperationRevert,
                        Status:            JobUpdateStatusPending,
                        NomadRaftIndex:    raftIndex,
                        DetectedAt:        nowRFC3339(),
                        RevertToVersion:   target,
                        RevertFromVersion: failedVersion,
                }
                superseded := d.updateQueue.Enqueue(u)
                if superseded > 0 {
                        d.jobUpdatesTotal.WithLabelValues(string(JobUpdateOperationRevert), string(JobUpdateStatusSuperseded)).Add(float64(superseded))
                }
                d.pendingUpdates.Set(float64(d.updateQueue.PendingCount()))
                d.rollbacks.WithLabelValues(jobID, "queued").Inc()
                slog.Info("Rollback: enqueued revert of failed deployment to last stable version",
                        "job", jobID, "failed_version", failedVersion, "target_version", target, "update_id", u.UpdateID)
                d.notifyApplier()
        }
}

package nomad

import (
        "context"
        "encoding/base64"
        "encoding/json"
        "errors"
        "fmt"
        "io/fs"
        "log/slog"
        "os"
        "path/filepath"
        "strings"
        "time"

        "github.com/gerrowadat/nomad-gitops/internal/config"
)

// Authentication to the Nomad API. Modes, in precedence order:
//
//  1. Workload-identity login (--nomad-login-auth-method) — the working way to
//     use Nomad workload identity. The identity JWT is exchanged for a real ACL
//     token (SecretID) via POST /v1/acl/login, and re-exchanged before it
//     expires. A raw WI JWT authenticates read RPCs but is *rejected* by
//     Nomad's Job.Plan RPC ("UUID must be 36 characters"), which nomad-gitops
//     needs for every drift check — so the JWT cannot be used directly as a
//     token (issue #74). Login exchange is the fix.
//  2. A token file (--nomad-token-file) — a real ACL SecretID in a file, re-read
//     periodically. For a sidecar-written token, or a rotating static token.
//  3. A static token (--nomad-token / NOMAD_TOKEN) — manual running and testing.
//  4. None — anonymous, which works only when the cluster has ACLs disabled.

// wiTokenFilename is the file Nomad writes the default workload-identity token
// (a JWT) to under the task secrets dir when `identity { file = true }` is set.
const wiTokenFilename = "nomad_token"

const (
        // loginSafetyMargin is how far before expiry a re-login must complete, so the
        // token is always refreshed before it expires even for a short TTL.
        loginSafetyMargin = 5 * time.Second
        // loginRetryBackoff is the delay before retrying after a failed login.
        loginRetryBackoff = 15 * time.Second
        // defaultLoginRefresh is used when an exchanged token carries no expiry
        // (unusual — WI login tokens are TTL-bounded).
        defaultLoginRefresh = 5 * time.Minute
)

// resolveNomadToken decides the initial token to authenticate with (for the
// file and static modes) and, when the token is sourced from a file, the path
// to keep re-reading. It does not handle login mode — the caller checks
// NomadLoginAuthMethod first. watchPath is empty for the static and anonymous
// cases. An explicitly-configured token file that cannot be read is a fatal
// misconfiguration and returns an error.
func resolveNomadToken(cfg *config.Config) (token, watchPath string, err error) {
        switch {
        case cfg.NomadTokenFile != "":
                watchPath = cfg.NomadTokenFile
                if cfg.NomadToken != "" {
                        slog.Warn("Both a static Nomad token and a token file are configured; using the token file (it refreshes) and ignoring the static token",
                                "token_file", watchPath)
                }
        case cfg.NomadToken != "":
                return cfg.NomadToken, "", nil
        default:
                return "", "", nil // anonymous
        }

        token, err = readTokenFile(watchPath)
        if err != nil {
                return "", "", err
        }
        return token, watchPath, nil
}

// loginJWTPath resolves the workload-identity JWT file to exchange in login
// mode: the explicit --nomad-login-jwt-file, else ${NOMAD_SECRETS_DIR}/nomad_token
// (the default identity), else "".
func loginJWTPath(cfg *config.Config) string {
        if cfg.NomadLoginJWTFile != "" {
                return cfg.NomadLoginJWTFile
        }
        if dir := os.Getenv("NOMAD_SECRETS_DIR"); dir != "" {
                return filepath.Join(dir, wiTokenFilename)
        }
        return ""
}

// defaultWorkloadTokenPath returns ${NOMAD_SECRETS_DIR}/nomad_token when that
// file exists, else "". Used only to detect a workload-identity deployment that
// has not configured login, so a clear hint can be logged.
func defaultWorkloadTokenPath() string {
        dir := os.Getenv("NOMAD_SECRETS_DIR")
        if dir == "" {
                return ""
        }
        p := filepath.Join(dir, wiTokenFilename)
        if _, err := os.Stat(p); errors.Is(err, fs.ErrNotExist) {
                return ""
        }
        return p
}

// looksLikeJWT reports whether s is (probably) a JWT rather than an ACL
// SecretID, so a misconfiguration (feeding a raw WI JWT as a token) can be
// flagged. A SecretID is a 36-char UUID; a JWT is a longer dotted base64url
// string, conventionally starting "ey".
func looksLikeJWT(s string) bool {
        return strings.HasPrefix(s, "ey") && strings.Count(s, ".") == 2
}

// jwtLacksExpiry reports whether token is a decodable JWT that carries no `exp`
// claim. A workload-identity JWT with no expiry means the task's `identity`
// block has no `ttl`, so Nomad issues a non-expiring token and never rewrites
// the file — login works at first but fails once the exchanged ACL token
// expires (issue #76). Returns false for anything it cannot positively decode
// as a JWT, so a real SecretID or a malformed value never trips the warning.
func jwtLacksExpiry(token string) bool {
        parts := strings.Split(token, ".")
        if len(parts) != 3 {
                return false
        }
        payload, err := base64.RawURLEncoding.DecodeString(parts[1])
        if err != nil {
                return false
        }
        var claims map[string]json.RawMessage
        if err := json.Unmarshal(payload, &claims); err != nil {
                return false
        }
        _, hasExp := claims["exp"]
        return !hasExp
}

// readTokenFile reads and trims a token (SecretID or JWT) from a file.
func readTokenFile(path string) (string, error) {
        b, err := os.ReadFile(path)
        if err != nil {
                return "", fmt.Errorf("reading Nomad token file %q: %w", path, err)
        }
        return strings.TrimSpace(string(b)), nil
}

// refreshTokenFile re-reads watchPath every interval and calls setToken whenever
// the (non-empty) token changes from current. Read errors are reported via onErr
// and the previous token is kept. It blocks until ctx is cancelled. setToken and
// onErr are injected so the loop is testable without a live Nomad client.
func refreshTokenFile(ctx context.Context, watchPath string, interval time.Duration, current string, setToken func(string), onErr func(error)) {
        ticker := time.NewTicker(interval)
        defer ticker.Stop()
        for {
                select {
                case <-ctx.Done():
                        return
                case <-ticker.C:
                        tok, err := readTokenFile(watchPath)
                        if err != nil {
                                onErr(err)
                                continue
                        }
                        // An empty file is treated as a transient write-in-progress, not an
                        // intentional clearing of the token.
                        if tok != "" && tok != current {
                                current = tok
                                setToken(tok)
                        }
                }
        }
}

// nextLoginDelay returns how long until the next re-login: half the remaining
// lifetime (so a fresh token is obtained well before this one expires), but
// never later than loginSafetyMargin before expiry — even for a short TTL the
// refresh completes before the token is invalid. A non-positive result means
// re-login now (the token is at or past expiry). Falls back to
// defaultLoginRefresh when the token has no expiry.
func nextLoginDelay(expiry *time.Time) time.Duration {
        if expiry == nil {
                return defaultLoginRefresh
        }
        remaining := time.Until(*expiry)
        d := remaining / 2
        if latest := remaining - loginSafetyMargin; d > latest {
                d = latest
        }
        if d < 0 {
                d = 0
        }
        return d
}

// runLoginRefresher re-exchanges the workload-identity JWT for a fresh ACL token
// before the current one expires. firstDelay is when to attempt the next login
// (computed by the caller from the startup login's expiry, or a short backoff if
// startup login failed). login performs the exchange, returning the new SecretID
// and its expiry. apply installs the token; onErr reports a failed exchange. It
// blocks until ctx is cancelled. login/apply/onErr are injected so the loop is
// testable without a live Nomad client.
func runLoginRefresher(ctx context.Context, firstDelay time.Duration, login func() (secretID string, expiry *time.Time, err error), apply func(string), onErr func(error)) {
        timer := time.NewTimer(firstDelay)
        defer timer.Stop()
        for {
                select {
                case <-ctx.Done():
                        return
                case <-timer.C:
                        secretID, expiry, err := login()
                        if err != nil {
                                onErr(err)
                                timer.Reset(loginRetryBackoff)
                                continue
                        }
                        apply(secretID)
                        timer.Reset(nextLoginDelay(expiry))
                }
        }
}

package nomad

import (
        "fmt"
        "sync"
        "time"

        nomadapi "github.com/hashicorp/nomad/api"
)

// UpdatePolicy controls how much detected drift may be applied to a job
// automatically. It is declared per job in HCL meta
// (<prefix>_update_policy) and falls back to --default-update-policy.
type UpdatePolicy string

const (
        // UpdatePolicyFull applies any detected drift.
        UpdatePolicyFull UpdatePolicy = "full"
        // UpdatePolicyImageOnly applies drift only when the entire plan diff is
        // confined to Docker image references.
        UpdatePolicyImageOnly UpdatePolicy = "image-only"
        // UpdatePolicyNone detects and surfaces drift but never applies it.
        UpdatePolicyNone UpdatePolicy = "none"
)

// ValidUpdatePolicy reports whether s is a recognised policy value.
func ValidUpdatePolicy(s string) bool {
        switch UpdatePolicy(s) {
        case UpdatePolicyFull, UpdatePolicyImageOnly, UpdatePolicyNone:
                return true
        }
        return false
}

// JobUpdateOperation is the kind of change a JobUpdate applies.
type JobUpdateOperation string

const (
        JobUpdateOperationRegister   JobUpdateOperation = "REGISTER"
        JobUpdateOperationDeregister JobUpdateOperation = "DEREGISTER"
        // JobUpdateOperationRevert rolls a job back to a prior stable version after
        // a failed deployment (active rollback, for jobs without auto_revert).
        JobUpdateOperationRevert JobUpdateOperation = "REVERT"
)

// JobUpdateStatus is the lifecycle state of a JobUpdate.
type JobUpdateStatus string

const (
        JobUpdateStatusPending    JobUpdateStatus = "PENDING"
        JobUpdateStatusInProgress JobUpdateStatus = "IN_PROGRESS"
        JobUpdateStatusSucceeded  JobUpdateStatus = "SUCCEEDED"
        JobUpdateStatusFailed     JobUpdateStatus = "FAILED"
        JobUpdateStatusSuperseded JobUpdateStatus = "SUPERSEDED"
)

// terminal reports whether a status is final.
func (s JobUpdateStatus) terminal() bool {
        switch s {
        case JobUpdateStatusSucceeded, JobUpdateStatusFailed, JobUpdateStatusSuperseded:
                return true
        }
        return false
}

// JobUpdate represents a single intended change to a Nomad job, derived from
// a detected diff between Git and the cluster. A JobDiff is an observation;
// a JobUpdate is an intended transition.
type JobUpdate struct {
        // UpdateID is <job_id>/<git_commit_short> — deliberately derived from
        // stable inputs so the same intent re-detected after a restart or a
        // failure is recognisably the same update.
        UpdateID string `json:"update_id"`

        JobID string `json:"job_id"`

        // HCLFile is the repo path that is the source of truth for this job.
        HCLFile string `json:"hcl_file,omitempty"`

        // GitCommit is the commit hash that triggered this update.
        GitCommit string `json:"git_commit"`

        Operation JobUpdateOperation `json:"operation"`
        Status    JobUpdateStatus    `json:"status"`

        // Policy is the effective update policy that allowed this update.
        Policy UpdatePolicy `json:"policy"`

        // NomadJobModifyIndex is the job's ModifyIndex at detection time, used
        // as the CAS token on Jobs.Register (EnforceIndex). Zero means the job
        // did not exist in Nomad at detection time.
        NomadJobModifyIndex uint64 `json:"nomad_job_modify_index"`

        // NomadRaftIndex is the cluster Raft index at detection time, recorded
        // for auditability.
        NomadRaftIndex uint64 `json:"nomad_raft_index"`

        DetectedAt string `json:"detected_at"`          // RFC3339
        AppliedAt  string `json:"applied_at,omitempty"` // RFC3339; empty until applied
        Error      string `json:"error,omitempty"`

        // Revert-only. RevertToVersion is the stable job version to roll back to.
        // RevertFromVersion is the failed version the job must still be at for the
        // revert to land (the enforcePriorVersion CAS guard).
        RevertToVersion   uint64 `json:"revert_to_version,omitempty"`
        RevertFromVersion uint64 `json:"revert_from_version,omitempty"`

        // job is the parsed HCL job to register. In-memory only; the queue is
        // rebuilt from a diff cycle after restart so this never needs to be
        // serialised.
        job *nomadapi.Job
        // preserveCounts is set when the job has autoscaled task groups, so the
        // register call does not overwrite autoscaler-owned counts.
        preserveCounts bool
}

// updateID builds the stable identifier for a job/commit pair.
func updateID(jobID, commit string) string {
        short := commit
        if len(short) > 7 {
                short = short[:7]
        }
        return fmt.Sprintf("%s/%s", jobID, short)
}

// nowRFC3339 is the timestamp format used on JobUpdate records.
func nowRFC3339() string {
        return time.Now().UTC().Format(time.RFC3339)
}

// revertUpdateID builds a stable identifier for a revert. It is keyed on the
// failed version (not a git commit) so the same recovery re-detected each cycle
// is recognisably the same update and dedups in the queue.
func revertUpdateID(jobID string, failedVersion uint64) string {
        return fmt.Sprintf("%s/revert-%d", jobID, failedVersion)
}

// maxTerminalUpdates caps how many terminal (SUCCEEDED/FAILED/SUPERSEDED)
// records the queue retains for API visibility. Oldest are pruned first.
const maxTerminalUpdates = 200

// UpdateQueue is the in-memory queue between detection and application.
// Restart loses it by design: the next diff cycle recreates any update whose
// drift still exists, and CAS plus re-planning make a re-apply harmless. See
// docs/design/gitops-job-updates.md ("Restart safety and recovery").
type UpdateQueue struct {
        mu      sync.Mutex
        updates []*JobUpdate // newest last
}

// NewUpdateQueue returns an empty queue.
func NewUpdateQueue() *UpdateQueue {
        return &UpdateQueue{}
}

// Enqueue records an intended update. Rules, keyed on any existing entry with
// the same UpdateID (same job, same commit):
//   - PENDING: refreshed in place (CAS token, job pointer) rather than
//     duplicated — it has not started, so mutating it is safe.
//   - IN_PROGRESS: left strictly untouched and no new entry is added. The
//     applier reads the update's fields (CAS token, job pointer,
//     preserveCounts) without holding the queue lock, so mutating an
//     in-flight update would race it and could make the apply use a
//     different token or job than it started with. If that apply fails, the
//     next diff cycle re-enqueues against the by-then terminal record.
//   - terminal: dropped and replaced with a fresh PENDING entry — the same
//     intent is being retried after a failure or a cluster-side change.
//
// A PENDING update for the same job with a *different* UpdateID (a newer
// commit arrived before the old one applied) is marked SUPERSEDED; the most
// recent intended state wins. An IN_PROGRESS update for a different UpdateID
// is also left alone for the same race reason.
//
// Returns the number of updates marked SUPERSEDED by this enqueue.
func (q *UpdateQueue) Enqueue(u JobUpdate) (superseded int) {
        q.mu.Lock()
        defer q.mu.Unlock()

        for _, existing := range q.updates {
                if existing.UpdateID != u.UpdateID {
                        continue
                }
                switch existing.Status {
                case JobUpdateStatusPending:
                        existing.NomadJobModifyIndex = u.NomadJobModifyIndex
                        existing.NomadRaftIndex = u.NomadRaftIndex
                        existing.job = u.job
                        existing.preserveCounts = u.preserveCounts
                        return 0
                case JobUpdateStatusInProgress:
                        // Being applied right now: do not touch it, do not duplicate it.
                        return 0
                }
                // Terminal: handled by the supersede + drop logic below.
                break
        }

        for _, existing := range q.updates {
                if existing.JobID == u.JobID && existing.Status == JobUpdateStatusPending {
                        existing.Status = JobUpdateStatusSuperseded
                        superseded++
                }
        }

        // Retry of the same intent: drop the old terminal record so the queue
        // holds one row per UpdateID. Only terminal records are removed; an
        // IN_PROGRESS record with this UpdateID already returned above.
        for i, existing := range q.updates {
                if existing.UpdateID == u.UpdateID && existing.Status.terminal() {
                        q.updates = append(q.updates[:i], q.updates[i+1:]...)
                        break
                }
        }

        u.Status = JobUpdateStatusPending
        q.updates = append(q.updates, &u)
        q.prune()
        return superseded
}

// NextPending returns the oldest PENDING update, marking it IN_PROGRESS, or
// nil when nothing is waiting.
func (q *UpdateQueue) NextPending() *JobUpdate {
        q.mu.Lock()
        defer q.mu.Unlock()
        for _, u := range q.updates {
                if u.Status == JobUpdateStatusPending {
                        u.Status = JobUpdateStatusInProgress
                        return u
                }
        }
        return nil
}

// Complete records the outcome of an apply attempt.
func (q *UpdateQueue) Complete(updateID string, status JobUpdateStatus, appliedIndex uint64, errMsg string) {
        q.mu.Lock()
        defer q.mu.Unlock()
        for _, u := range q.updates {
                if u.UpdateID == updateID && u.Status == JobUpdateStatusInProgress {
                        u.Status = status
                        u.Error = errMsg
                        if status == JobUpdateStatusSucceeded {
                                u.AppliedAt = time.Now().UTC().Format(time.RFC3339)
                                if appliedIndex != 0 {
                                        u.NomadJobModifyIndex = appliedIndex
                                }
                        }
                        q.prune()
                        return
                }
        }
}

// Snapshot returns a copy of all queue entries, newest last. The internal
// job pointer is not exposed.
func (q *UpdateQueue) Snapshot() []JobUpdate {
        q.mu.Lock()
        defer q.mu.Unlock()
        out := make([]JobUpdate, 0, len(q.updates))
        for _, u := range q.updates {
                c := *u
                c.job = nil
                out = append(out, c)
        }
        return out
}

// PendingCount returns the number of PENDING updates.
func (q *UpdateQueue) PendingCount() int {
        q.mu.Lock()
        defer q.mu.Unlock()
        n := 0
        for _, u := range q.updates {
                if u.Status == JobUpdateStatusPending {
                        n++
                }
        }
        return n
}

// prune drops the oldest terminal records beyond maxTerminalUpdates.
// Caller must hold q.mu.
func (q *UpdateQueue) prune() {
        terminal := 0
        for _, u := range q.updates {
                if u.Status.terminal() {
                        terminal++
                }
        }
        if terminal <= maxTerminalUpdates {
                return
        }
        keep := q.updates[:0]
        for _, u := range q.updates {
                if terminal > maxTerminalUpdates && u.Status.terminal() {
                        terminal--
                        continue
                }
                keep = append(keep, u)
        }
        q.updates = keep
}

package server

import (
        "crypto/subtle"
        "encoding/json"
        "net/http"
        "time"

        "github.com/gerrowadat/nomad-gitops/internal/nomad"
)

// BuildInfo holds version metadata injected at link time.
type BuildInfo struct {
        Version   string
        Commit    string
        BuildDate string
}

// API response types. These are the canonical JSON shapes for all /api/v1/ endpoints.

type diffsResponse struct {
        Diffs         []nomad.JobDiff `json:"diffs"`
        LastCheckTime string          `json:"last_check_time,omitempty"`
        LastCommit    string          `json:"last_commit,omitempty"`
}

type selectedJobsResponse struct {
        Jobs          []nomad.SelectedJob `json:"jobs"`
        LastCheckTime string              `json:"last_check_time,omitempty"`
        LastCommit    string              `json:"last_commit,omitempty"`
}

type updatesResponse struct {
        Updates []nomad.JobUpdate `json:"updates"`
}

type statusResponse struct {
        LastCommit  string `json:"last_commit,omitempty"`
        LastUpdated string `json:"last_updated,omitempty"`
}

type versionResponse struct {
        Version   string `json:"version"`
        Commit    string `json:"commit"`
        BuildDate string `json:"build_date"`
}

type refreshResponse struct {
        Message string `json:"message"`
}

type errorResponse struct {
        Error string `json:"error"`
}

// requireAPIKey returns a middleware that enforces Bearer token authentication.
// If apiKey is empty every request is rejected with a clear 401.
// The expected "Bearer <key>" value is compared against the Authorization header
// in constant time. subtle.ConstantTimeCompare only runs in constant time for
// equal-length inputs, so a length check gates it; that leaks the expected
// length but never the key contents. expected and its length are computed once
// per middleware instance to avoid allocating on every request.
func requireAPIKey(apiKey string) func(http.Handler) http.Handler {
        expected := []byte("Bearer " + apiKey)
        expectedLen := len(expected)
        return func(next http.Handler) http.Handler {
                return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
                        got := r.Header.Get("Authorization")
                        if apiKey == "" || len(got) != expectedLen || subtle.ConstantTimeCompare([]byte(got), expected) != 1 {
                                writeJSON(w, http.StatusUnauthorized, errorResponse{Error: "unauthorized"})
                                return
                        }
                        next.ServeHTTP(w, r)
                })
        }
}

// writeJSON encodes v as JSON and writes it with the given status code.
func writeJSON(w http.ResponseWriter, code int, v any) {
        w.Header().Set("Content-Type", "application/json")
        w.WriteHeader(code)
        json.NewEncoder(w).Encode(v)
}

// fmtTime formats t as UTC RFC3339, returning "" for the zero value.
func fmtTime(t time.Time) string {
        if t.IsZero() {
                return ""
        }
        return t.UTC().Format(time.RFC3339)
}

// apiNotReady writes a 503 JSON response for endpoints that need a completed check.
func apiNotReady(w http.ResponseWriter) {
        writeJSON(w, http.StatusServiceUnavailable, errorResponse{Error: "server is not ready: initial state not yet built"})
}

// ── API handlers ──────────────────────────────────────────────────────────────

func (s *Server) handleAPIDiffs(w http.ResponseWriter, r *http.Request) {
        if !s.git.Ready() || !s.diffs.Ready() {
                apiNotReady(w)
                return
        }
        diffs, lastCheck, lastCommit := s.diffs.Diffs()
        writeJSON(w, http.StatusOK, diffsResponse{
                Diffs:         diffs,
                LastCheckTime: fmtTime(lastCheck),
                LastCommit:    lastCommit,
        })
}

func (s *Server) handleAPISelectedJobs(w http.ResponseWriter, r *http.Request) {
        if !s.git.Ready() || !s.diffs.Ready() {
                apiNotReady(w)
                return
        }
        jobs, lastCheck, lastCommit := s.diffs.SelectedJobs()
        writeJSON(w, http.StatusOK, selectedJobsResponse{
                Jobs:          jobs,
                LastCheckTime: fmtTime(lastCheck),
                LastCommit:    lastCommit,
        })
}

func (s *Server) handleAPIUpdates(w http.ResponseWriter, r *http.Request) {
        updates := s.diffs.Updates()
        if updates == nil {
                updates = []nomad.JobUpdate{}
        }
        writeJSON(w, http.StatusOK, updatesResponse{Updates: updates})
}

func (s *Server) handleAPIStatus(w http.ResponseWriter, r *http.Request) {
        if !s.git.Ready() {
                apiNotReady(w)
                return
        }
        lastCommit, lastUpdated := s.git.Status()
        writeJSON(w, http.StatusOK, statusResponse{
                LastCommit:  lastCommit,
                LastUpdated: fmtTime(lastUpdated),
        })
}

func (s *Server) handleAPIVersion(w http.ResponseWriter, r *http.Request) {
        writeJSON(w, http.StatusOK, versionResponse{
                Version:   s.buildInfo.Version,
                Commit:    s.buildInfo.Commit,
                BuildDate: s.buildInfo.BuildDate,
        })
}

func (s *Server) handleAPIRefresh(w http.ResponseWriter, r *http.Request) {
        s.git.Trigger()
        writeJSON(w, http.StatusOK, refreshResponse{Message: "refresh triggered"})
}

// handleAPISpec serves the OpenAPI 3.0 specification for the /api/v1/ endpoints.
// This endpoint does not require authentication.
func (s *Server) handleAPISpec(w http.ResponseWriter, r *http.Request) {
        w.Header().Set("Content-Type", "application/json")
        w.WriteHeader(http.StatusOK)
        w.Write([]byte(openAPISpec))
}

// openAPISpec is the OpenAPI 3.0 JSON specification for the /api/v1/ endpoints.
const openAPISpec = `{
  "openapi": "3.0.3",
  "info": {
    "title": "nomad-gitops",
    "description": "Query drift state between a Git repo and a Nomad cluster.",
    "version": "v1"
  },
  "servers": [{"url": "/api/v1"}],
  "security": [{"bearerAuth": []}],
  "components": {
    "securitySchemes": {
      "bearerAuth": {"type": "http", "scheme": "bearer"}
    },
    "schemas": {
      "JobDiff": {
        "type": "object",
        "properties": {
          "job_id":    {"type": "string"},
          "hcl_file":  {"type": "string"},
          "diff_type": {"type": "string", "enum": ["modified", "missing_from_nomad", "missing_from_hcl"]},
          "detail":    {"type": "string"},
          "apply_action": {"type": "string", "description": "Disposition of this diff: whether it will be applied and, if not, why.", "enum": ["queued", "blocked_by_policy", "blocked_preexisting_drift", "blocked_creation_disabled", "skipped_meta_only", "observation_only", "queued_deregister", "deregister_pending_grace", "no_actionable_change", "blocked_known_failed"]},
          "apply_detail": {"type": "string", "description": "Optional human-readable explanation refining apply_action with the specific values involved (e.g. the effective update policy, its source, and what to change to apply the diff). Omitted when apply_action alone is self-explanatory."}
        }
      },
      "SelectedJob": {
        "type": "object",
        "properties": {
          "job_id":           {"type": "string"},
          "selection_reason": {"type": "string", "enum": ["glob", "meta", "both"]}
        }
      },
      "DiffsResponse": {
        "type": "object",
        "properties": {
          "diffs":           {"type": "array", "items": {"$ref": "#/components/schemas/JobDiff"}},
          "last_check_time": {"type": "string", "format": "date-time"},
          "last_commit":     {"type": "string"}
        }
      },
      "SelectedJobsResponse": {
        "type": "object",
        "properties": {
          "jobs":            {"type": "array", "items": {"$ref": "#/components/schemas/SelectedJob"}},
          "last_check_time": {"type": "string", "format": "date-time"},
          "last_commit":     {"type": "string"}
        }
      },
      "JobUpdate": {
        "type": "object",
        "properties": {
          "update_id":              {"type": "string", "description": "Stable across restarts. REGISTER/DEREGISTER: <job_id>/<git_commit_short>. REVERT: <job_id>/revert-<failed_version>."},
          "job_id":                 {"type": "string"},
          "hcl_file":               {"type": "string"},
          "git_commit":             {"type": "string"},
          "operation":              {"type": "string", "enum": ["REGISTER", "DEREGISTER", "REVERT"]},
          "status":                 {"type": "string", "enum": ["PENDING", "IN_PROGRESS", "SUCCEEDED", "FAILED", "SUPERSEDED"]},
          "policy":                 {"type": "string", "enum": ["full", "image-only", "none"]},
          "nomad_job_modify_index": {"type": "integer", "description": "CAS token captured at detection time; 0 = job did not exist"},
          "nomad_raft_index":       {"type": "integer"},
          "detected_at":            {"type": "string", "format": "date-time"},
          "applied_at":             {"type": "string", "format": "date-time"},
          "error":                  {"type": "string"},
          "revert_to_version":      {"type": "integer", "description": "REVERT only: the stable job version rolled back to"},
          "revert_from_version":    {"type": "integer", "description": "REVERT only: the failed version used as the CAS guard"}
        }
      },
      "UpdatesResponse": {
        "type": "object",
        "properties": {
          "updates": {"type": "array", "items": {"$ref": "#/components/schemas/JobUpdate"}}
        }
      },
      "StatusResponse": {
        "type": "object",
        "properties": {
          "last_commit":  {"type": "string"},
          "last_updated": {"type": "string", "format": "date-time"}
        }
      },
      "VersionResponse": {
        "type": "object",
        "properties": {
          "version":    {"type": "string"},
          "commit":     {"type": "string"},
          "build_date": {"type": "string"}
        }
      },
      "RefreshResponse": {
        "type": "object",
        "properties": {
          "message": {"type": "string"}
        }
      },
      "ErrorResponse": {
        "type": "object",
        "properties": {
          "error": {"type": "string"}
        }
      }
    },
    "responses": {
      "Unauthorized": {
        "description": "Missing or invalid API key",
        "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ErrorResponse"}}}
      },
      "ServiceUnavailable": {
        "description": "Server has not completed its initial check",
        "content": {"application/json": {"schema": {"$ref": "#/components/schemas/ErrorResponse"}}}
      }
    }
  },
  "paths": {
    "/diffs": {
      "get": {
        "summary": "Current job diffs",
        "description": "Returns all jobs where drift was detected between Git and Nomad.",
        "responses": {
          "200": {"description": "Diff results", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/DiffsResponse"}}}},
          "401": {"$ref": "#/components/responses/Unauthorized"},
          "503": {"$ref": "#/components/responses/ServiceUnavailable"}
        }
      }
    },
    "/selected-jobs": {
      "get": {
        "summary": "Jobs currently selected for monitoring",
        "description": "Returns all jobs that matched the configured selection criteria during the last check, with the reason each was included.",
        "responses": {
          "200": {"description": "Selected jobs", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/SelectedJobsResponse"}}}},
          "401": {"$ref": "#/components/responses/Unauthorized"},
          "503": {"$ref": "#/components/responses/ServiceUnavailable"}
        }
      }
    },
    "/updates": {
      "get": {
        "summary": "GitOps update queue",
        "description": "Returns the queue of intended job changes derived from detected drift: pending, in-progress, and recently completed updates.",
        "responses": {
          "200": {"description": "Update queue", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/UpdatesResponse"}}}},
          "401": {"$ref": "#/components/responses/Unauthorized"}
        }
      }
    },
    "/status": {
      "get": {
        "summary": "Git watcher status",
        "description": "Returns the last known git commit and fetch time.",
        "responses": {
          "200": {"description": "Git status", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/StatusResponse"}}}},
          "401": {"$ref": "#/components/responses/Unauthorized"},
          "503": {"$ref": "#/components/responses/ServiceUnavailable"}
        }
      }
    },
    "/version": {
      "get": {
        "summary": "Build version",
        "description": "Returns the version, commit hash, and build date of the running binary.",
        "responses": {
          "200": {"description": "Version info", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/VersionResponse"}}}},
          "401": {"$ref": "#/components/responses/Unauthorized"}
        }
      }
    },
    "/refresh": {
      "post": {
        "summary": "Trigger an immediate git pull and diff check",
        "responses": {
          "200": {"description": "Refresh triggered", "content": {"application/json": {"schema": {"$ref": "#/components/schemas/RefreshResponse"}}}},
          "401": {"$ref": "#/components/responses/Unauthorized"}
        }
      }
    }
  }
}
`

package server

import (
        "fmt"
        "sort"
        "strings"
        "time"

        nomadapi "github.com/hashicorp/nomad/api"

        "github.com/gerrowadat/nomad-gitops/internal/nomad"
)

// renderDiffsText produces a nomad-job-plan-style plain-text representation
// of the current diff state. When redactionEnabled is true a banner states
// that potentially sensitive values have been replaced with [REDACTED].
func renderDiffsText(diffs []nomad.JobDiff, lastCheck time.Time, commit string, redactionEnabled bool) string {
        var b strings.Builder

        fmt.Fprintln(&b, "nomad-gitops diff report")
        if !lastCheck.IsZero() {
                fmt.Fprintf(&b, "Last check: %s | Commit: %s\n", lastCheck.Format(time.RFC3339), commit)
        }
        fmt.Fprintln(&b)

        if len(diffs) == 0 {
                fmt.Fprintln(&b, "No differences detected.")
                return b.String()
        }

        fmt.Fprintf(&b, "%d difference(s) detected:\n", len(diffs))
        if redactionEnabled {
                fmt.Fprintf(&b, "NOTE: potentially sensitive values (env vars, template bodies, secret-like keys) are shown as %s. Disable with --redact-secrets=false.\n", nomad.RedactedValue)
        }

        for _, d := range diffs {
                fmt.Fprintln(&b)
                switch d.DiffType {
                case nomad.DiffTypeMissingFromNomad:
                        fmt.Fprintf(&b, "+ Job: %q\n", d.JobID)
                        fmt.Fprintf(&b, "  Defined in %s but not registered in Nomad.\n", d.HCLFile)
                case nomad.DiffTypeMissingFromHCL:
                        fmt.Fprintf(&b, "- Job: %q\n", d.JobID)
                        fmt.Fprintf(&b, "  %s\n", d.Detail)
                case nomad.DiffTypeModified:
                        if d.PlanDiff != nil {
                                renderJobDiff(&b, d.PlanDiff, d.HCLFile)
                        } else {
                                fmt.Fprintf(&b, "+/- Job: %q\n", d.JobID)
                                fmt.Fprintf(&b, "  %s\n", d.Detail)
                        }
                }
                if d.ApplyDetail != "" {
                        fmt.Fprintf(&b, "  → %s\n", d.ApplyDetail)
                } else if d.ApplyAction != "" {
                        fmt.Fprintf(&b, "  → %s\n", d.ApplyAction.Describe())
                }
        }

        return b.String()
}

func renderJobDiff(b *strings.Builder, jd *nomadapi.JobDiff, hclFile string) {
        if hclFile != "" {
                fmt.Fprintf(b, "%s Job: %q  (%s)\n", diffSymbol(jd.Type), jd.ID, hclFile)
        } else {
                fmt.Fprintf(b, "%s Job: %q\n", diffSymbol(jd.Type), jd.ID)
        }
        renderFields(b, jd.Fields, "  ")
        renderObjects(b, jd.Objects, "  ")
        for _, tg := range jd.TaskGroups {
                renderTaskGroupDiff(b, tg, "  ")
        }
}

func renderTaskGroupDiff(b *strings.Builder, tg *nomadapi.TaskGroupDiff, indent string) {
        var updates string
        if len(tg.Updates) > 0 {
                parts := make([]string, 0, len(tg.Updates))
                for k, v := range tg.Updates {
                        if v > 0 {
                                parts = append(parts, fmt.Sprintf("%d %s", v, k))
                        }
                }
                sort.Strings(parts)
                if len(parts) > 0 {
                        updates = " (" + strings.Join(parts, ", ") + ")"
                }
        }
        fmt.Fprintf(b, "%s%s Task Group: %q%s\n", indent, diffSymbol(tg.Type), tg.Name, updates)
        renderFields(b, tg.Fields, indent+"  ")
        renderObjects(b, tg.Objects, indent+"  ")
        for _, t := range tg.Tasks {
                renderTaskDiff(b, t, indent+"  ")
        }
}

func renderTaskDiff(b *strings.Builder, t *nomadapi.TaskDiff, indent string) {
        ann := ""
        if len(t.Annotations) > 0 {
                ann = " (" + strings.Join(t.Annotations, ", ") + ")"
        }
        fmt.Fprintf(b, "%s%s Task: %q%s\n", indent, diffSymbol(t.Type), t.Name, ann)
        renderFields(b, t.Fields, indent+"  ")
        renderObjects(b, t.Objects, indent+"  ")
}

func renderFields(b *strings.Builder, fields []*nomadapi.FieldDiff, indent string) {
        for _, f := range fields {
                ann := ""
                if len(f.Annotations) > 0 {
                        ann = " (" + strings.Join(f.Annotations, ", ") + ")"
                }
                switch f.Type {
                case "Added":
                        fmt.Fprintf(b, "%s+ %s: %q%s\n", indent, f.Name, f.New, ann)
                case "Deleted":
                        fmt.Fprintf(b, "%s- %s: %q%s\n", indent, f.Name, f.Old, ann)
                case "Edited":
                        fmt.Fprintf(b, "%s~ %s: %q => %q%s\n", indent, f.Name, f.Old, f.New, ann)
                }
        }
}

func renderObjects(b *strings.Builder, objects []*nomadapi.ObjectDiff, indent string) {
        renderObjectsDepth(b, objects, indent, 1)
}

// renderObjectsDepth is renderObjects with an explicit nesting level. depth is
// 1 at the top; beyond nomad.MaxPlanDiffObjectDepth, rendering stops rather
// than recursing without bound, mirroring the cap classification and
// redaction apply to the same plan-diff tree (internal/nomad/diffdepth.go).
func renderObjectsDepth(b *strings.Builder, objects []*nomadapi.ObjectDiff, indent string, depth int) {
        if depth > nomad.MaxPlanDiffObjectDepth {
                fmt.Fprintf(b, "%s... (diff truncated: exceeds maximum nesting depth)\n", indent)
                return
        }
        for _, o := range objects {
                fmt.Fprintf(b, "%s%s %s {\n", indent, diffSymbol(o.Type), o.Name)
                renderFields(b, o.Fields, indent+"  ")
                renderObjectsDepth(b, o.Objects, indent+"  ", depth+1)
                fmt.Fprintf(b, "%s}\n", indent)
        }
}

func diffSymbol(t string) string {
        switch t {
        case "Added":
                return "+"
        case "Deleted":
                return "-"
        case "Edited":
                return "+/-"
        default:
                return "?"
        }
}

// Package server provides the HTTP server exposing /healthz, /metrics, and
// the git webhook endpoint.
package server

import (
        "context"
        "encoding/json"
        "fmt"
        "html/template"
        "log/slog"
        "net/http"
        "strings"
        "sync"
        "time"

        webhookgithub "github.com/go-playground/webhooks/v6/github"
        "github.com/prometheus/client_golang/prometheus"
        "github.com/prometheus/client_golang/prometheus/promauto"
        "github.com/prometheus/client_golang/prometheus/promhttp"

        "github.com/gerrowadat/nomad-gitops/internal/config"
        "github.com/gerrowadat/nomad-gitops/internal/nomad"
)

// DiffSource is satisfied by *nomad.Differ.
type DiffSource interface {
        Diffs() ([]nomad.JobDiff, time.Time, string)
        SelectedJobs() ([]nomad.SelectedJob, time.Time, string)
        // Updates returns a snapshot of the GitOps update queue.
        Updates() []nomad.JobUpdate
        // Ready reports whether at least one diff check has completed.
        Ready() bool
}

// GitStatusSource is satisfied by *gitwatch.Watcher.
type GitStatusSource interface {
        Trigger()
        Status() (lastCommit string, lastUpdate time.Time)
        // Ready reports whether the initial git clone has completed.
        Ready() bool
}

// maxWebhookBodyBytes caps the webhook request body. GitHub limits webhook
// payloads to 25 MB; anything larger is not a legitimate delivery. Without a
// cap the webhook library reads the entire body into memory, which lets an
// attacker exhaust memory by streaming an arbitrarily large request.
const maxWebhookBodyBytes = 25 << 20

// Server holds the HTTP mux and all dependencies.
type Server struct {
        cfg       *config.Config
        diffs     DiffSource
        git       GitStatusSource
        buildInfo BuildInfo
        mux       *http.ServeMux
        handler   http.Handler // mux wrapped in securityHeaders

        webhookMu          sync.RWMutex
        lastWebhookSuccess time.Time
        lastWebhookFailure time.Time

        // Prometheus metrics
        webhookEvents           *prometheus.CounterVec
        lastWebhookSuccessGauge prometheus.Gauge
        lastWebhookFailureGauge prometheus.Gauge
}

// New creates a Server that registers Prometheus metrics into the default registry.
func New(cfg *config.Config, diffs DiffSource, git GitStatusSource, info BuildInfo) *Server {
        return NewWithRegistry(cfg, diffs, git, info, prometheus.DefaultRegisterer)
}

// NewWithRegistry creates a Server with a custom Prometheus Registerer.
// Useful in tests to avoid duplicate-registration panics when creating multiple servers.
func NewWithRegistry(cfg *config.Config, diffs DiffSource, git GitStatusSource, info BuildInfo, reg prometheus.Registerer) *Server {
        s := &Server{
                cfg:       cfg,
                diffs:     diffs,
                git:       git,
                buildInfo: info,

                webhookEvents: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
                        Name: "nomad_gitops_webhook_events_total",
                        Help: "Total number of webhook events received, by event type.",
                }, []string{"event"}),
                lastWebhookSuccessGauge: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
                        Name: "nomad_gitops_last_webhook_success_timestamp_seconds",
                        Help: "Unix timestamp of the most recent successfully parsed webhook.",
                }),
                lastWebhookFailureGauge: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
                        Name: "nomad_gitops_last_webhook_failure_timestamp_seconds",
                        Help: "Unix timestamp of the most recent webhook that failed to parse.",
                }),
        }

        // Static info metric carrying the build version.
        promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
                Name: "nomad_gitops_info",
                Help: "Build information.",
        }, []string{"version"}).WithLabelValues(info.Version).Set(1)

        // Use the provided registry as the Prometheus gatherer if possible,
        // otherwise fall back to the global default.
        var metricsHandler http.Handler
        if g, ok := reg.(prometheus.Gatherer); ok {
                metricsHandler = promhttp.HandlerFor(g, promhttp.HandlerOpts{})
        } else {
                metricsHandler = promhttp.Handler()
        }

        s.mux = http.NewServeMux()
        s.mux.HandleFunc("/{$}", s.handleIndex)
        s.mux.HandleFunc("/healthz", s.handleHealthz)
        s.mux.HandleFunc("/diffs", s.handleDiffs)
        s.mux.Handle("/metrics", metricsHandler)
        s.mux.HandleFunc(cfg.WebhookPath, s.handleWebhook())

        // Mount authenticated JSON API if a key is configured.
        if cfg.APIKey != "" {
                apiMux := http.NewServeMux()
                apiMux.HandleFunc("GET /api/v1/diffs", s.handleAPIDiffs)
                apiMux.HandleFunc("GET /api/v1/selected-jobs", s.handleAPISelectedJobs)
                apiMux.HandleFunc("GET /api/v1/updates", s.handleAPIUpdates)
                apiMux.HandleFunc("GET /api/v1/status", s.handleAPIStatus)
                apiMux.HandleFunc("GET /api/v1/version", s.handleAPIVersion)
                apiMux.HandleFunc("POST /api/v1/refresh", s.handleAPIRefresh)
                s.mux.Handle("/api/v1/", requireAPIKey(cfg.APIKey)(apiMux))
                // OpenAPI spec is public — no auth required.
                s.mux.HandleFunc("GET /api/openapi.json", s.handleAPISpec)
        } else {
                slog.Warn("API key not configured; /api/ endpoints are disabled. Set --api-key / API_KEY to enable.")
        }

        s.handler = securityHeaders{next: s.mux}

        return s
}

// securityHeaders sets standard hardening headers on every response. The web
// console serves no scripts and is never meant to be framed or sniffed.
// It is a comparable struct (not an http.HandlerFunc) so values returned by
// Server.Handler can be compared with ==.
type securityHeaders struct {
        next http.Handler
}

func (s securityHeaders) ServeHTTP(w http.ResponseWriter, r *http.Request) {
        h := w.Header()
        h.Set("X-Content-Type-Options", "nosniff")
        h.Set("X-Frame-Options", "DENY")
        h.Set("Content-Security-Policy", "default-src 'none'; style-src 'unsafe-inline'")
        h.Set("Referrer-Policy", "no-referrer")
        s.next.ServeHTTP(w, r)
}

// newHTTPServer constructs the http.Server with timeouts to prevent slowloris
// and other connection-exhaustion attacks.
func (s *Server) newHTTPServer() *http.Server {
        return &http.Server{
                Addr:              s.cfg.ListenAddr,
                Handler:           s.handler,
                ReadHeaderTimeout: 10 * time.Second,
                ReadTimeout:       30 * time.Second,
                WriteTimeout:      30 * time.Second,
        }
}

// Run starts the HTTP server and blocks until ctx is cancelled.
func (s *Server) Run(ctx context.Context) error {
        srv := s.newHTTPServer()

        // ListenAndServe runs in the background so the main path can wait on both
        // it and ctx. Keeping it here (rather than a separate ctx.Done waiter
        // goroutine) means the goroutine always ends before Run returns: on
        // shutdown ListenAndServe returns ErrServerClosed, and on an early bind
        // error it returns that error directly — neither outlives Run.
        serveErr := make(chan error, 1)
        go func() {
                err := srv.ListenAndServe()
                if err == http.ErrServerClosed {
                        err = nil
                }
                serveErr <- err
        }()

        slog.Info("HTTP server listening", "addr", s.cfg.ListenAddr)

        select {
        case <-ctx.Done():
                shutCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
                defer cancel()
                _ = srv.Shutdown(shutCtx)
                // Drain the serve goroutine so it does not outlive Run.
                <-serveErr
                return nil
        case err := <-serveErr:
                if err != nil {
                        return fmt.Errorf("http server: %w", err)
                }
                return nil
        }
}

// Handler returns the underlying http.Handler, useful for testing without a
// real listener.
func (s *Server) Handler() http.Handler {
        return s.handler
}

var indexTmpl = template.Must(template.New("index").Parse(`<!DOCTYPE html>
<html>
<head>
  <meta charset="utf-8">
  <title>nomad-gitops</title>
  <style>
    body { font-family: sans-serif; max-width: 640px; margin: 2em auto; color: #222; }
    h1   { margin-bottom: 0.2em; }
    .ok  { color: #2a7a2a; font-weight: bold; }
    .bad { color: #b94040; font-weight: bold; }
    .starting { color: #7a6a00; font-weight: bold; }
    code { background: #f4f4f4; padding: 0.1em 0.3em; border-radius: 3px; }
    ul   { line-height: 1.8; }
  </style>
</head>
<body>
  <h1>nomad-gitops <small>{{.Version}}</small></h1>
  <p>Status:
    {{- if .Starting}}
    <span class="starting">starting — initial state not yet built</span>
    {{- else if .DiffCount}}
    <span class="bad">{{.DiffCount}} difference(s) detected</span>
    {{- else}}
    <span class="ok">OK — no differences</span>
    {{- end}}
  </p>
  <p>Watching:
    {{- if .SelectionGlob}} jobs matching <code>{{.SelectionGlob}}</code>{{end}}
    {{- if and .SelectionGlob .ManagedMetaKey}}, or{{end}}
    {{- if .ManagedMetaKey}} jobs with <code>{{.ManagedMetaKey}}=true</code> in job meta{{end}}
    {{- if not (or .SelectionGlob .ManagedMetaKey)}} <em>no jobs — no selection criteria configured</em>{{end}}
  </p>
  {{- if .ManagedMetaKey}}
  <p><small>To include a job, add <code>meta { &#34;{{.ManagedMetaKey}}&#34; = &#34;true&#34; }</code> to its HCL definition.</small></p>
  {{- end}}
  <p>Apply mode: default policy <code>{{.DefaultPolicy}}</code>
    {{- if eq .DefaultPolicy "none"}} (detection only unless a job&#39;s meta opts in){{end}},
    job creation {{if .JobCreationEnabled}}<span class="bad">enabled</span>{{else}}disabled{{end}}
    {{- if .PendingUpdates}}, <span class="starting">{{.PendingUpdates}} update(s) pending</span>{{end}}
  </p>
  {{- if .LastCheck}}
  <p>Last diff check: {{.LastCheck}}{{if .Commit}} (commit <code>{{.Commit}}</code>){{end}}</p>
  {{- end}}
  {{- if (or .LastWebhookOK .LastWebhookFail)}}
  <p>Last webhook:
    {{- if .LastWebhookOK}} ok <code>{{.LastWebhookOK}}</code>{{end}}
    {{- if .LastWebhookFail}} &nbsp; failed <code>{{.LastWebhookFail}}</code>{{end}}
  </p>
  {{- end}}
  {{- if .SelectedJobs}}
  <h2>Selected jobs ({{len .SelectedJobs}})</h2>
  <table style="border-collapse:collapse;width:100%">
    <thead><tr style="text-align:left;border-bottom:1px solid #ccc">
      <th style="padding:0.3em 1em 0.3em 0">Job</th>
      <th style="padding:0.3em 0">Selected by</th>
    </tr></thead>
    <tbody>
    {{- range .SelectedJobs}}
    <tr>
      <td style="padding:0.25em 1em 0.25em 0"><code>{{.JobID}}</code></td>
      <td style="padding:0.25em 0">{{.Reason}}</td>
    </tr>
    {{- end}}
    </tbody>
  </table>
  {{- end}}
  <ul>
    <li><a href="/diffs">/diffs</a> — current job diffs (plan-style)</li>
    <li><a href="/healthz">/healthz</a> — JSON health check</li>
    <li><a href="/metrics">/metrics</a> — Prometheus metrics</li>
  </ul>
</body>
</html>
`))

func (s *Server) handleIndex(w http.ResponseWriter, r *http.Request) {
        starting := !s.git.Ready() || !s.diffs.Ready()

        var diffs []nomad.JobDiff
        var selectedJobs []nomad.SelectedJob
        var lastCheck time.Time
        var commit string
        if !starting {
                diffs, lastCheck, commit = s.diffs.Diffs()
                selectedJobs, _, _ = s.diffs.SelectedJobs()
        }

        s.webhookMu.RLock()
        lastOK := s.lastWebhookSuccess
        lastFail := s.lastWebhookFailure
        s.webhookMu.RUnlock()

        managedMetaKey := ""
        if s.cfg.ManagedMetaPrefix != "" {
                managedMetaKey = s.cfg.ManagedMetaPrefix + "_managed"
        }

        pendingUpdates := 0
        for _, u := range s.diffs.Updates() {
                if u.Status == nomad.JobUpdateStatusPending || u.Status == nomad.JobUpdateStatusInProgress {
                        pendingUpdates++
                }
        }
        defaultPolicy := s.cfg.DefaultUpdatePolicy
        if defaultPolicy == "" {
                defaultPolicy = "none"
        }

        data := struct {
                Version            string
                Starting           bool
                DiffCount          int
                SelectedJobs       []nomad.SelectedJob
                LastCheck          string
                Commit             string
                LastWebhookOK      string
                LastWebhookFail    string
                SelectionGlob      string
                ManagedMetaKey     string
                DefaultPolicy      string
                JobCreationEnabled bool
                PendingUpdates     int
        }{
                Version:            s.buildInfo.Version,
                Starting:           starting,
                DiffCount:          len(diffs),
                SelectedJobs:       selectedJobs,
                Commit:             commit,
                SelectionGlob:      s.cfg.JobSelectorGlob,
                ManagedMetaKey:     managedMetaKey,
                DefaultPolicy:      defaultPolicy,
                JobCreationEnabled: s.cfg.EnableJobCreation,
                PendingUpdates:     pendingUpdates,
        }
        if !lastCheck.IsZero() {
                data.LastCheck = lastCheck.Format(time.RFC3339)
        }
        if !lastOK.IsZero() {
                data.LastWebhookOK = lastOK.Format(time.RFC3339)
        }
        if !lastFail.IsZero() {
                data.LastWebhookFail = lastFail.Format(time.RFC3339)
        }

        w.Header().Set("Content-Type", "text/html; charset=utf-8")
        if starting {
                w.WriteHeader(http.StatusServiceUnavailable)
        }
        _ = indexTmpl.Execute(w, data)
}

func (s *Server) handleDiffs(w http.ResponseWriter, r *http.Request) {
        if !s.git.Ready() || !s.diffs.Ready() {
                http.Error(w, "not ready: initial state not yet built", http.StatusServiceUnavailable)
                return
        }
        diffs, lastCheck, commit := s.diffs.Diffs()
        w.Header().Set("Content-Type", "text/plain; charset=utf-8")
        fmt.Fprint(w, renderDiffsText(diffs, lastCheck, commit, s.cfg.RedactSecrets))
}

// HealthResponse is the JSON body returned by /healthz.
type HealthResponse struct {
        Status     string      `json:"status"`
        Message    string      `json:"message,omitempty"`
        DiffCount  int         `json:"diff_count"`
        Diffs      []DiffEntry `json:"diffs"`
        LastCheck  string      `json:"last_check,omitempty"`
        GitCommit  string      `json:"git_commit,omitempty"`
        GitUpdated string      `json:"git_updated,omitempty"`
}

// DiffEntry is one element of HealthResponse.Diffs.
type DiffEntry struct {
        JobID       string `json:"job_id"`
        HCLFile     string `json:"hcl_file,omitempty"`
        DiffType    string `json:"diff_type"`
        Detail      string `json:"detail"`
        ApplyAction string `json:"apply_action,omitempty"`
        ApplyDetail string `json:"apply_detail,omitempty"`
}

func (s *Server) handleHealthz(w http.ResponseWriter, r *http.Request) {
        if !s.git.Ready() || !s.diffs.Ready() {
                w.Header().Set("Content-Type", "application/json")
                w.WriteHeader(http.StatusServiceUnavailable)
                _ = json.NewEncoder(w).Encode(HealthResponse{
                        Status:  "starting",
                        Message: "initial state not yet built",
                })
                return
        }

        diffs, lastCheck, gitCommit := s.diffs.Diffs()
        _, gitUpdated := s.git.Status()

        status := "ok"
        if len(diffs) > 0 {
                status = "diffs_detected"
        }

        entries := make([]DiffEntry, 0, len(diffs))
        for _, d := range diffs {
                entries = append(entries, DiffEntry{
                        JobID:       d.JobID,
                        HCLFile:     d.HCLFile,
                        DiffType:    string(d.DiffType),
                        Detail:      d.Detail,
                        ApplyAction: string(d.ApplyAction),
                        ApplyDetail: d.ApplyDetail,
                })
        }

        resp := HealthResponse{
                Status:    status,
                DiffCount: len(diffs),
                Diffs:     entries,
        }
        if !lastCheck.IsZero() {
                resp.LastCheck = lastCheck.Format(time.RFC3339)
        }
        if gitCommit != "" {
                resp.GitCommit = gitCommit
        }
        if !gitUpdated.IsZero() {
                resp.GitUpdated = gitUpdated.Format(time.RFC3339)
        }

        w.Header().Set("Content-Type", "application/json")
        w.WriteHeader(http.StatusOK)
        _ = json.NewEncoder(w).Encode(resp)
}

// recordWebhookOutcome records the current time into field under the webhook
// mutex and updates the corresponding Prometheus gauge.
func (s *Server) recordWebhookOutcome(field *time.Time, gauge prometheus.Gauge) {
        now := time.Now()
        s.webhookMu.Lock()
        *field = now
        s.webhookMu.Unlock()
        gauge.Set(float64(now.Unix()))
}

func (s *Server) handleWebhook() http.HandlerFunc {
        if s.cfg.WebhookSecret == "" {
                slog.Warn("Webhook secret is empty: webhook endpoint accepts unsigned requests. " +
                        "Set --webhook-secret / WEBHOOK_SECRET to require HMAC-SHA256 signatures.")
        }
        hook, err := webhookgithub.New(webhookgithub.Options.Secret(s.cfg.WebhookSecret))
        if err != nil {
                // This only errors with an invalid secret; log and serve a stub.
                slog.Error("Failed to initialise GitHub webhook handler", "err", err)
                return func(w http.ResponseWriter, r *http.Request) {
                        http.Error(w, "webhook handler misconfigured", http.StatusInternalServerError)
                }
        }

        return func(w http.ResponseWriter, r *http.Request) {
                eventType := r.Header.Get("X-GitHub-Event")
                deliveryID := r.Header.Get("X-GitHub-Delivery")

                // The webhook library reads the whole body into memory; cap it so an
                // oversized request fails instead of exhausting memory.
                r.Body = http.MaxBytesReader(w, r.Body, maxWebhookBodyBytes)

                payload, err := hook.Parse(r, webhookgithub.PushEvent, webhookgithub.PingEvent)
                if err != nil {
                        if err == webhookgithub.ErrEventNotFound {
                                s.webhookEvents.WithLabelValues("unknown").Inc()
                                slog.Debug("Ignoring unhandled webhook event", "event", eventType, "delivery", deliveryID)
                                w.WriteHeader(http.StatusOK)
                                return
                        }
                        s.webhookEvents.WithLabelValues("error").Inc()
                        slog.Warn("Webhook rejected", "event", eventType, "delivery", deliveryID, "err", err)
                        s.recordWebhookOutcome(&s.lastWebhookFailure, s.lastWebhookFailureGauge)
                        http.Error(w, "bad webhook payload", http.StatusBadRequest)
                        return
                }

                switch p := payload.(type) {
                case webhookgithub.PushPayload:
                        s.webhookEvents.WithLabelValues("push").Inc()
                        branch := strings.TrimPrefix(p.Ref, "refs/heads/")
                        slog.Info("Received push webhook",
                                "delivery", deliveryID,
                                "repo", p.Repository.FullName,
                                "branch", branch,
                                "before", p.Before,
                                "after", p.After,
                                "commits", len(p.Commits),
                                "pusher", p.Pusher.Name,
                                "compare", p.Compare,
                        )
                        if branch == s.cfg.Branch {
                                s.git.Trigger()
                        }
                case webhookgithub.PingPayload:
                        s.webhookEvents.WithLabelValues("ping").Inc()
                        slog.Info("Received ping webhook",
                                "delivery", deliveryID,
                                "hook_id", p.HookID,
                                "repo", p.Repository.FullName,
                                "events", p.Hook.Events,
                        )
                }

                s.recordWebhookOutcome(&s.lastWebhookSuccess, s.lastWebhookSuccessGauge)

                w.WriteHeader(http.StatusOK)
        }
}