This commit is contained in:
@@ -0,0 +1,65 @@
|
||||
package actions
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/store"
|
||||
)
|
||||
|
||||
type Result struct {
|
||||
Action string `json:"action"`
|
||||
DryRun bool `json:"dry_run"`
|
||||
Detail string `json:"detail"`
|
||||
}
|
||||
|
||||
type Runner struct {
|
||||
dryRun bool
|
||||
}
|
||||
|
||||
func NewRunner(dryRun bool) *Runner { return &Runner{dryRun: dryRun} }
|
||||
|
||||
func (r *Runner) Run(ctx context.Context, rule store.Rule, action store.Action) (Result, error) {
|
||||
_ = ctx
|
||||
if !action.Enabled {
|
||||
return Result{Action: action.Type, DryRun: r.effectiveDryRun(action), Detail: "disabled"}, nil
|
||||
}
|
||||
switch action.Type {
|
||||
case "generic_webhook", "webhook":
|
||||
url := action.Params["url"]
|
||||
if url == "" {
|
||||
return Result{}, fmt.Errorf("webhook action requires url")
|
||||
}
|
||||
return Result{Action: action.Type, DryRun: true, Detail: "would POST webhook for rule " + rule.Name}, nil
|
||||
case "record_recommendation":
|
||||
return Result{Action: action.Type, DryRun: false, Detail: "recommendation recorded for operator review"}, nil
|
||||
case "allowlisted_command", "command":
|
||||
id := action.Params["command_id"]
|
||||
if id == "" {
|
||||
id = action.Params["id"]
|
||||
}
|
||||
if id == "" {
|
||||
return Result{}, fmt.Errorf("allowlisted command requires command_id")
|
||||
}
|
||||
return Result{Action: action.Type, DryRun: true, Detail: "would run allowlisted command " + id}, nil
|
||||
case "portainer_restart":
|
||||
service := action.Params["service"]
|
||||
if service == "" {
|
||||
return Result{}, fmt.Errorf("portainer_restart requires service")
|
||||
}
|
||||
return Result{Action: action.Type, DryRun: true, Detail: "would restart Portainer service " + service}, nil
|
||||
default:
|
||||
return Result{}, fmt.Errorf("unsupported action type %q", action.Type)
|
||||
}
|
||||
return Result{}, fmt.Errorf("unsupported action type %q", action.Type)
|
||||
}
|
||||
|
||||
func (r *Runner) effectiveDryRun(action store.Action) bool {
|
||||
if r.dryRun {
|
||||
return true
|
||||
}
|
||||
if action.DryRun == nil {
|
||||
return true
|
||||
}
|
||||
return *action.DryRun
|
||||
}
|
||||
@@ -0,0 +1,108 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/analysis"
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/store"
|
||||
)
|
||||
|
||||
type Result struct {
|
||||
ChannelID string
|
||||
Detail string
|
||||
}
|
||||
|
||||
type Dispatcher struct {
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
func NewDispatcher() *Dispatcher { return &Dispatcher{http: &http.Client{Timeout: 15 * time.Second}} }
|
||||
|
||||
func (d *Dispatcher) Send(ctx context.Context, rule store.Rule, finding analysis.Finding, channels []store.AlertChannel) ([]Result, error) {
|
||||
if len(channels) == 0 {
|
||||
log.Printf("alert for rule %q: %s", rule.Name, finding.Summary)
|
||||
return []Result{{Detail: "no channels configured; logged alert intent"}}, nil
|
||||
}
|
||||
var results []Result
|
||||
var firstErr error
|
||||
for _, channel := range channels {
|
||||
if !channel.Enabled {
|
||||
results = append(results, Result{ChannelID: channel.ID, Detail: "disabled"})
|
||||
continue
|
||||
}
|
||||
var err error
|
||||
switch channel.Type {
|
||||
case "ntfy":
|
||||
err = d.sendNtfy(ctx, rule, finding, channel)
|
||||
case "gotify", "generic_webhook", "smtp":
|
||||
err = fmt.Errorf("%s delivery is scaffolded but not implemented in phase 1", channel.Type)
|
||||
default:
|
||||
err = fmt.Errorf("unsupported channel type %q", channel.Type)
|
||||
}
|
||||
if err != nil {
|
||||
if firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
results = append(results, Result{ChannelID: channel.ID, Detail: err.Error()})
|
||||
continue
|
||||
}
|
||||
results = append(results, Result{ChannelID: channel.ID, Detail: "sent"})
|
||||
}
|
||||
return results, firstErr
|
||||
}
|
||||
|
||||
func (d *Dispatcher) Test(ctx context.Context, channel store.AlertChannel) error {
|
||||
finding := analysis.Finding{Summary: "Log Guardian test notification", Confidence: "test"}
|
||||
rule := store.Rule{Name: "Test alert", Severity: "info"}
|
||||
switch channel.Type {
|
||||
case "ntfy":
|
||||
return d.sendNtfy(ctx, rule, finding, channel)
|
||||
case "gotify", "generic_webhook", "smtp":
|
||||
return fmt.Errorf("%s delivery is scaffolded but not implemented in phase 1", channel.Type)
|
||||
default:
|
||||
return fmt.Errorf("unsupported channel type %q", channel.Type)
|
||||
}
|
||||
}
|
||||
|
||||
func (d *Dispatcher) sendNtfy(ctx context.Context, rule store.Rule, finding analysis.Finding, channel store.AlertChannel) error {
|
||||
server := strings.TrimRight(channel.Params["server_url"], "/")
|
||||
topic := strings.Trim(channel.Params["topic"], "/")
|
||||
if server == "" || topic == "" {
|
||||
return fmt.Errorf("ntfy requires server_url and topic")
|
||||
}
|
||||
target, err := url.JoinPath(server, topic)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body := rule.Name + ": " + finding.Summary
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, target, bytes.NewBufferString(body))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Title", "Log Guardian: "+rule.Severity)
|
||||
if priority := channel.Params["priority"]; priority != "" {
|
||||
req.Header.Set("Priority", priority)
|
||||
}
|
||||
if tags := channel.Params["tags"]; tags != "" {
|
||||
req.Header.Set("Tags", tags)
|
||||
}
|
||||
if token := channel.Params["token"]; token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+token)
|
||||
}
|
||||
resp, err := d.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
return fmt.Errorf("ntfy delivery failed: status %d", resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
package alerts
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/analysis"
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/store"
|
||||
)
|
||||
|
||||
func TestNtfyDelivery(t *testing.T) {
|
||||
var gotAuth string
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/alerts" {
|
||||
t.Fatalf("unexpected path %s", r.URL.Path)
|
||||
}
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
d := NewDispatcher()
|
||||
_, err := d.Send(t.Context(), store.Rule{Name: "API", Severity: "warning"}, analysis.Finding{Summary: "failed"}, []store.AlertChannel{{ID: "c1", Name: "ntfy", Type: "ntfy", Enabled: true, Params: map[string]string{"server_url": ts.URL, "topic": "alerts", "token": "secret"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if gotAuth != "Bearer secret" {
|
||||
t.Fatalf("missing auth header: %q", gotAuth)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
package analysis
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/loki"
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/store"
|
||||
)
|
||||
|
||||
type Finding struct {
|
||||
Summary string `json:"summary"`
|
||||
RecommendedActions []string `json:"recommended_actions,omitempty"`
|
||||
Confidence string `json:"confidence"`
|
||||
}
|
||||
|
||||
type Analyzer interface {
|
||||
Analyze(ctx context.Context, rule store.Rule, matches []loki.Match) (Finding, error)
|
||||
}
|
||||
|
||||
type NoopAnalyzer struct{}
|
||||
|
||||
func (NoopAnalyzer) Analyze(ctx context.Context, rule store.Rule, matches []loki.Match) (Finding, error) {
|
||||
_ = ctx
|
||||
return Finding{Summary: summarize(rule, matches), Confidence: "rule-only"}, nil
|
||||
}
|
||||
|
||||
type OllamaAnalyzer struct {
|
||||
URL string
|
||||
Model string
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
func NewOllamaAnalyzer(url, model string) *OllamaAnalyzer {
|
||||
return &OllamaAnalyzer{URL: strings.TrimRight(url, "/"), Model: model, http: &http.Client{Timeout: 60 * time.Second}}
|
||||
}
|
||||
|
||||
func (a *OllamaAnalyzer) Analyze(ctx context.Context, rule store.Rule, matches []loki.Match) (Finding, error) {
|
||||
if a.URL == "" || a.Model == "" {
|
||||
return NoopAnalyzer{}.Analyze(ctx, rule, matches)
|
||||
}
|
||||
prompt := buildPrompt(rule, matches)
|
||||
body, _ := json.Marshal(map[string]any{"model": a.Model, "prompt": prompt, "stream": false})
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, a.URL+"/api/generate", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return Finding{}, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := a.http.Do(req)
|
||||
if err != nil {
|
||||
return Finding{}, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
return Finding{}, fmt.Errorf("ollama analysis failed: status %d", resp.StatusCode)
|
||||
}
|
||||
var raw struct {
|
||||
Response string `json:"response"`
|
||||
}
|
||||
if err := json.NewDecoder(resp.Body).Decode(&raw); err != nil {
|
||||
return Finding{}, err
|
||||
}
|
||||
summary := strings.TrimSpace(raw.Response)
|
||||
if summary == "" {
|
||||
return NoopAnalyzer{}.Analyze(ctx, rule, matches)
|
||||
}
|
||||
if len(summary) > 500 {
|
||||
summary = summary[:500] + "..."
|
||||
}
|
||||
return Finding{Summary: summary, Confidence: "ollama-local"}, nil
|
||||
}
|
||||
|
||||
func buildPrompt(rule store.Rule, matches []loki.Match) string {
|
||||
var b strings.Builder
|
||||
b.WriteString("You are analyzing private application logs locally. Summarize likely cause and operator next steps in 4 concise sentences or fewer. Do not suggest destructive actions.\n")
|
||||
b.WriteString("Rule: ")
|
||||
b.WriteString(rule.Name)
|
||||
b.WriteString("\nSeverity: ")
|
||||
b.WriteString(rule.Severity)
|
||||
b.WriteString("\nSanitized samples:\n")
|
||||
limit := len(matches)
|
||||
if limit > 5 {
|
||||
limit = 5
|
||||
}
|
||||
for i := 0; i < limit; i++ {
|
||||
line := sanitizeLine(matches[i].Line)
|
||||
line = redactSecrets(line)
|
||||
if len(line) > 300 {
|
||||
line = line[:300] + "..."
|
||||
}
|
||||
b.WriteString("- ")
|
||||
b.WriteString(line)
|
||||
b.WriteByte('\n')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func summarize(rule store.Rule, matches []loki.Match) string {
|
||||
if len(matches) == 0 {
|
||||
return "No matching log entries."
|
||||
}
|
||||
line := redactSecrets(sanitizeLine(matches[0].Line))
|
||||
if len(line) > 180 {
|
||||
line = line[:180] + "..."
|
||||
}
|
||||
return rule.Name + " matched " + plural(len(matches), "log entry", "log entries") + "; first match: " + line
|
||||
}
|
||||
|
||||
func sanitizeLine(line string) string {
|
||||
line = strings.ReplaceAll(line, "\n", " ")
|
||||
line = strings.ReplaceAll(line, "\r", " ")
|
||||
return strings.TrimSpace(line)
|
||||
}
|
||||
|
||||
func redactSecrets(line string) string {
|
||||
fields := strings.Fields(line)
|
||||
for i, field := range fields {
|
||||
lower := strings.ToLower(field)
|
||||
if strings.Contains(lower, "token=") || strings.Contains(lower, "password=") || strings.Contains(lower, "secret=") || strings.Contains(lower, "apikey=") || strings.Contains(lower, "api_key=") || strings.HasPrefix(lower, "bearer") {
|
||||
fields[i] = redactField(field)
|
||||
}
|
||||
}
|
||||
return strings.Join(fields, " ")
|
||||
}
|
||||
|
||||
func redactField(field string) string {
|
||||
idx := strings.IndexAny(field, "=:")
|
||||
if idx < 0 {
|
||||
return "[REDACTED]"
|
||||
}
|
||||
prefix := strings.TrimRightFunc(field[:idx], func(r rune) bool { return unicode.IsSpace(r) })
|
||||
return prefix + "=[REDACTED]"
|
||||
}
|
||||
|
||||
func plural(count int, singular, plural string) string {
|
||||
if count == 1 {
|
||||
return "1 " + singular
|
||||
}
|
||||
return strconvItoa(count) + " " + plural
|
||||
}
|
||||
|
||||
func strconvItoa(i int) string {
|
||||
if i == 0 {
|
||||
return "0"
|
||||
}
|
||||
var b [20]byte
|
||||
bp := len(b)
|
||||
for i > 0 {
|
||||
bp--
|
||||
b[bp] = byte('0' + i%10)
|
||||
i /= 10
|
||||
}
|
||||
return string(b[bp:])
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
package analysis
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSanitizeLine(t *testing.T) {
|
||||
got := sanitizeLine(" error\nwith\rspaces ")
|
||||
if got != "error with spaces" {
|
||||
t.Fatalf("got %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"os"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Addr string
|
||||
DataPath string
|
||||
PollInterval time.Duration
|
||||
LokiURL string
|
||||
LokiTenantID string
|
||||
LokiUsername string
|
||||
LokiPassword string
|
||||
DryRunActions bool
|
||||
AnalysisProvider string
|
||||
AnalysisEndpoint string
|
||||
AnalysisToken string
|
||||
OllamaURL string
|
||||
OllamaModel string
|
||||
AuthUsername string
|
||||
AuthPassword string
|
||||
}
|
||||
|
||||
func Load() Config {
|
||||
return Config{
|
||||
Addr: env("LOG_GUARDIAN_ADDR", ":8080"),
|
||||
DataPath: env("LOG_GUARDIAN_DATA_PATH", "/data/config.json"),
|
||||
PollInterval: durationEnv("LOG_GUARDIAN_POLL_INTERVAL", 1*time.Minute),
|
||||
LokiURL: trimRightSlash(env("LOKI_URL", "http://loki:3100")),
|
||||
LokiTenantID: env("LOKI_TENANT_ID", ""),
|
||||
LokiUsername: env("LOKI_USERNAME", ""),
|
||||
LokiPassword: env("LOKI_PASSWORD", ""),
|
||||
DryRunActions: boolEnv("LOG_GUARDIAN_DRY_RUN_ACTIONS", true),
|
||||
AnalysisProvider: env("LOG_GUARDIAN_ANALYSIS_PROVIDER", "none"),
|
||||
AnalysisEndpoint: env("LOG_GUARDIAN_ANALYSIS_ENDPOINT", ""),
|
||||
AnalysisToken: env("LOG_GUARDIAN_ANALYSIS_TOKEN", ""),
|
||||
OllamaURL: trimRightSlash(env("LOG_GUARDIAN_OLLAMA_URL", "http://ollama:11434")),
|
||||
OllamaModel: env("LOG_GUARDIAN_OLLAMA_MODEL", "llama3.1"),
|
||||
AuthUsername: env("LOG_GUARDIAN_AUTH_USERNAME", ""),
|
||||
AuthPassword: env("LOG_GUARDIAN_AUTH_PASSWORD", ""),
|
||||
}
|
||||
}
|
||||
|
||||
func env(key, fallback string) string {
|
||||
if value := os.Getenv(key); value != "" {
|
||||
return value
|
||||
}
|
||||
return fallback
|
||||
}
|
||||
|
||||
func boolEnv(key string, fallback bool) bool {
|
||||
value := os.Getenv(key)
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := strconv.ParseBool(value)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func durationEnv(key string, fallback time.Duration) time.Duration {
|
||||
value := os.Getenv(key)
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
parsed, err := time.ParseDuration(value)
|
||||
if err != nil {
|
||||
return fallback
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
func trimRightSlash(value string) string {
|
||||
for len(value) > 1 && value[len(value)-1] == '/' {
|
||||
value = value[:len(value)-1]
|
||||
}
|
||||
return value
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
package loki
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Client struct {
|
||||
baseURL string
|
||||
tenantID string
|
||||
username string
|
||||
password string
|
||||
http *http.Client
|
||||
}
|
||||
|
||||
func New(baseURL, tenantID, username, password string) *Client {
|
||||
return &Client{baseURL: strings.TrimRight(baseURL, "/"), tenantID: tenantID, username: username, password: password, http: &http.Client{Timeout: 20 * time.Second}}
|
||||
}
|
||||
|
||||
type Match struct {
|
||||
Timestamp time.Time `json:"timestamp"`
|
||||
Line string `json:"line"`
|
||||
}
|
||||
|
||||
type QueryResult struct {
|
||||
Count int `json:"count"`
|
||||
Matches []Match `json:"matches"`
|
||||
}
|
||||
|
||||
func (c *Client) Ready(ctx context.Context) error {
|
||||
req, err := c.request(ctx, http.MethodGet, "/ready", nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
return fmt.Errorf("loki ready failed: status %d", resp.StatusCode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) Labels(ctx context.Context) ([]string, error) {
|
||||
var raw labelsResponse
|
||||
if err := c.getJSON(ctx, "/loki/api/v1/labels", nil, &raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return raw.Data, nil
|
||||
}
|
||||
|
||||
func (c *Client) LabelValues(ctx context.Context, name string) ([]string, error) {
|
||||
if name == "" || strings.Contains(name, "/") {
|
||||
return nil, fmt.Errorf("invalid label name")
|
||||
}
|
||||
var raw labelsResponse
|
||||
if err := c.getJSON(ctx, "/loki/api/v1/label/"+url.PathEscape(name)+"/values", nil, &raw); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return raw.Data, nil
|
||||
}
|
||||
|
||||
func (c *Client) QueryRange(ctx context.Context, logql string, since time.Duration, limit int) (QueryResult, error) {
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
values := url.Values{}
|
||||
values.Set("query", logql)
|
||||
values.Set("start", strconv.FormatInt(time.Now().Add(-since).UnixNano(), 10))
|
||||
values.Set("end", strconv.FormatInt(time.Now().UnixNano(), 10))
|
||||
values.Set("limit", strconv.Itoa(limit))
|
||||
var raw response
|
||||
if err := c.getJSON(ctx, "/loki/api/v1/query_range", values, &raw); err != nil {
|
||||
return QueryResult{}, err
|
||||
}
|
||||
var result QueryResult
|
||||
for _, stream := range raw.Data.Result {
|
||||
for _, pair := range stream.Values {
|
||||
if len(pair) != 2 {
|
||||
continue
|
||||
}
|
||||
ns, _ := strconv.ParseInt(pair[0], 10, 64)
|
||||
result.Matches = append(result.Matches, Match{Timestamp: time.Unix(0, ns), Line: pair[1]})
|
||||
}
|
||||
}
|
||||
result.Count = len(result.Matches)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *Client) getJSON(ctx context.Context, path string, values url.Values, out any) error {
|
||||
req, err := c.request(ctx, http.MethodGet, path, values)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode < 200 || resp.StatusCode > 299 {
|
||||
return fmt.Errorf("loki request failed: status %d", resp.StatusCode)
|
||||
}
|
||||
return json.NewDecoder(resp.Body).Decode(out)
|
||||
}
|
||||
|
||||
func (c *Client) request(ctx context.Context, method, path string, values url.Values) (*http.Request, error) {
|
||||
if c.baseURL == "" {
|
||||
return nil, fmt.Errorf("LOKI_URL is not configured")
|
||||
}
|
||||
target := c.baseURL + path
|
||||
if len(values) > 0 {
|
||||
target += "?" + values.Encode()
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, target, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.tenantID != "" {
|
||||
req.Header.Set("X-Scope-OrgID", c.tenantID)
|
||||
}
|
||||
if c.username != "" || c.password != "" {
|
||||
req.SetBasicAuth(c.username, c.password)
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
type labelsResponse struct {
|
||||
Status string `json:"status"`
|
||||
Data []string `json:"data"`
|
||||
}
|
||||
|
||||
type response struct {
|
||||
Status string `json:"status"`
|
||||
Data struct {
|
||||
Result []struct {
|
||||
Stream map[string]string `json:"stream"`
|
||||
Values [][]string `json:"values"`
|
||||
} `json:"result"`
|
||||
} `json:"data"`
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package loki
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestLabelsValuesAndQueryRange(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/loki/api/v1/labels":
|
||||
_, _ = w.Write([]byte(`{"status":"success","data":["service","container"]}`))
|
||||
case "/loki/api/v1/label/service/values":
|
||||
_, _ = w.Write([]byte(`{"status":"success","data":["api"]}`))
|
||||
case "/loki/api/v1/query_range":
|
||||
if !strings.Contains(r.URL.Query().Get("query"), "api") {
|
||||
t.Fatalf("query not passed through: %s", r.URL.RawQuery)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"status":"success","data":{"result":[{"stream":{"service":"api"},"values":[["1700000000000000000","error token=secret"]]}]}}`))
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
client := New(ts.URL, "", "", "")
|
||||
labels, err := client.Labels(t.Context())
|
||||
if err != nil || len(labels) != 2 {
|
||||
t.Fatalf("labels=%v err=%v", labels, err)
|
||||
}
|
||||
values, err := client.LabelValues(t.Context(), "service")
|
||||
if err != nil || values[0] != "api" {
|
||||
t.Fatalf("values=%v err=%v", values, err)
|
||||
}
|
||||
result, err := client.QueryRange(t.Context(), `{service="api"}`, time.Minute, 10)
|
||||
if err != nil || result.Count != 1 {
|
||||
t.Fatalf("result=%#v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,152 @@
|
||||
package rules
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/actions"
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/alerts"
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/analysis"
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/loki"
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/store"
|
||||
)
|
||||
|
||||
type Store interface {
|
||||
Load() (store.Data, error)
|
||||
UpdateRule(id string, fn func(*store.Rule)) error
|
||||
AddIncident(incident store.Incident) error
|
||||
}
|
||||
|
||||
type Engine struct {
|
||||
store Store
|
||||
loki *loki.Client
|
||||
analyzer analysis.Analyzer
|
||||
actions *actions.Runner
|
||||
dispatcher *alerts.Dispatcher
|
||||
}
|
||||
|
||||
func NewEngine(s Store, lokiClient *loki.Client, analyzer analysis.Analyzer, actionRunner *actions.Runner, dispatcher *alerts.Dispatcher) *Engine {
|
||||
return &Engine{store: s, loki: lokiClient, analyzer: analyzer, actions: actionRunner, dispatcher: dispatcher}
|
||||
}
|
||||
|
||||
func (e *Engine) Start(ctx context.Context, interval time.Duration) {
|
||||
if interval <= 0 {
|
||||
interval = time.Minute
|
||||
}
|
||||
ticker := time.NewTicker(interval)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
e.CheckAll(ctx)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-ticker.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) CheckAll(ctx context.Context) {
|
||||
data, err := e.store.Load()
|
||||
if err != nil {
|
||||
log.Printf("load rules failed: %v", err)
|
||||
return
|
||||
}
|
||||
channels := map[string]store.AlertChannel{}
|
||||
for _, channel := range data.AlertChannels {
|
||||
channels[channel.ID] = channel
|
||||
}
|
||||
for _, rule := range data.Rules {
|
||||
if !rule.Enabled {
|
||||
continue
|
||||
}
|
||||
if err := e.checkRule(ctx, rule, channels); err != nil {
|
||||
log.Printf("rule %q check failed: %v", rule.Name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) checkRule(ctx context.Context, rule store.Rule, channels map[string]store.AlertChannel) error {
|
||||
window, err := time.ParseDuration(rule.Window)
|
||||
if err != nil {
|
||||
window = 5 * time.Minute
|
||||
}
|
||||
result, err := e.loki.QueryRange(ctx, rule.LogQL, window, 200)
|
||||
if err != nil {
|
||||
e.record(rule.ID, 0, err)
|
||||
return err
|
||||
}
|
||||
if result.Count < rule.Threshold {
|
||||
return e.record(rule.ID, result.Count, nil)
|
||||
}
|
||||
now := time.Now().UTC()
|
||||
cooldown, err := time.ParseDuration(rule.Cooldown)
|
||||
if err != nil || cooldown <= 0 {
|
||||
cooldown = time.Hour
|
||||
}
|
||||
if !rule.LastAlertedAt.IsZero() && now.Sub(rule.LastAlertedAt) < cooldown {
|
||||
return e.store.UpdateRule(rule.ID, func(r *store.Rule) {
|
||||
r.LastCheckedAt = now
|
||||
r.LastMatchedAt = now
|
||||
r.LastMatchCount = result.Count
|
||||
r.SuppressedCount++
|
||||
r.LastError = ""
|
||||
})
|
||||
}
|
||||
finding, err := e.analyzer.Analyze(ctx, rule, result.Matches)
|
||||
if err != nil {
|
||||
e.record(rule.ID, result.Count, err)
|
||||
return err
|
||||
}
|
||||
selected := make([]store.AlertChannel, 0, len(rule.AlertChannels))
|
||||
for _, id := range rule.AlertChannels {
|
||||
if channel, ok := channels[id]; ok {
|
||||
selected = append(selected, channel)
|
||||
}
|
||||
}
|
||||
alertResults, alertErr := e.dispatcher.Send(ctx, rule, finding, selected)
|
||||
var alertEvidence []string
|
||||
for _, result := range alertResults {
|
||||
if result.ChannelID == "" {
|
||||
alertEvidence = append(alertEvidence, result.Detail)
|
||||
} else {
|
||||
alertEvidence = append(alertEvidence, result.ChannelID+": "+result.Detail)
|
||||
}
|
||||
}
|
||||
var actionEvidence []string
|
||||
for _, action := range rule.Actions {
|
||||
result, err := e.actions.Run(ctx, rule, action)
|
||||
if err != nil {
|
||||
log.Printf("action %q for rule %q failed: %v", action.Type, rule.Name, err)
|
||||
actionEvidence = append(actionEvidence, action.Type+": "+err.Error())
|
||||
continue
|
||||
}
|
||||
actionEvidence = append(actionEvidence, result.Action+": "+result.Detail)
|
||||
}
|
||||
incident := store.Incident{RuleID: rule.ID, RuleName: rule.Name, Severity: rule.Severity, Count: result.Count, Summary: finding.Summary, AlertResults: alertEvidence, RemediationEvidence: actionEvidence, CreatedAt: now}
|
||||
if err := e.store.AddIncident(incident); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := e.store.UpdateRule(rule.ID, func(r *store.Rule) {
|
||||
r.LastCheckedAt = now
|
||||
r.LastMatchedAt = now
|
||||
r.LastAlertedAt = now
|
||||
r.LastMatchCount = result.Count
|
||||
r.LastError = ""
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
return alertErr
|
||||
}
|
||||
|
||||
func (e *Engine) record(id string, count int, err error) error {
|
||||
return e.store.UpdateRule(id, func(r *store.Rule) {
|
||||
r.LastCheckedAt = time.Now().UTC()
|
||||
r.LastMatchCount = count
|
||||
if err != nil {
|
||||
r.LastError = err.Error()
|
||||
} else {
|
||||
r.LastError = ""
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package rules
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/actions"
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/alerts"
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/analysis"
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/loki"
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/store"
|
||||
)
|
||||
|
||||
func TestCooldownSuppressesDuplicateIncident(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_, _ = w.Write([]byte(`{"status":"success","data":{"result":[{"stream":{},"values":[["1700000000000000000","error"]]}]}}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
s := store.NewFileStore(t.TempDir() + "/config.json")
|
||||
if err := s.UpsertRule(store.Rule{Name: "Errors", Enabled: true, LogQL: `{service="api"}`, Threshold: 1, Window: "5m", Cooldown: "1h", Actions: []store.Action{{Type: "record_recommendation", Enabled: true}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
engine := NewEngine(s, loki.New(ts.URL, "", "", ""), analysis.NoopAnalyzer{}, actions.NewRunner(true), alerts.NewDispatcher())
|
||||
engine.CheckAll(t.Context())
|
||||
engine.CheckAll(t.Context())
|
||||
data, err := s.Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(data.Incidents) != 1 {
|
||||
t.Fatalf("expected one incident, got %d", len(data.Incidents))
|
||||
}
|
||||
if data.Rules[0].SuppressedCount != 1 {
|
||||
t.Fatalf("expected suppressed count 1, got %d", data.Rules[0].SuppressedCount)
|
||||
}
|
||||
if time.Since(data.Rules[0].LastAlertedAt) > time.Minute {
|
||||
t.Fatalf("last alerted not set: %s", data.Rules[0].LastAlertedAt)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"crypto/subtle"
|
||||
"encoding/json"
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
func writeJSON(w http.ResponseWriter, v any) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(v)
|
||||
}
|
||||
|
||||
func requestLog(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
next.ServeHTTP(w, r)
|
||||
log.Printf("%s %s %s", r.Method, r.URL.Path, time.Since(start).Round(time.Millisecond))
|
||||
})
|
||||
}
|
||||
|
||||
func securityHeaders(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("X-Content-Type-Options", "nosniff")
|
||||
w.Header().Set("Referrer-Policy", "no-referrer")
|
||||
w.Header().Set("Content-Security-Policy", "default-src 'self'; style-src 'self' 'unsafe-inline'; form-action 'self'")
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func basicAuth(next http.Handler, username, password string) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path == "/healthz" {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
if username == "" || password == "" {
|
||||
http.Error(w, "basic auth is not configured", http.StatusServiceUnavailable)
|
||||
return
|
||||
}
|
||||
gotUser, gotPass, ok := r.BasicAuth()
|
||||
if !ok || subtle.ConstantTimeCompare([]byte(gotUser), []byte(username)) != 1 || subtle.ConstantTimeCompare([]byte(gotPass), []byte(password)) != 1 {
|
||||
w.Header().Set("WWW-Authenticate", `Basic realm="log-guardian"`)
|
||||
http.Error(w, "unauthorized", http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBasicAuthAllowsHealthzWithoutCredentials(t *testing.T) {
|
||||
h := basicAuth(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) }), "admin", "secret")
|
||||
req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
h.ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusNoContent {
|
||||
t.Fatalf("got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBasicAuthProtectsUI(t *testing.T) {
|
||||
h := basicAuth(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusNoContent) }), "admin", "secret")
|
||||
req := httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
rr := httptest.NewRecorder()
|
||||
h.ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusUnauthorized {
|
||||
t.Fatalf("got %d", rr.Code)
|
||||
}
|
||||
req = httptest.NewRequest(http.MethodGet, "/", nil)
|
||||
req.SetBasicAuth("admin", "secret")
|
||||
rr = httptest.NewRecorder()
|
||||
h.ServeHTTP(rr, req)
|
||||
if rr.Code != http.StatusNoContent {
|
||||
t.Fatalf("got %d", rr.Code)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"html/template"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/alerts"
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/loki"
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/rules"
|
||||
"gitea.wayfinderak.com/wayfinderak/log-guardian/internal/store"
|
||||
)
|
||||
|
||||
type Server struct {
|
||||
store *store.FileStore
|
||||
engine *rules.Engine
|
||||
loki *loki.Client
|
||||
dispatcher *alerts.Dispatcher
|
||||
authUsername string
|
||||
authPassword string
|
||||
}
|
||||
|
||||
func New(s *store.FileStore, engine *rules.Engine, lokiClient *loki.Client, dispatcher *alerts.Dispatcher, authUsername, authPassword string) *Server {
|
||||
return &Server{store: s, engine: engine, loki: lokiClient, dispatcher: dispatcher, authUsername: authUsername, authPassword: authPassword}
|
||||
}
|
||||
|
||||
func (s *Server) Routes() http.Handler {
|
||||
mux := http.NewServeMux()
|
||||
mux.HandleFunc("GET /healthz", s.health)
|
||||
mux.HandleFunc("GET /", s.index)
|
||||
mux.HandleFunc("POST /rules", s.saveRule)
|
||||
mux.HandleFunc("POST /check", s.checkNow)
|
||||
mux.HandleFunc("POST /channels", s.saveChannel)
|
||||
mux.HandleFunc("POST /channels/delete", s.deleteChannel)
|
||||
mux.HandleFunc("POST /channels/test", s.testChannel)
|
||||
mux.HandleFunc("GET /loki/labels", s.lokiLabels)
|
||||
mux.HandleFunc("GET /loki/label-values", s.lokiLabelValues)
|
||||
mux.HandleFunc("POST /loki/query-test", s.lokiQueryTest)
|
||||
mux.Handle("GET /static/", http.StripPrefix("/static/", http.FileServer(http.Dir("web/static"))))
|
||||
return securityHeaders(requestLog(basicAuth(mux, s.authUsername, s.authPassword)))
|
||||
}
|
||||
|
||||
func (s *Server) health(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, map[string]string{"status": "ok", "service": "log-guardian"})
|
||||
}
|
||||
|
||||
func (s *Server) index(w http.ResponseWriter, r *http.Request) {
|
||||
data, err := s.store.Load()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
render(w, data)
|
||||
}
|
||||
|
||||
func (s *Server) saveRule(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
threshold, _ := strconv.Atoi(r.FormValue("threshold"))
|
||||
rule := store.Rule{ID: strings.TrimSpace(r.FormValue("id")), Name: strings.TrimSpace(r.FormValue("name")), Enabled: r.FormValue("enabled") == "on", LogQL: strings.TrimSpace(r.FormValue("logql")), Severity: strings.TrimSpace(r.FormValue("severity")), Threshold: threshold, Window: strings.TrimSpace(r.FormValue("window")), Cooldown: strings.TrimSpace(r.FormValue("cooldown")), AnalysisEnabled: r.FormValue("analysis_enabled") == "on", AlertChannels: r.Form["alert_channels"]}
|
||||
if rule.Name == "" || rule.LogQL == "" {
|
||||
http.Error(w, "name and LogQL are required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if rule.Severity == "" {
|
||||
rule.Severity = "warning"
|
||||
}
|
||||
if err := s.store.UpsertRule(rule); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (s *Server) saveChannel(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
channel := store.AlertChannel{ID: strings.TrimSpace(r.FormValue("id")), Name: strings.TrimSpace(r.FormValue("name")), Type: strings.TrimSpace(r.FormValue("type")), Enabled: r.FormValue("enabled") == "on", Params: map[string]string{}}
|
||||
for _, key := range []string{"server_url", "topic", "token", "priority", "tags", "url", "api_url", "email"} {
|
||||
if value := strings.TrimSpace(r.FormValue(key)); value != "" {
|
||||
channel.Params[key] = value
|
||||
}
|
||||
}
|
||||
if channel.Name == "" || channel.Type == "" {
|
||||
http.Error(w, "name and type are required", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := s.store.UpsertAlertChannel(channel); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (s *Server) deleteChannel(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
if err := s.store.DeleteAlertChannel(r.FormValue("id")); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (s *Server) testChannel(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
data, err := s.store.Load()
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
for _, channel := range data.AlertChannels {
|
||||
if channel.ID == r.FormValue("id") {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 15*time.Second)
|
||||
defer cancel()
|
||||
if err := s.dispatcher.Test(ctx, channel); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
writeJSON(w, map[string]string{"status": "sent"})
|
||||
return
|
||||
}
|
||||
}
|
||||
http.Error(w, "channel not found", http.StatusNotFound)
|
||||
}
|
||||
|
||||
func (s *Server) checkNow(w http.ResponseWriter, r *http.Request) {
|
||||
s.engine.CheckAll(context.Background())
|
||||
http.Redirect(w, r, "/", http.StatusSeeOther)
|
||||
}
|
||||
|
||||
func (s *Server) lokiLabels(w http.ResponseWriter, r *http.Request) {
|
||||
labels, err := s.loki.Labels(r.Context())
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
writeJSON(w, map[string]any{"labels": labels})
|
||||
}
|
||||
|
||||
func (s *Server) lokiLabelValues(w http.ResponseWriter, r *http.Request) {
|
||||
values, err := s.loki.LabelValues(r.Context(), r.URL.Query().Get("name"))
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
writeJSON(w, map[string]any{"values": values})
|
||||
}
|
||||
|
||||
func (s *Server) lokiQueryTest(w http.ResponseWriter, r *http.Request) {
|
||||
if err := r.ParseForm(); err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
window, err := time.ParseDuration(r.FormValue("window"))
|
||||
if err != nil {
|
||||
window = 5 * time.Minute
|
||||
}
|
||||
result, err := s.loki.QueryRange(r.Context(), strings.TrimSpace(r.FormValue("logql")), window, 20)
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadGateway)
|
||||
return
|
||||
}
|
||||
if len(result.Matches) > 5 {
|
||||
result.Matches = result.Matches[:5]
|
||||
}
|
||||
writeJSON(w, result)
|
||||
}
|
||||
|
||||
var tmpl = template.Must(template.New("index").Funcs(template.FuncMap{"secretState": secretState}).Parse(`<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1"><title>Log Guardian</title><link rel="stylesheet" href="/static/app.css"></head>
|
||||
<body><main class="shell"><header><h1>Log Guardian</h1><p>Loki-backed log monitoring, alerting, and guarded remediation.</p></header>
|
||||
<section class="card"><h2>Loki diagnostics</h2><p>Use authenticated endpoints: <code>/loki/labels</code>, <code>/loki/label-values?name=service</code>.</p><form method="post" action="/loki/query-test" class="grid"><label class="wide">Test LogQL <input name="logql" placeholder='{service="api"} |= "error"'></label><label>Window <input name="window" value="5m"></label><button type="submit">Return JSON sample</button></form></section>
|
||||
<section class="card"><h2>Add or replace rule</h2><form method="post" action="/rules" class="grid"><label>Name <input name="name" required placeholder="API errors"></label><label>ID <input name="id" placeholder="optional stable id"></label><label>Severity <input name="severity" value="warning"></label><label>Window <input name="window" value="5m"></label><label>Cooldown <input name="cooldown" value="1h"></label><label>Threshold <input name="threshold" type="number" min="1" value="1"></label><label class="wide">LogQL <input name="logql" required placeholder='{service="api"} |= "error"'></label><fieldset class="wide"><legend>Alert channels</legend>{{range .AlertChannels}}<label class="check"><input type="checkbox" name="alert_channels" value="{{.ID}}"> {{.Name}} ({{.Type}})</label>{{else}}<span class="muted">No channels configured.</span>{{end}}</fieldset><label class="check"><input type="checkbox" name="enabled" checked> Enabled</label><label class="check"><input type="checkbox" name="analysis_enabled"> AI analysis</label><button type="submit">Save rule</button></form></section>
|
||||
<section class="card"><h2>Alert channels</h2><form method="post" action="/channels" class="grid"><label>Name <input name="name" required></label><label>ID <input name="id" placeholder="optional stable id"></label><label>Type <select name="type"><option value="ntfy">ntfy</option><option value="gotify">gotify (scaffold)</option><option value="generic_webhook">generic webhook (scaffold)</option><option value="smtp">smtp (scaffold)</option></select></label><label class="check"><input type="checkbox" name="enabled" checked> Enabled</label><label>Server URL <input name="server_url" placeholder="https://ntfy.example.com"></label><label>Topic <input name="topic"></label><label>Token <input name="token" type="password" placeholder="leave blank to preserve"></label><label>Priority <input name="priority" placeholder="default, high, urgent"></label><label>Tags <input name="tags" placeholder="warning,rotating_light"></label><button type="submit">Save channel</button></form><table><thead><tr><th>Name</th><th>Type</th><th>Status</th><th>Secret</th><th>Actions</th></tr></thead><tbody>{{range .AlertChannels}}<tr><td>{{.Name}}</td><td>{{.Type}}</td><td>{{if .Enabled}}enabled{{else}}disabled{{end}}</td><td>{{secretState .Params}}</td><td><form method="post" action="/channels/test" class="inline"><input type="hidden" name="id" value="{{.ID}}"><button>Test</button></form><form method="post" action="/channels/delete" class="inline"><input type="hidden" name="id" value="{{.ID}}"><button>Delete</button></form></td></tr>{{else}}<tr><td colspan="5" class="muted">No alert channels configured.</td></tr>{{end}}</tbody></table></section>
|
||||
<section class="card"><div class="row"><h2>Rules</h2><form method="post" action="/check"><button type="submit">Check now</button></form></div><table><thead><tr><th>Name</th><th>LogQL</th><th>Cooldown</th><th>Last count</th><th>Suppressed</th><th>Status</th></tr></thead><tbody>{{range .Rules}}<tr><td>{{.Name}}{{if not .Enabled}} <span class="muted">disabled</span>{{end}}</td><td><code>{{.LogQL}}</code></td><td>{{.Cooldown}}</td><td>{{.LastMatchCount}}</td><td>{{.SuppressedCount}}</td><td>{{if .LastError}}<span class="error">{{.LastError}}</span>{{else}}checked {{.LastCheckedAt}}{{end}}</td></tr>{{else}}<tr><td colspan="6" class="muted">No rules configured yet.</td></tr>{{end}}</tbody></table></section>
|
||||
<section class="card"><h2>Recent incidents</h2><table><thead><tr><th>Time</th><th>Rule</th><th>Severity</th><th>Summary</th><th>Evidence</th></tr></thead><tbody>{{range .Incidents}}<tr><td>{{.CreatedAt}}</td><td>{{.RuleName}}</td><td>{{.Severity}}</td><td>{{.Summary}}</td><td>{{range .AlertResults}}<div>{{.}}</div>{{end}}{{range .RemediationEvidence}}<div>{{.}}</div>{{end}}</td></tr>{{else}}<tr><td colspan="5" class="muted">No incidents recorded.</td></tr>{{end}}</tbody></table></section>
|
||||
</main></body></html>`))
|
||||
|
||||
func secretState(params map[string]string) string {
|
||||
if params["token"] != "" || params["password"] != "" || params["secret"] != "" {
|
||||
return "configured"
|
||||
}
|
||||
return "not configured"
|
||||
}
|
||||
|
||||
func render(w http.ResponseWriter, data store.Data) {
|
||||
w.Header().Set("Content-Type", "text/html; charset=utf-8")
|
||||
_ = tmpl.Execute(w, data)
|
||||
}
|
||||
@@ -0,0 +1,260 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Rule struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Enabled bool `json:"enabled"`
|
||||
LogQL string `json:"logql"`
|
||||
Severity string `json:"severity"`
|
||||
Threshold int `json:"threshold"`
|
||||
Window string `json:"window"`
|
||||
Cooldown string `json:"cooldown,omitempty"`
|
||||
AlertChannels []string `json:"alert_channels,omitempty"`
|
||||
AnalysisEnabled bool `json:"analysis_enabled"`
|
||||
Actions []Action `json:"actions,omitempty"`
|
||||
LastCheckedAt time.Time `json:"last_checked_at,omitempty"`
|
||||
LastMatchedAt time.Time `json:"last_matched_at,omitempty"`
|
||||
LastAlertedAt time.Time `json:"last_alerted_at,omitempty"`
|
||||
LastMatchCount int `json:"last_match_count,omitempty"`
|
||||
SuppressedCount int `json:"suppressed_count,omitempty"`
|
||||
LastError string `json:"last_error,omitempty"`
|
||||
}
|
||||
|
||||
type Action struct {
|
||||
Type string `json:"type"`
|
||||
Enabled bool `json:"enabled"`
|
||||
DryRun *bool `json:"dry_run,omitempty"`
|
||||
Params map[string]string `json:"params,omitempty"`
|
||||
}
|
||||
|
||||
type AlertChannel struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Enabled bool `json:"enabled"`
|
||||
Params map[string]string `json:"params,omitempty"`
|
||||
}
|
||||
|
||||
type Incident struct {
|
||||
ID string `json:"id"`
|
||||
RuleID string `json:"rule_id"`
|
||||
RuleName string `json:"rule_name"`
|
||||
Severity string `json:"severity"`
|
||||
Count int `json:"count"`
|
||||
Summary string `json:"summary"`
|
||||
AlertResults []string `json:"alert_results,omitempty"`
|
||||
RemediationEvidence []string `json:"remediation_evidence,omitempty"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
}
|
||||
|
||||
type Data struct {
|
||||
Rules []Rule `json:"rules"`
|
||||
AlertChannels []AlertChannel `json:"alert_channels"`
|
||||
Incidents []Incident `json:"incidents"`
|
||||
}
|
||||
|
||||
type FileStore struct {
|
||||
path string
|
||||
mu sync.Mutex
|
||||
}
|
||||
|
||||
func NewFileStore(path string) *FileStore { return &FileStore{path: path} }
|
||||
|
||||
func (s *FileStore) Load() (Data, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.loadLocked()
|
||||
}
|
||||
|
||||
func (s *FileStore) Save(data Data) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
return s.saveLocked(data)
|
||||
}
|
||||
|
||||
func (s *FileStore) UpsertRule(rule Rule) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
data, err := s.loadLocked()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rule.ID == "" {
|
||||
rule.ID = slug(rule.Name)
|
||||
}
|
||||
if rule.ID == "" {
|
||||
return errors.New("rule id or name is required")
|
||||
}
|
||||
if rule.Threshold <= 0 {
|
||||
rule.Threshold = 1
|
||||
}
|
||||
if rule.Window == "" {
|
||||
rule.Window = "5m"
|
||||
}
|
||||
if rule.Cooldown == "" {
|
||||
rule.Cooldown = "1h"
|
||||
}
|
||||
for i := range data.Rules {
|
||||
if data.Rules[i].ID == rule.ID {
|
||||
rule.LastCheckedAt = data.Rules[i].LastCheckedAt
|
||||
rule.LastMatchedAt = data.Rules[i].LastMatchedAt
|
||||
rule.LastAlertedAt = data.Rules[i].LastAlertedAt
|
||||
rule.LastMatchCount = data.Rules[i].LastMatchCount
|
||||
rule.SuppressedCount = data.Rules[i].SuppressedCount
|
||||
rule.LastError = data.Rules[i].LastError
|
||||
data.Rules[i] = rule
|
||||
return s.saveLocked(data)
|
||||
}
|
||||
}
|
||||
data.Rules = append(data.Rules, rule)
|
||||
return s.saveLocked(data)
|
||||
}
|
||||
|
||||
func (s *FileStore) UpsertAlertChannel(channel AlertChannel) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
data, err := s.loadLocked()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if channel.ID == "" {
|
||||
channel.ID = slug(channel.Name)
|
||||
}
|
||||
if channel.ID == "" {
|
||||
return errors.New("channel id or name is required")
|
||||
}
|
||||
if channel.Params == nil {
|
||||
channel.Params = map[string]string{}
|
||||
}
|
||||
for i := range data.AlertChannels {
|
||||
if data.AlertChannels[i].ID == channel.ID {
|
||||
preserveSecrets(data.AlertChannels[i].Params, channel.Params)
|
||||
data.AlertChannels[i] = channel
|
||||
return s.saveLocked(data)
|
||||
}
|
||||
}
|
||||
data.AlertChannels = append(data.AlertChannels, channel)
|
||||
return s.saveLocked(data)
|
||||
}
|
||||
|
||||
func preserveSecrets(oldParams, newParams map[string]string) {
|
||||
for _, key := range []string{"token", "password", "secret", "authorization"} {
|
||||
if newParams[key] == "" && oldParams[key] != "" {
|
||||
newParams[key] = oldParams[key]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *FileStore) DeleteAlertChannel(id string) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
data, err := s.loadLocked()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
channels := data.AlertChannels[:0]
|
||||
for _, channel := range data.AlertChannels {
|
||||
if channel.ID != id {
|
||||
channels = append(channels, channel)
|
||||
}
|
||||
}
|
||||
data.AlertChannels = channels
|
||||
for i := range data.Rules {
|
||||
ids := data.Rules[i].AlertChannels[:0]
|
||||
for _, channelID := range data.Rules[i].AlertChannels {
|
||||
if channelID != id {
|
||||
ids = append(ids, channelID)
|
||||
}
|
||||
}
|
||||
data.Rules[i].AlertChannels = ids
|
||||
}
|
||||
return s.saveLocked(data)
|
||||
}
|
||||
|
||||
func (s *FileStore) UpdateRule(id string, fn func(*Rule)) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
data, err := s.loadLocked()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range data.Rules {
|
||||
if data.Rules[i].ID == id {
|
||||
fn(&data.Rules[i])
|
||||
return s.saveLocked(data)
|
||||
}
|
||||
}
|
||||
return errors.New("rule not found")
|
||||
}
|
||||
|
||||
func (s *FileStore) AddIncident(incident Incident) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
data, err := s.loadLocked()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if incident.ID == "" {
|
||||
incident.ID = slug(incident.RuleName) + "-" + incident.CreatedAt.Format("20060102150405")
|
||||
}
|
||||
data.Incidents = append([]Incident{incident}, data.Incidents...)
|
||||
if len(data.Incidents) > 200 {
|
||||
data.Incidents = data.Incidents[:200]
|
||||
}
|
||||
return s.saveLocked(data)
|
||||
}
|
||||
|
||||
func (s *FileStore) loadLocked() (Data, error) {
|
||||
var data Data
|
||||
contents, err := os.ReadFile(s.path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return data, nil
|
||||
}
|
||||
if err != nil {
|
||||
return data, err
|
||||
}
|
||||
if len(contents) == 0 {
|
||||
return data, nil
|
||||
}
|
||||
return data, json.Unmarshal(contents, &data)
|
||||
}
|
||||
|
||||
func (s *FileStore) saveLocked(data Data) error {
|
||||
if err := os.MkdirAll(filepath.Dir(s.path), 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
contents, err := json.MarshalIndent(data, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(s.path, append(contents, '\n'), 0o600)
|
||||
}
|
||||
|
||||
func slug(value string) string {
|
||||
value = strings.ToLower(strings.TrimSpace(value))
|
||||
var b strings.Builder
|
||||
lastDash := false
|
||||
for _, r := range value {
|
||||
ok := r >= 'a' && r <= 'z' || r >= '0' && r <= '9'
|
||||
if ok {
|
||||
b.WriteRune(r)
|
||||
lastDash = false
|
||||
continue
|
||||
}
|
||||
if !lastDash {
|
||||
b.WriteByte('-')
|
||||
lastDash = true
|
||||
}
|
||||
}
|
||||
return strings.Trim(b.String(), "-")
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
package store
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestUpsertRuleDefaults(t *testing.T) {
|
||||
s := NewFileStore(t.TempDir() + "/config.json")
|
||||
if err := s.UpsertRule(Rule{Name: "API Errors", LogQL: `{service="api"} |= "error"`}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := s.Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(data.Rules) != 1 {
|
||||
t.Fatalf("expected one rule, got %d", len(data.Rules))
|
||||
}
|
||||
rule := data.Rules[0]
|
||||
if rule.ID != "api-errors" || rule.Threshold != 1 || rule.Window != "5m" || rule.Cooldown != "1h" {
|
||||
t.Fatalf("unexpected defaults: %#v", rule)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUpsertAlertChannelPreservesSecret(t *testing.T) {
|
||||
s := NewFileStore(t.TempDir() + "/config.json")
|
||||
if err := s.UpsertAlertChannel(AlertChannel{ID: "ntfy", Name: "ntfy", Type: "ntfy", Enabled: true, Params: map[string]string{"token": "secret", "topic": "alerts"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.UpsertAlertChannel(AlertChannel{ID: "ntfy", Name: "ntfy", Type: "ntfy", Enabled: true, Params: map[string]string{"topic": "alerts2"}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
data, err := s.Load()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := data.AlertChannels[0].Params["token"]; got != "secret" {
|
||||
t.Fatalf("secret was not preserved: %q", got)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user