// FILE: src/internal/sink/http_client.go package sink import ( "context" "encoding/json" "fmt" "net/url" "sync" "sync/atomic" "time" "logwisp/src/internal/source" "github.com/lixenwraith/log" "github.com/valyala/fasthttp" ) // HTTPClientSink forwards log entries to a remote HTTP endpoint type HTTPClientSink struct { input chan source.LogEntry config HTTPClientConfig client *fasthttp.Client batch []source.LogEntry batchMu sync.Mutex done chan struct{} wg sync.WaitGroup startTime time.Time logger *log.Logger // Statistics totalProcessed atomic.Uint64 totalBatches atomic.Uint64 failedBatches atomic.Uint64 lastProcessed atomic.Value // time.Time lastBatchSent atomic.Value // time.Time activeConnections atomic.Int32 } // HTTPClientConfig holds HTTP client sink configuration type HTTPClientConfig struct { URL string BufferSize int BatchSize int BatchDelay time.Duration Timeout time.Duration Headers map[string]string // Retry configuration MaxRetries int RetryDelay time.Duration RetryBackoff float64 // Multiplier for exponential backoff // TLS configuration InsecureSkipVerify bool } // NewHTTPClientSink creates a new HTTP client sink func NewHTTPClientSink(options map[string]any, logger *log.Logger) (*HTTPClientSink, error) { cfg := HTTPClientConfig{ BufferSize: 1000, BatchSize: 100, BatchDelay: time.Second, Timeout: 30 * time.Second, MaxRetries: 3, RetryDelay: time.Second, RetryBackoff: 2.0, Headers: make(map[string]string), } // Extract URL urlStr, ok := options["url"].(string) if !ok || urlStr == "" { return nil, fmt.Errorf("http_client sink requires 'url' option") } // Validate URL parsedURL, err := url.Parse(urlStr) if err != nil { return nil, fmt.Errorf("invalid URL: %w", err) } if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" { return nil, fmt.Errorf("URL must use http or https scheme") } cfg.URL = urlStr // Extract other options if bufSize, ok := toInt(options["buffer_size"]); ok && bufSize > 0 { cfg.BufferSize = bufSize } if batchSize, ok := toInt(options["batch_size"]); ok && batchSize > 0 { cfg.BatchSize = batchSize } if delayMs, ok := toInt(options["batch_delay_ms"]); ok && delayMs > 0 { cfg.BatchDelay = time.Duration(delayMs) * time.Millisecond } if timeoutSec, ok := toInt(options["timeout_seconds"]); ok && timeoutSec > 0 { cfg.Timeout = time.Duration(timeoutSec) * time.Second } if maxRetries, ok := toInt(options["max_retries"]); ok && maxRetries >= 0 { cfg.MaxRetries = maxRetries } if retryDelayMs, ok := toInt(options["retry_delay_ms"]); ok && retryDelayMs > 0 { cfg.RetryDelay = time.Duration(retryDelayMs) * time.Millisecond } if backoff, ok := toFloat(options["retry_backoff"]); ok && backoff >= 1.0 { cfg.RetryBackoff = backoff } if insecure, ok := options["insecure_skip_verify"].(bool); ok { cfg.InsecureSkipVerify = insecure } // Extract headers if headers, ok := options["headers"].(map[string]any); ok { for k, v := range headers { if strVal, ok := v.(string); ok { cfg.Headers[k] = strVal } } } // Set default Content-Type if not specified if _, exists := cfg.Headers["Content-Type"]; !exists { cfg.Headers["Content-Type"] = "application/json" } h := &HTTPClientSink{ input: make(chan source.LogEntry, cfg.BufferSize), config: cfg, batch: make([]source.LogEntry, 0, cfg.BatchSize), done: make(chan struct{}), startTime: time.Now(), logger: logger, } h.lastProcessed.Store(time.Time{}) h.lastBatchSent.Store(time.Time{}) // Create fasthttp client h.client = &fasthttp.Client{ MaxConnsPerHost: 10, MaxIdleConnDuration: 10 * time.Second, ReadTimeout: cfg.Timeout, WriteTimeout: cfg.Timeout, DisableHeaderNamesNormalizing: true, } // TODO: Implement custom TLS configuration, including InsecureSkipVerify, // by setting a custom dialer on the fasthttp.Client. // For example: // if cfg.InsecureSkipVerify { // h.client.Dial = func(addr string) (net.Conn, error) { // return fasthttp.DialDualStackTimeout(addr, cfg.Timeout, &tls.Config{ // InsecureSkipVerify: true, // }) // } // } // FIXED: Removed incorrect TLS configuration that referenced non-existent field return h, nil } func (h *HTTPClientSink) Input() chan<- source.LogEntry { return h.input } func (h *HTTPClientSink) Start(ctx context.Context) error { h.wg.Add(2) go h.processLoop(ctx) go h.batchTimer(ctx) h.logger.Info("msg", "HTTP client sink started", "component", "http_client_sink", "url", h.config.URL, "batch_size", h.config.BatchSize, "batch_delay", h.config.BatchDelay) return nil } func (h *HTTPClientSink) Stop() { h.logger.Info("msg", "Stopping HTTP client sink") close(h.done) h.wg.Wait() // Send any remaining batched entries h.batchMu.Lock() if len(h.batch) > 0 { batch := h.batch h.batch = make([]source.LogEntry, 0, h.config.BatchSize) h.batchMu.Unlock() h.sendBatch(batch) } else { h.batchMu.Unlock() } h.logger.Info("msg", "HTTP client sink stopped", "total_processed", h.totalProcessed.Load(), "total_batches", h.totalBatches.Load(), "failed_batches", h.failedBatches.Load()) } func (h *HTTPClientSink) GetStats() SinkStats { lastProc, _ := h.lastProcessed.Load().(time.Time) lastBatch, _ := h.lastBatchSent.Load().(time.Time) h.batchMu.Lock() pendingEntries := len(h.batch) h.batchMu.Unlock() return SinkStats{ Type: "http_client", TotalProcessed: h.totalProcessed.Load(), ActiveConnections: h.activeConnections.Load(), StartTime: h.startTime, LastProcessed: lastProc, Details: map[string]any{ "url": h.config.URL, "batch_size": h.config.BatchSize, "pending_entries": pendingEntries, "total_batches": h.totalBatches.Load(), "failed_batches": h.failedBatches.Load(), "last_batch_sent": lastBatch, }, } } func (h *HTTPClientSink) processLoop(ctx context.Context) { defer h.wg.Done() for { select { case entry, ok := <-h.input: if !ok { return } h.totalProcessed.Add(1) h.lastProcessed.Store(time.Now()) // Add to batch h.batchMu.Lock() h.batch = append(h.batch, entry) // Check if batch is full if len(h.batch) >= h.config.BatchSize { batch := h.batch h.batch = make([]source.LogEntry, 0, h.config.BatchSize) h.batchMu.Unlock() // Send batch in background go h.sendBatch(batch) } else { h.batchMu.Unlock() } case <-ctx.Done(): return case <-h.done: return } } } func (h *HTTPClientSink) batchTimer(ctx context.Context) { defer h.wg.Done() ticker := time.NewTicker(h.config.BatchDelay) defer ticker.Stop() for { select { case <-ticker.C: h.batchMu.Lock() if len(h.batch) > 0 { batch := h.batch h.batch = make([]source.LogEntry, 0, h.config.BatchSize) h.batchMu.Unlock() // Send batch in background go h.sendBatch(batch) } else { h.batchMu.Unlock() } case <-ctx.Done(): return case <-h.done: return } } } func (h *HTTPClientSink) sendBatch(batch []source.LogEntry) { h.activeConnections.Add(1) defer h.activeConnections.Add(-1) h.totalBatches.Add(1) h.lastBatchSent.Store(time.Now()) // Prepare request body body, err := json.Marshal(batch) if err != nil { h.logger.Error("msg", "Failed to marshal batch", "component", "http_client_sink", "error", err, "batch_size", len(batch)) h.failedBatches.Add(1) return } // Retry logic var lastErr error retryDelay := h.config.RetryDelay for attempt := 0; attempt <= h.config.MaxRetries; attempt++ { if attempt > 0 { // Wait before retry time.Sleep(retryDelay) retryDelay = time.Duration(float64(retryDelay) * h.config.RetryBackoff) } // Create request req := fasthttp.AcquireRequest() resp := fasthttp.AcquireResponse() defer fasthttp.ReleaseRequest(req) defer fasthttp.ReleaseResponse(resp) req.SetRequestURI(h.config.URL) req.Header.SetMethod("POST") req.SetBody(body) // Set headers for k, v := range h.config.Headers { req.Header.Set(k, v) } // Send request err := h.client.DoTimeout(req, resp, h.config.Timeout) if err != nil { lastErr = fmt.Errorf("request failed: %w", err) h.logger.Warn("msg", "HTTP request failed", "component", "http_client_sink", "attempt", attempt+1, "error", err) continue } // Check response status statusCode := resp.StatusCode() if statusCode >= 200 && statusCode < 300 { // Success h.logger.Debug("msg", "Batch sent successfully", "component", "http_client_sink", "batch_size", len(batch), "status_code", statusCode) return } // Non-2xx status lastErr = fmt.Errorf("server returned status %d: %s", statusCode, resp.Body()) // Don't retry on 4xx errors (client errors) if statusCode >= 400 && statusCode < 500 { h.logger.Error("msg", "Batch rejected by server", "component", "http_client_sink", "status_code", statusCode, "response", string(resp.Body()), "batch_size", len(batch)) h.failedBatches.Add(1) return } h.logger.Warn("msg", "Server returned error status", "component", "http_client_sink", "attempt", attempt+1, "status_code", statusCode, "response", string(resp.Body())) } // All retries failed h.logger.Error("msg", "Failed to send batch after retries", "component", "http_client_sink", "batch_size", len(batch), "last_error", lastErr) h.failedBatches.Add(1) }