v0.3.0 tcp/http client/server add for logwisp chain connection support, config refactor
This commit is contained in:
385
src/internal/sink/http_client.go
Normal file
385
src/internal/sink/http_client.go
Normal file
@ -0,0 +1,385 @@
|
||||
// FILE: src/internal/sink/http_client.go
|
||||
package sink
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"logwisp/src/internal/source"
|
||||
|
||||
"github.com/lixenwraith/log"
|
||||
"github.com/valyala/fasthttp"
|
||||
)
|
||||
|
||||
// HTTPClientSink forwards log entries to a remote HTTP endpoint
|
||||
type HTTPClientSink struct {
|
||||
input chan source.LogEntry
|
||||
config HTTPClientConfig
|
||||
client *fasthttp.Client
|
||||
batch []source.LogEntry
|
||||
batchMu sync.Mutex
|
||||
done chan struct{}
|
||||
wg sync.WaitGroup
|
||||
startTime time.Time
|
||||
logger *log.Logger
|
||||
|
||||
// Statistics
|
||||
totalProcessed atomic.Uint64
|
||||
totalBatches atomic.Uint64
|
||||
failedBatches atomic.Uint64
|
||||
lastProcessed atomic.Value // time.Time
|
||||
lastBatchSent atomic.Value // time.Time
|
||||
activeConnections atomic.Int32
|
||||
}
|
||||
|
||||
// HTTPClientConfig holds HTTP client sink configuration
|
||||
type HTTPClientConfig struct {
|
||||
URL string
|
||||
BufferSize int
|
||||
BatchSize int
|
||||
BatchDelay time.Duration
|
||||
Timeout time.Duration
|
||||
Headers map[string]string
|
||||
|
||||
// Retry configuration
|
||||
MaxRetries int
|
||||
RetryDelay time.Duration
|
||||
RetryBackoff float64 // Multiplier for exponential backoff
|
||||
|
||||
// TLS configuration
|
||||
InsecureSkipVerify bool
|
||||
}
|
||||
|
||||
// NewHTTPClientSink creates a new HTTP client sink
|
||||
func NewHTTPClientSink(options map[string]any, logger *log.Logger) (*HTTPClientSink, error) {
|
||||
cfg := HTTPClientConfig{
|
||||
BufferSize: 1000,
|
||||
BatchSize: 100,
|
||||
BatchDelay: time.Second,
|
||||
Timeout: 30 * time.Second,
|
||||
MaxRetries: 3,
|
||||
RetryDelay: time.Second,
|
||||
RetryBackoff: 2.0,
|
||||
Headers: make(map[string]string),
|
||||
}
|
||||
|
||||
// Extract URL
|
||||
urlStr, ok := options["url"].(string)
|
||||
if !ok || urlStr == "" {
|
||||
return nil, fmt.Errorf("http_client sink requires 'url' option")
|
||||
}
|
||||
|
||||
// Validate URL
|
||||
parsedURL, err := url.Parse(urlStr)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid URL: %w", err)
|
||||
}
|
||||
if parsedURL.Scheme != "http" && parsedURL.Scheme != "https" {
|
||||
return nil, fmt.Errorf("URL must use http or https scheme")
|
||||
}
|
||||
cfg.URL = urlStr
|
||||
|
||||
// Extract other options
|
||||
if bufSize, ok := toInt(options["buffer_size"]); ok && bufSize > 0 {
|
||||
cfg.BufferSize = bufSize
|
||||
}
|
||||
if batchSize, ok := toInt(options["batch_size"]); ok && batchSize > 0 {
|
||||
cfg.BatchSize = batchSize
|
||||
}
|
||||
if delayMs, ok := toInt(options["batch_delay_ms"]); ok && delayMs > 0 {
|
||||
cfg.BatchDelay = time.Duration(delayMs) * time.Millisecond
|
||||
}
|
||||
if timeoutSec, ok := toInt(options["timeout_seconds"]); ok && timeoutSec > 0 {
|
||||
cfg.Timeout = time.Duration(timeoutSec) * time.Second
|
||||
}
|
||||
if maxRetries, ok := toInt(options["max_retries"]); ok && maxRetries >= 0 {
|
||||
cfg.MaxRetries = maxRetries
|
||||
}
|
||||
if retryDelayMs, ok := toInt(options["retry_delay_ms"]); ok && retryDelayMs > 0 {
|
||||
cfg.RetryDelay = time.Duration(retryDelayMs) * time.Millisecond
|
||||
}
|
||||
if backoff, ok := toFloat(options["retry_backoff"]); ok && backoff >= 1.0 {
|
||||
cfg.RetryBackoff = backoff
|
||||
}
|
||||
if insecure, ok := options["insecure_skip_verify"].(bool); ok {
|
||||
cfg.InsecureSkipVerify = insecure
|
||||
}
|
||||
|
||||
// Extract headers
|
||||
if headers, ok := options["headers"].(map[string]any); ok {
|
||||
for k, v := range headers {
|
||||
if strVal, ok := v.(string); ok {
|
||||
cfg.Headers[k] = strVal
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Set default Content-Type if not specified
|
||||
if _, exists := cfg.Headers["Content-Type"]; !exists {
|
||||
cfg.Headers["Content-Type"] = "application/json"
|
||||
}
|
||||
|
||||
h := &HTTPClientSink{
|
||||
input: make(chan source.LogEntry, cfg.BufferSize),
|
||||
config: cfg,
|
||||
batch: make([]source.LogEntry, 0, cfg.BatchSize),
|
||||
done: make(chan struct{}),
|
||||
startTime: time.Now(),
|
||||
logger: logger,
|
||||
}
|
||||
h.lastProcessed.Store(time.Time{})
|
||||
h.lastBatchSent.Store(time.Time{})
|
||||
|
||||
// Create fasthttp client
|
||||
h.client = &fasthttp.Client{
|
||||
MaxConnsPerHost: 10,
|
||||
MaxIdleConnDuration: 10 * time.Second,
|
||||
ReadTimeout: cfg.Timeout,
|
||||
WriteTimeout: cfg.Timeout,
|
||||
DisableHeaderNamesNormalizing: true,
|
||||
}
|
||||
|
||||
// TODO: Implement custom TLS configuration, including InsecureSkipVerify,
|
||||
// by setting a custom dialer on the fasthttp.Client.
|
||||
// For example:
|
||||
// if cfg.InsecureSkipVerify {
|
||||
// h.client.Dial = func(addr string) (net.Conn, error) {
|
||||
// return fasthttp.DialDualStackTimeout(addr, cfg.Timeout, &tls.Config{
|
||||
// InsecureSkipVerify: true,
|
||||
// })
|
||||
// }
|
||||
// }
|
||||
// FIXED: Removed incorrect TLS configuration that referenced non-existent field
|
||||
|
||||
return h, nil
|
||||
}
|
||||
|
||||
func (h *HTTPClientSink) Input() chan<- source.LogEntry {
|
||||
return h.input
|
||||
}
|
||||
|
||||
func (h *HTTPClientSink) Start(ctx context.Context) error {
|
||||
h.wg.Add(2)
|
||||
go h.processLoop(ctx)
|
||||
go h.batchTimer(ctx)
|
||||
|
||||
h.logger.Info("msg", "HTTP client sink started",
|
||||
"component", "http_client_sink",
|
||||
"url", h.config.URL,
|
||||
"batch_size", h.config.BatchSize,
|
||||
"batch_delay", h.config.BatchDelay)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *HTTPClientSink) Stop() {
|
||||
h.logger.Info("msg", "Stopping HTTP client sink")
|
||||
close(h.done)
|
||||
h.wg.Wait()
|
||||
|
||||
// Send any remaining batched entries
|
||||
h.batchMu.Lock()
|
||||
if len(h.batch) > 0 {
|
||||
batch := h.batch
|
||||
h.batch = make([]source.LogEntry, 0, h.config.BatchSize)
|
||||
h.batchMu.Unlock()
|
||||
h.sendBatch(batch)
|
||||
} else {
|
||||
h.batchMu.Unlock()
|
||||
}
|
||||
|
||||
h.logger.Info("msg", "HTTP client sink stopped",
|
||||
"total_processed", h.totalProcessed.Load(),
|
||||
"total_batches", h.totalBatches.Load(),
|
||||
"failed_batches", h.failedBatches.Load())
|
||||
}
|
||||
|
||||
func (h *HTTPClientSink) GetStats() SinkStats {
|
||||
lastProc, _ := h.lastProcessed.Load().(time.Time)
|
||||
lastBatch, _ := h.lastBatchSent.Load().(time.Time)
|
||||
|
||||
h.batchMu.Lock()
|
||||
pendingEntries := len(h.batch)
|
||||
h.batchMu.Unlock()
|
||||
|
||||
return SinkStats{
|
||||
Type: "http_client",
|
||||
TotalProcessed: h.totalProcessed.Load(),
|
||||
ActiveConnections: h.activeConnections.Load(),
|
||||
StartTime: h.startTime,
|
||||
LastProcessed: lastProc,
|
||||
Details: map[string]any{
|
||||
"url": h.config.URL,
|
||||
"batch_size": h.config.BatchSize,
|
||||
"pending_entries": pendingEntries,
|
||||
"total_batches": h.totalBatches.Load(),
|
||||
"failed_batches": h.failedBatches.Load(),
|
||||
"last_batch_sent": lastBatch,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (h *HTTPClientSink) processLoop(ctx context.Context) {
|
||||
defer h.wg.Done()
|
||||
|
||||
for {
|
||||
select {
|
||||
case entry, ok := <-h.input:
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
h.totalProcessed.Add(1)
|
||||
h.lastProcessed.Store(time.Now())
|
||||
|
||||
// Add to batch
|
||||
h.batchMu.Lock()
|
||||
h.batch = append(h.batch, entry)
|
||||
|
||||
// Check if batch is full
|
||||
if len(h.batch) >= h.config.BatchSize {
|
||||
batch := h.batch
|
||||
h.batch = make([]source.LogEntry, 0, h.config.BatchSize)
|
||||
h.batchMu.Unlock()
|
||||
|
||||
// Send batch in background
|
||||
go h.sendBatch(batch)
|
||||
} else {
|
||||
h.batchMu.Unlock()
|
||||
}
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-h.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *HTTPClientSink) batchTimer(ctx context.Context) {
|
||||
defer h.wg.Done()
|
||||
|
||||
ticker := time.NewTicker(h.config.BatchDelay)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
h.batchMu.Lock()
|
||||
if len(h.batch) > 0 {
|
||||
batch := h.batch
|
||||
h.batch = make([]source.LogEntry, 0, h.config.BatchSize)
|
||||
h.batchMu.Unlock()
|
||||
|
||||
// Send batch in background
|
||||
go h.sendBatch(batch)
|
||||
} else {
|
||||
h.batchMu.Unlock()
|
||||
}
|
||||
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-h.done:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *HTTPClientSink) sendBatch(batch []source.LogEntry) {
|
||||
h.activeConnections.Add(1)
|
||||
defer h.activeConnections.Add(-1)
|
||||
|
||||
h.totalBatches.Add(1)
|
||||
h.lastBatchSent.Store(time.Now())
|
||||
|
||||
// Prepare request body
|
||||
body, err := json.Marshal(batch)
|
||||
if err != nil {
|
||||
h.logger.Error("msg", "Failed to marshal batch",
|
||||
"component", "http_client_sink",
|
||||
"error", err,
|
||||
"batch_size", len(batch))
|
||||
h.failedBatches.Add(1)
|
||||
return
|
||||
}
|
||||
|
||||
// Retry logic
|
||||
var lastErr error
|
||||
retryDelay := h.config.RetryDelay
|
||||
|
||||
for attempt := 0; attempt <= h.config.MaxRetries; attempt++ {
|
||||
if attempt > 0 {
|
||||
// Wait before retry
|
||||
time.Sleep(retryDelay)
|
||||
retryDelay = time.Duration(float64(retryDelay) * h.config.RetryBackoff)
|
||||
}
|
||||
|
||||
// Create request
|
||||
req := fasthttp.AcquireRequest()
|
||||
resp := fasthttp.AcquireResponse()
|
||||
defer fasthttp.ReleaseRequest(req)
|
||||
defer fasthttp.ReleaseResponse(resp)
|
||||
|
||||
req.SetRequestURI(h.config.URL)
|
||||
req.Header.SetMethod("POST")
|
||||
req.SetBody(body)
|
||||
|
||||
// Set headers
|
||||
for k, v := range h.config.Headers {
|
||||
req.Header.Set(k, v)
|
||||
}
|
||||
|
||||
// Send request
|
||||
err := h.client.DoTimeout(req, resp, h.config.Timeout)
|
||||
if err != nil {
|
||||
lastErr = fmt.Errorf("request failed: %w", err)
|
||||
h.logger.Warn("msg", "HTTP request failed",
|
||||
"component", "http_client_sink",
|
||||
"attempt", attempt+1,
|
||||
"error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Check response status
|
||||
statusCode := resp.StatusCode()
|
||||
if statusCode >= 200 && statusCode < 300 {
|
||||
// Success
|
||||
h.logger.Debug("msg", "Batch sent successfully",
|
||||
"component", "http_client_sink",
|
||||
"batch_size", len(batch),
|
||||
"status_code", statusCode)
|
||||
return
|
||||
}
|
||||
|
||||
// Non-2xx status
|
||||
lastErr = fmt.Errorf("server returned status %d: %s", statusCode, resp.Body())
|
||||
|
||||
// Don't retry on 4xx errors (client errors)
|
||||
if statusCode >= 400 && statusCode < 500 {
|
||||
h.logger.Error("msg", "Batch rejected by server",
|
||||
"component", "http_client_sink",
|
||||
"status_code", statusCode,
|
||||
"response", string(resp.Body()),
|
||||
"batch_size", len(batch))
|
||||
h.failedBatches.Add(1)
|
||||
return
|
||||
}
|
||||
|
||||
h.logger.Warn("msg", "Server returned error status",
|
||||
"component", "http_client_sink",
|
||||
"attempt", attempt+1,
|
||||
"status_code", statusCode,
|
||||
"response", string(resp.Body()))
|
||||
}
|
||||
|
||||
// All retries failed
|
||||
h.logger.Error("msg", "Failed to send batch after retries",
|
||||
"component", "http_client_sink",
|
||||
"batch_size", len(batch),
|
||||
"last_error", lastErr)
|
||||
h.failedBatches.Add(1)
|
||||
}
|
||||
Reference in New Issue
Block a user