lixen/logwisp
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v0.3.10 config auto-save added, dependency update, limiter packages refactored

Browse Source
This commit is contained in:
Lixen Wraith
2025-09-01 16:16:52 -04:00
parent 3c74a6336e
commit fce6ee5c65
32 changed files with 309 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
src/internal/sink/console.go
View File

@ -9,8 +9,8 @@ import (
"sync/atomic"
"time"
"logwisp/src/internal/core"
"logwisp/src/internal/format"
"logwisp/src/internal/source"
"github.com/lixenwraith/log"
)
@ -23,7 +23,7 @@ type ConsoleConfig struct {
// StdoutSink writes log entries to stdout
type StdoutSink struct {
input chan source.LogEntry
input chan core.LogEntry
config ConsoleConfig
output io.Writer
done chan struct{}
@ -53,7 +53,7 @@ func NewStdoutSink(options map[string]any, logger *log.Logger, formatter format.
}
s := &StdoutSink{
input: make(chan source.LogEntry, config.BufferSize),
input: make(chan core.LogEntry, config.BufferSize),
config: config,
output: os.Stdout,
done: make(chan struct{}),
@ -66,7 +66,7 @@ func NewStdoutSink(options map[string]any, logger *log.Logger, formatter format.
return s, nil
}
func (s *StdoutSink) Input() chan<- source.LogEntry {
func (s *StdoutSink) Input() chan<- core.LogEntry {
return s.input
}
@ -136,7 +136,7 @@ func (s *StdoutSink) processLoop(ctx context.Context) {
// StderrSink writes log entries to stderr
type StderrSink struct {
input chan source.LogEntry
input chan core.LogEntry
config ConsoleConfig
output io.Writer
done chan struct{}
@ -166,7 +166,7 @@ func NewStderrSink(options map[string]any, logger *log.Logger, formatter format.
}
s := &StderrSink{
input: make(chan source.LogEntry, config.BufferSize),
input: make(chan core.LogEntry, config.BufferSize),
config: config,
output: os.Stderr,
done: make(chan struct{}),
@ -179,7 +179,7 @@ func NewStderrSink(options map[string]any, logger *log.Logger, formatter format.
return s, nil
}
func (s *StderrSink) Input() chan<- source.LogEntry {
func (s *StderrSink) Input() chan<- core.LogEntry {
return s.input
}

8
src/internal/sink/file.go
View File

@ -7,15 +7,15 @@ import (
"sync/atomic"
"time"
"logwisp/src/internal/core"
"logwisp/src/internal/format"
"logwisp/src/internal/source"
"github.com/lixenwraith/log"
)
// FileSink writes log entries to files with rotation
type FileSink struct {
input chan source.LogEntry
input chan core.LogEntry
writer *log.Logger // Internal logger instance for file writing
done chan struct{}
startTime time.Time
@ -83,7 +83,7 @@ func NewFileSink(options map[string]any, logger *log.Logger, formatter format.Fo
}
fs := &FileSink{
input: make(chan source.LogEntry, bufferSize),
input: make(chan core.LogEntry, bufferSize),
writer: writer,
done: make(chan struct{}),
startTime: time.Now(),
@ -95,7 +95,7 @@ func NewFileSink(options map[string]any, logger *log.Logger, formatter format.Fo
return fs, nil
}
func (fs *FileSink) Input() chan<- source.LogEntry {
func (fs *FileSink) Input() chan<- core.LogEntry {
return fs.input
}

33
src/internal/sink/http.go
View File

@ -12,9 +12,9 @@ import (
"time"
"logwisp/src/internal/config"
"logwisp/src/internal/core"
"logwisp/src/internal/format"
"logwisp/src/internal/netlimit"
"logwisp/src/internal/source"
"logwisp/src/internal/limit"
"logwisp/src/internal/version"
"github.com/lixenwraith/log"
@ -24,7 +24,7 @@ import (
// HTTPSink streams log entries via Server-Sent Events
type HTTPSink struct {
input chan source.LogEntry
input chan core.LogEntry
config HTTPConfig
server *fasthttp.Server
activeClients atomic.Int64
@ -43,7 +43,7 @@ type HTTPSink struct {
standalone bool
// Net limiting
netLimiter *netlimit.Limiter
netLimiter *limit.NetLimiter
// Statistics
totalProcessed atomic.Uint64
@ -126,7 +126,7 @@ func NewHTTPSink(options map[string]any, logger *log.Logger, formatter format.Fo
}
h := &HTTPSink{
input: make(chan source.LogEntry, cfg.BufferSize),
input: make(chan core.LogEntry, cfg.BufferSize),
config: cfg,
startTime: time.Now(),
done: make(chan struct{}),
@ -140,18 +140,17 @@ func NewHTTPSink(options map[string]any, logger *log.Logger, formatter format.Fo
// Initialize net limiter if configured
if cfg.NetLimit != nil && cfg.NetLimit.Enabled {
h.netLimiter = netlimit.New(*cfg.NetLimit, logger)
h.netLimiter = limit.NewNetLimiter(*cfg.NetLimit, logger)
}
return h, nil
}
func (h *HTTPSink) Input() chan<- source.LogEntry {
func (h *HTTPSink) Input() chan<- core.LogEntry {
return h.input
}
func (h *HTTPSink) Start(ctx context.Context) error {
// TODO: use or remove unused ctx
if !h.standalone {
// In router mode, don't start our own server
h.logger.Debug("msg", "HTTP sink in router mode, skipping server start",
@ -185,6 +184,16 @@ func (h *HTTPSink) Start(ctx context.Context) error {
}
}()
// Monitor context for shutdown signal
go func() {
<-ctx.Done()
if h.server != nil {
shutdownCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
h.server.ShutdownWithContext(shutdownCtx)
}
}()
// Check if server started successfully
select {
case err := <-errChan:
@ -299,7 +308,7 @@ func (h *HTTPSink) handleStream(ctx *fasthttp.RequestCtx) {
ctx.Response.Header.Set("X-Accel-Buffering", "no")
// Create subscription for this client
clientChan := make(chan source.LogEntry, h.config.BufferSize)
clientChan := make(chan core.LogEntry, h.config.BufferSize)
clientDone := make(chan struct{})
// Subscribe to input channel
@ -415,7 +424,7 @@ func (h *HTTPSink) handleStream(ctx *fasthttp.RequestCtx) {
ctx.SetBodyStreamWriter(streamFunc)
}
func (h *HTTPSink) formatEntryForSSE(w *bufio.Writer, entry source.LogEntry) error {
func (h *HTTPSink) formatEntryForSSE(w *bufio.Writer, entry core.LogEntry) error {
formatted, err := h.formatter.Format(entry)
if err != nil {
return err
@ -434,7 +443,7 @@ func (h *HTTPSink) formatEntryForSSE(w *bufio.Writer, entry source.LogEntry) err
return nil
}
func (h *HTTPSink) createHeartbeatEntry() source.LogEntry {
func (h *HTTPSink) createHeartbeatEntry() core.LogEntry {
message := "heartbeat"
// Build fields for heartbeat metadata
@ -448,7 +457,7 @@ func (h *HTTPSink) createHeartbeatEntry() source.LogEntry {
fieldsJSON, _ := json.Marshal(fields)
return source.LogEntry{
return core.LogEntry{
Time: time.Now(),
Source: "logwisp-http",
Level: "INFO",

20
src/internal/sink/http_client.go
View File

@ -10,8 +10,8 @@ import (
"sync/atomic"
"time"
"logwisp/src/internal/core"
"logwisp/src/internal/format"
"logwisp/src/internal/source"
"github.com/lixenwraith/log"
"github.com/valyala/fasthttp"
@ -19,10 +19,10 @@ import (
// HTTPClientSink forwards log entries to a remote HTTP endpoint
type HTTPClientSink struct {
input chan source.LogEntry
input chan core.LogEntry
config HTTPClientConfig
client *fasthttp.Client
batch []source.LogEntry
batch []core.LogEntry
batchMu sync.Mutex
done chan struct{}
wg sync.WaitGroup
@ -127,9 +127,9 @@ func NewHTTPClientSink(options map[string]any, logger *log.Logger, formatter for
}
h := &HTTPClientSink{
input: make(chan source.LogEntry, cfg.BufferSize),
input: make(chan core.LogEntry, cfg.BufferSize),
config: cfg,
batch: make([]source.LogEntry, 0, cfg.BatchSize),
batch: make([]core.LogEntry, 0, cfg.BatchSize),
done: make(chan struct{}),
startTime: time.Now(),
logger: logger,
@ -162,7 +162,7 @@ func NewHTTPClientSink(options map[string]any, logger *log.Logger, formatter for
return h, nil
}
func (h *HTTPClientSink) Input() chan<- source.LogEntry {
func (h *HTTPClientSink) Input() chan<- core.LogEntry {
return h.input
}
@ -188,7 +188,7 @@ func (h *HTTPClientSink) Stop() {
h.batchMu.Lock()
if len(h.batch) > 0 {
batch := h.batch
h.batch = make([]source.LogEntry, 0, h.config.BatchSize)
h.batch = make([]core.LogEntry, 0, h.config.BatchSize)
h.batchMu.Unlock()
h.sendBatch(batch)
} else {
@ -246,7 +246,7 @@ func (h *HTTPClientSink) processLoop(ctx context.Context) {
// Check if batch is full
if int64(len(h.batch)) >= h.config.BatchSize {
batch := h.batch
h.batch = make([]source.LogEntry, 0, h.config.BatchSize)
h.batch = make([]core.LogEntry, 0, h.config.BatchSize)
h.batchMu.Unlock()
// Send batch in background
@ -275,7 +275,7 @@ func (h *HTTPClientSink) batchTimer(ctx context.Context) {
h.batchMu.Lock()
if len(h.batch) > 0 {
batch := h.batch
h.batch = make([]source.LogEntry, 0, h.config.BatchSize)
h.batch = make([]core.LogEntry, 0, h.config.BatchSize)
h.batchMu.Unlock()
// Send batch in background
@ -292,7 +292,7 @@ func (h *HTTPClientSink) batchTimer(ctx context.Context) {
}
}
func (h *HTTPClientSink) sendBatch(batch []source.LogEntry) {
func (h *HTTPClientSink) sendBatch(batch []core.LogEntry) {
h.activeConnections.Add(1)
defer h.activeConnections.Add(-1)

4
src/internal/sink/sink.go
View File

@ -5,13 +5,13 @@ import (
"context"
"time"
"logwisp/src/internal/source"
"logwisp/src/internal/core"
)
// Sink represents an output destination for log entries
type Sink interface {
// Input returns the channel for sending log entries to this sink
Input() chan<- source.LogEntry
Input() chan<- core.LogEntry
// Start begins processing log entries
Start(ctx context.Context) error

49
src/internal/sink/tcp.go
View File

@ -5,24 +5,24 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/lixenwraith/log/compat"
"net"
"sync"
"sync/atomic"
"time"
"logwisp/src/internal/config"
"logwisp/src/internal/core"
"logwisp/src/internal/format"
"logwisp/src/internal/netlimit"
"logwisp/src/internal/source"
"logwisp/src/internal/limit"
"github.com/lixenwraith/log"
"github.com/lixenwraith/log/compat"
"github.com/panjf2000/gnet/v2"
)
// TCPSink streams log entries via TCP
type TCPSink struct {
input chan source.LogEntry
input chan core.LogEntry
config TCPConfig
server *tcpServer
done chan struct{}
@ -31,7 +31,7 @@ type TCPSink struct {
engine *gnet.Engine
engineMu sync.Mutex
wg sync.WaitGroup
netLimiter *netlimit.Limiter
netLimiter *limit.NetLimiter
logger *log.Logger
formatter format.Formatter
@ -106,7 +106,7 @@ func NewTCPSink(options map[string]any, logger *log.Logger, formatter format.For
}
t := &TCPSink{
input: make(chan source.LogEntry, cfg.BufferSize),
input: make(chan core.LogEntry, cfg.BufferSize),
config: cfg,
done: make(chan struct{}),
startTime: time.Now(),
@ -116,13 +116,13 @@ func NewTCPSink(options map[string]any, logger *log.Logger, formatter format.For
t.lastProcessed.Store(time.Time{})
if cfg.NetLimit != nil && cfg.NetLimit.Enabled {
t.netLimiter = netlimit.New(*cfg.NetLimit, logger)
t.netLimiter = limit.NewNetLimiter(*cfg.NetLimit, logger)
}
return t, nil
}
func (t *TCPSink) Input() chan<- source.LogEntry {
func (t *TCPSink) Input() chan<- core.LogEntry {
return t.input
}
@ -133,7 +133,7 @@ func (t *TCPSink) Start(ctx context.Context) error {
t.wg.Add(1)
go func() {
defer t.wg.Done()
t.broadcastLoop()
t.broadcastLoop(ctx)
}()
// Configure gnet
@ -163,6 +163,18 @@ func (t *TCPSink) Start(ctx context.Context) error {
errChan <- err
}()
// Monitor context for shutdown
go func() {
<-ctx.Done()
t.engineMu.Lock()
if t.engine != nil {
shutdownCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
(*t.engine).Stop(shutdownCtx)
}
t.engineMu.Unlock()
}()
// Wait briefly for server to start or fail
select {
case err := <-errChan:
@ -221,7 +233,7 @@ func (t *TCPSink) GetStats() SinkStats {
}
}
func (t *TCPSink) broadcastLoop() {
func (t *TCPSink) broadcastLoop(ctx context.Context) {
var ticker *time.Ticker
var tickerChan <-chan time.Time
@ -233,6 +245,8 @@ func (t *TCPSink) broadcastLoop() {
for {
select {
case <-ctx.Done():
return
case entry, ok := <-t.input:
if !ok {
return
@ -278,7 +292,7 @@ func (t *TCPSink) broadcastLoop() {
}
// Create heartbeat as a proper LogEntry
func (t *TCPSink) createHeartbeatEntry() source.LogEntry {
func (t *TCPSink) createHeartbeatEntry() core.LogEntry {
message := "heartbeat"
// Build fields for heartbeat metadata
@ -292,7 +306,7 @@ func (t *TCPSink) createHeartbeatEntry() source.LogEntry {
fieldsJSON, _ := json.Marshal(fields)
return source.LogEntry{
return core.LogEntry{
Time: time.Now(),
Source: "logwisp-tcp",
Level: "INFO",
@ -384,13 +398,4 @@ func (s *tcpServer) OnTraffic(c gnet.Conn) gnet.Action {
// We don't expect input from clients, just discard
c.Discard(-1)
return gnet.None
}
// noopLogger implements gnet Logger interface but discards everything
// type noopLogger struct{}
//
// func (n noopLogger) Debugf(format string, args ...any) {}
// func (n noopLogger) Infof(format string, args ...any) {}
// func (n noopLogger) Warnf(format string, args ...any) {}
// func (n noopLogger) Errorf(format string, args ...any) {}
// func (n noopLogger) Fatalf(format string, args ...any) {}
}

10
src/internal/sink/tcp_client.go
View File

@ -10,15 +10,15 @@ import (
"sync/atomic"
"time"
"logwisp/src/internal/core"
"logwisp/src/internal/format"
"logwisp/src/internal/source"
"github.com/lixenwraith/log"
)
// TCPClientSink forwards log entries to a remote TCP endpoint
type TCPClientSink struct {
input chan source.LogEntry
input chan core.LogEntry
config TCPClientConfig
conn net.Conn
connMu sync.RWMutex
@ -104,7 +104,7 @@ func NewTCPClientSink(options map[string]any, logger *log.Logger, formatter form
}
t := &TCPClientSink{
input: make(chan source.LogEntry, cfg.BufferSize),
input: make(chan core.LogEntry, cfg.BufferSize),
config: cfg,
done: make(chan struct{}),
startTime: time.Now(),
@ -117,7 +117,7 @@ func NewTCPClientSink(options map[string]any, logger *log.Logger, formatter form
return t, nil
}
func (t *TCPClientSink) Input() chan<- source.LogEntry {
func (t *TCPClientSink) Input() chan<- core.LogEntry {
return t.input
}
@ -345,7 +345,7 @@ func (t *TCPClientSink) processLoop(ctx context.Context) {
}
}
func (t *TCPClientSink) sendEntry(entry source.LogEntry) error {
func (t *TCPClientSink) sendEntry(entry core.LogEntry) error {
// Get current connection
t.connMu.RLock()
conn := t.conn