v0.3.8 refactor and cli help add

go.mod

@@ -4,7 +4,7 @@ go 1.24.5
 
 require (
 	github.com/lixenwraith/config v0.0.0-20250721005322-3b1023974d3d
-	github.com/lixenwraith/log v0.0.0-20250720221103-db34b7e4a2aa
+	github.com/lixenwraith/log v0.0.0-20250722012845-16a3079e46e2
 	github.com/panjf2000/gnet/v2 v2.9.1
 	github.com/valyala/fasthttp v1.64.0
 )
@@ -23,3 +23,5 @@ require (
 	golang.org/x/sys v0.34.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect
 )
+
+replace github.com/mitchellh/mapstructure => github.com/go-viper/mapstructure v1.6.0

go.sum

@@ -4,14 +4,16 @@ github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwTo
 github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/go-viper/mapstructure v1.6.0 h1:0WdPOF2rmmQDN1xo8qIgxyugvLp71HrZSWyGLxofobw=
+github.com/go-viper/mapstructure v1.6.0/go.mod h1:FcbLReH7/cjaC0RVQR+LHFIrBhHF3s1e/ud1KMDoBVw=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/lixenwraith/config v0.0.0-20250721005322-3b1023974d3d h1:h3IWdUA6Fyl5/lvNmPdtKtLFVnZos71aV3RHILYKY/M=
 github.com/lixenwraith/config v0.0.0-20250721005322-3b1023974d3d/go.mod h1:F8ieHeZgOCPsoym5eynx4kjupfLXBpvJfnX1GzX++EA=
 github.com/lixenwraith/log v0.0.0-20250720221103-db34b7e4a2aa h1:7x25rdA8azXtY46/MgDQIKTLpZv6TXtqMBCfzL5wSJ4=
 github.com/lixenwraith/log v0.0.0-20250720221103-db34b7e4a2aa/go.mod h1:asd0/TQplmacopOKWcqW0jysau/lWohR2Fe29KBSp2w=
-github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
-github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
+github.com/lixenwraith/log v0.0.0-20250722012845-16a3079e46e2 h1:nP/12l+gKkZnZRoM3Vy4iT2anBQm1jCtrppyZq9pcq4=
+github.com/lixenwraith/log v0.0.0-20250722012845-16a3079e46e2/go.mod h1:sLCRfKeLInCj2LcMnAo2knULwfszU8QPuIFOQ8crcFo=
 github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg=
 github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek=
 github.com/panjf2000/gnet/v2 v2.9.1 h1:bKewICy/0xnQ9PMzNaswpe/Ah14w1TrRk91LHTcbIlA=

src/cmd/logwisp/bootstrap.go

@@ -1,4 +1,4 @@
-// FILE: src/cmd/logwisp/bootstrap.go
+// FILE: logwisp/src/cmd/logwisp/bootstrap.go
 package main
 
 import (

src/cmd/logwisp/help.go

@@ -0,0 +1,56 @@
+// FILE: logwisp/src/cmd/logwisp/help.go
+package main
+
+import (
+	"fmt"
+	"os"
+)
+
+const helpText = `LogWisp: A flexible log transport and processing tool.
+
+Usage: logwisp [options]
+
+Application Control:
+  -c, --config <path>      (string) Path to configuration file (default: logwisp.toml).
+  -h, --help               Display this help message and exit.
+  -v, --version            Display version information and exit.
+  -b, --background         Run LogWisp in the background as a daemon.
+  -q, --quiet              Suppress all console output, including errors.
+      --router             Enable HTTP router mode for multiplexing pipelines.
+
+Runtime Behavior:
+      --disable-status-reporter  Disable the periodic status reporter.
+
+Configuration Sources (Precedence: CLI > Env > File > Defaults):
+  - CLI flags override all other settings.
+  - Environment variables (e.g., LOGWISP_ROUTER=true) override file settings.
+  - TOML configuration file is the primary method for defining pipelines.
+
+Logging ([logging] section or LOGWISP_LOGGING_* env vars):
+  output = "stderr"        (string) Log output: none, stdout, stderr, file, both.
+  level = "info"           (string) Log level: debug, info, warn, error.
+  [logging.file]           Settings for file logging (directory, name, rotation).
+  [logging.console]        Settings for console logging (target, format).
+
+Pipelines ([[pipelines]] array in TOML):
+  Each pipeline defines a complete data flow from sources to sinks.
+  name = "my_pipeline"     (string) Unique name for the pipeline.
+  sources = [...]          (array)  Data inputs (e.g., directory, stdin, http, tcp).
+  sinks = [...]            (array)  Data outputs (e.g., http, tcp, file, stdout, stderr, http_client).
+  filters = [...]          (array)  Optional filters to include/exclude logs based on regex.
+  rate_limit = {...}       (object) Optional rate limiting for the entire pipeline.
+  auth = {...}             (object) Optional authentication for network sinks.
+  format = "json"          (string) Optional output formatter for the pipeline (raw, text, json).
+
+For detailed configuration options, please refer to the documentation.
+`
+
+// CheckAndDisplayHelp scans arguments for help flags and prints help text if found.
+func CheckAndDisplayHelp(args []string) {
+	for _, arg := range args {
+		if arg == "-h" || arg == "--help" {
+			fmt.Fprint(os.Stdout, helpText)
+			os.Exit(0)
+		}
+	}
+}

src/cmd/logwisp/main.go

@@ -1,4 +1,4 @@
-// FILE: src/cmd/logwisp/main.go
+// FILE: logwisp/src/cmd/logwisp/main.go
 package main
 
 import (
@@ -23,6 +23,9 @@ func main() {
 	// Emulates nohup
 	signal.Ignore(syscall.SIGHUP)
 
+	// Early check for help flag to avoid unnecessary config loading
+	CheckAndDisplayHelp(os.Args[1:])
+
 	// Load configuration with automatic CLI parsing
 	cfg, err := config.Load(os.Args[1:])
 	if err != nil {
@@ -58,12 +61,17 @@ func main() {
 		os.Exit(0) // The parent process exits successfully.
 	}
 
-	// Initialize logger
+	// Initialize logger instance and apply configuration
 	if err := initializeLogger(cfg); err != nil {
 		FatalError(1, "Failed to initialize logger: %v\n", err)
 	}
 	defer shutdownLogger()
 
+	// Start the logger
+	if err := logger.Start(); err != nil {
+		FatalError(1, "Failed to start logger: %v\n", err)
+	}
+
 	// Log startup information
 	logger.Info("msg", "LogWisp starting",
 		"version", version.String(),

src/cmd/logwisp/output.go

@@ -1,4 +1,4 @@
-// FILE: src/cmd/logwisp/output.go
+// FILE: logwisp/src/cmd/logwisp/output.go
 package main
 
 import (

src/cmd/logwisp/status.go

@@ -1,4 +1,4 @@
-// FILE: src/cmd/logwisp/status.go
+// FILE: logwisp/src/cmd/logwisp/status.go
 package main
 
 import (

src/internal/config/auth.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/config/auth.go
+// FILE: logwisp/src/internal/config/auth.go
 package config
 
 import "fmt"

src/internal/config/config.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/config/config.go
+// FILE: logwisp/src/internal/config/config.go
 package config
 
 type Config struct {

src/internal/config/filter.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/config/filter.go
+// FILE: logwisp/src/internal/config/filter.go
 package config
 
 import (

src/internal/config/loader.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/config/loader.go
+// FILE: logwisp/src/internal/config/loader.go
 package config
 
 import (

src/internal/config/logging.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/config/logging.go
+// FILE: logwisp/src/internal/config/logging.go
 package config
 
 import "fmt"

src/internal/config/pipeline.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/config/pipeline.go
+// FILE: logwisp/src/internal/config/pipeline.go
 package config
 
 import (

src/internal/config/ratelimit.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/config/ratelimit.go
+// FILE: logwisp/src/internal/config/ratelimit.go
 package config
 
 import (

src/internal/config/server.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/config/server.go
+// FILE: logwisp/src/internal/config/server.go
 package config
 
 import "fmt"

src/internal/config/ssl.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/config/ssl.go
+// FILE: logwisp/src/internal/config/ssl.go
 package config
 
 import "fmt"

src/internal/config/validation.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/config/validation.go
+// FILE: logwisp/src/internal/config/validation.go
 package config
 
 import (

src/internal/filter/chain.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/filter/chain.go
+// FILE: logwisp/src/internal/filter/chain.go
 package filter
 
 import (

src/internal/filter/filter.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/filter/filter.go
+// FILE: logwisp/src/internal/filter/filter.go
 package filter
 
 import (

src/internal/format/format.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/format/format.go
+// FILE: logwisp/src/internal/format/format.go
 package format
 
 import (

src/internal/format/json.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/format/json.go
+// FILE: logwisp/src/internal/format/json.go
 package format
 
 import (

src/internal/format/raw.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/format/raw.go
+// FILE: logwisp/src/internal/format/raw.go
 package format
 
 import (

src/internal/format/text.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/format/text.go
+// FILE: logwisp/src/internal/format/text.go
 package format
 
 import (

src/internal/limiter/token_bucket.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/limiter/token_bucket.go
+// FILE: logwisp/src/internal/limiter/token_bucket.go
 package limiter
 
 import (

src/internal/netlimit/limiter.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/netlimit/limiter.go
+// FILE: logwisp/src/internal/netlimit/limiter.go
 package netlimit
 
 import (

src/internal/ratelimit/limiter.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/ratelimit/limiter.go
+// FILE: logwisp/src/internal/ratelimit/limiter.go
 package ratelimit
 
 import (

src/internal/service/httprouter.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/service/httprouter.go
+// FILE: logwisp/src/internal/service/httprouter.go
 package service
 
 import (

src/internal/service/pipeline.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/service/pipeline.go
+// FILE: logwisp/src/internal/service/pipeline.go
 package service
 
 import (

src/internal/service/routerserver.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/service/routerserver.go
+// FILE: logwisp/src/internal/service/routerserver.go
 package service
 
 import (

src/internal/service/service.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/service/service.go
+// FILE: logwisp/src/internal/service/service.go
 package service
 
 import (

src/internal/sink/console.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/sink/console.go
+// FILE: logwisp/src/internal/sink/console.go
 package sink
 
 import (

src/internal/sink/file.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/sink/file.go
+// FILE: logwisp/src/internal/sink/file.go
 package sink
 
 import (
@@ -70,7 +70,13 @@ func NewFileSink(options map[string]any, logger *log.Logger, formatter format.Fo
 		return nil, fmt.Errorf("failed to initialize file writer: %w", err)
 	}
 
+	// Start the internal file writer
+	if err := writer.Start(); err != nil {
+		return nil, fmt.Errorf("failed to start file writer: %w", err)
+	}
+
 	// Buffer size for input channel
+	// TODO: Make this configurable
 	bufferSize := int64(1000)
 	if bufSize, ok := options["buffer_size"].(int64); ok && bufSize > 0 {
 		bufferSize = bufSize

src/internal/sink/http.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/sink/http.go
+// FILE: logwisp/src/internal/sink/http.go
 package sink
 
 import (

src/internal/sink/http_client.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/sink/http_client.go
+// FILE: logwisp/src/internal/sink/http_client.go
 package sink
 
 import (

src/internal/sink/sink.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/sink/sink.go
+// FILE: logwisp/src/internal/sink/sink.go
 package sink
 
 import (

src/internal/sink/tcp.go

@@ -1,10 +1,11 @@
-// FILE: src/internal/sink/tcp.go
+// FILE: logwisp/src/internal/sink/tcp.go
 package sink
 
 import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/lixenwraith/log/compat"
 	"net"
 	"sync"
 	"sync/atomic"
@@ -138,7 +139,10 @@ func (t *TCPSink) Start(ctx context.Context) error {
 	// Configure gnet
 	addr := fmt.Sprintf("tcp://:%d", t.config.Port)
 
-	// Run gnet in separate goroutine to avoid blocking
+	// Create a gnet adapter using the existing logger instance
+	gnetLogger := compat.NewGnetAdapter(t.logger)
+
+	// Start gnet server
 	errChan := make(chan error, 1)
 	go func() {
 		t.logger.Info("msg", "Starting TCP server",
@@ -146,7 +150,7 @@ func (t *TCPSink) Start(ctx context.Context) error {
 			"port", t.config.Port)
 
 		err := gnet.Run(t.server, addr,
-			gnet.WithLogger(noopLogger{}),
+			gnet.WithLogger(gnetLogger),
 			gnet.WithMulticore(true),
 			gnet.WithReusePort(true),
 		)
@@ -383,10 +387,10 @@ func (s *tcpServer) OnTraffic(c gnet.Conn) gnet.Action {
 }
 
 // noopLogger implements gnet Logger interface but discards everything
-type noopLogger struct{}
-
-func (n noopLogger) Debugf(format string, args ...any) {}
-func (n noopLogger) Infof(format string, args ...any)  {}
-func (n noopLogger) Warnf(format string, args ...any)  {}
-func (n noopLogger) Errorf(format string, args ...any) {}
-func (n noopLogger) Fatalf(format string, args ...any) {}
+// type noopLogger struct{}
+//
+// func (n noopLogger) Debugf(format string, args ...any) {}
+// func (n noopLogger) Infof(format string, args ...any)  {}
+// func (n noopLogger) Warnf(format string, args ...any)  {}
+// func (n noopLogger) Errorf(format string, args ...any) {}
+// func (n noopLogger) Fatalf(format string, args ...any) {}

src/internal/sink/tcp_client.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/sink/tcp_client.go
+// FILE: logwisp/src/internal/sink/tcp_client.go
 package sink
 
 import (

src/internal/source/directory.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/source/directory.go
+// FILE: logwisp/src/internal/source/directory.go
 package source
 
 import (

src/internal/source/file_watcher.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/source/file_watcher.go
+// FILE: logwisp/src/internal/source/file_watcher.go
 package source
 
 import (
@@ -79,7 +79,7 @@ func (w *fileWatcher) watch(ctx context.Context) error {
 	}
 }
 
-// FILE: src/internal/source/file_watcher.go
+// FILE: logwisp/src/internal/source/file_watcher.go
 func (w *fileWatcher) seekToEnd() error {
 	file, err := os.Open(w.path)
 	if err != nil {

src/internal/source/http.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/source/http.go
+// FILE: logwisp/src/internal/source/http.go
 package source
 
 import (

src/internal/source/source.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/source/source.go
+// FILE: logwisp/src/internal/source/source.go
 package source
 
 import (

src/internal/source/stdin.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/source/stdin.go
+// FILE: logwisp/src/internal/source/stdin.go
 package source
 
 import (

src/internal/source/tcp.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/source/tcp.go
+// FILE: logwisp/src/internal/source/tcp.go
 package source
 
 import (
@@ -15,6 +15,7 @@ import (
 	"logwisp/src/internal/netlimit"
 
 	"github.com/lixenwraith/log"
+	"github.com/lixenwraith/log/compat"
 	"github.com/panjf2000/gnet/v2"
 )
 
@@ -109,7 +110,11 @@ func (t *TCPSource) Start() error {
 
 	addr := fmt.Sprintf("tcp://:%d", t.port)
 
-	// Start gnet server in background
+	// Create a gnet adapter using the existing logger instance
+	gnetLogger := compat.NewGnetAdapter(t.logger)
+
+	// Start gnet server
+	errChan := make(chan error, 1)
 	t.wg.Add(1)
 	go func() {
 		defer t.wg.Done()
@@ -118,7 +123,7 @@ func (t *TCPSource) Start() error {
 			"port", t.port)
 
 		err := gnet.Run(t.server, addr,
-			gnet.WithLogger(noopLogger{}),
+			gnet.WithLogger(gnetLogger),
 			gnet.WithMulticore(true),
 			gnet.WithReusePort(true),
 		)
@@ -128,12 +133,22 @@ func (t *TCPSource) Start() error {
 				"port", t.port,
 				"error", err)
 		}
+		errChan <- err
 	}()
 
-	// Give server time to start
-	time.Sleep(100 * time.Millisecond)
+	// Wait briefly for server to start or fail
+	select {
+	case err := <-errChan:
+		// Server failed immediately
+		close(t.done)
+		t.wg.Wait()
+		return err
+	case <-time.After(100 * time.Millisecond):
+		// Server started successfully
+		t.logger.Info("msg", "TCP server started", "port", t.port)
 		return nil
 	}
+}
 
 func (t *TCPSource) Stop() {
 	t.logger.Info("msg", "Stopping TCP source")
@@ -378,10 +393,11 @@ func (s *tcpSourceServer) OnTraffic(c gnet.Conn) gnet.Action {
 }
 
 // noopLogger implements gnet's Logger interface but discards everything
-type noopLogger struct{}
+// type noopLogger struct{}
+// func (n noopLogger) Debugf(format string, args ...any) {}
+// func (n noopLogger) Infof(format string, args ...any)  {}
+// func (n noopLogger) Warnf(format string, args ...any)  {}
+// func (n noopLogger) Errorf(format string, args ...any) {}
+// func (n noopLogger) Fatalf(format string, args ...any) {}
 
-func (n noopLogger) Debugf(format string, args ...any) {}
-func (n noopLogger) Infof(format string, args ...any)  {}
-func (n noopLogger) Warnf(format string, args ...any)  {}
-func (n noopLogger) Errorf(format string, args ...any) {}
-func (n noopLogger) Fatalf(format string, args ...any) {}
+// Usage: gnet.Run(..., gnet.WithLogger(noopLogger{}), ...)

src/internal/version/version.go

@@ -1,4 +1,4 @@
-// FILE: src/internal/version/version.go
+// FILE: logwisp/src/internal/version/version.go
 package version
 
 import "fmt"