diff --git a/.gitignore b/.gitignore index 24efab4..cebbce6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,14 @@ .idea -data -dev -log -logs -cert -bin -script -build +data/ +dev/ +log/ +logs/ +cert/ +bin/ +script/ +build/ *.log *.toml build.sh catalog.txt +combined.txt diff --git a/config/logwisp.toml.defaults b/config/logwisp.toml.defaults deleted file mode 100644 index 8a80a4f..0000000 --- a/config/logwisp.toml.defaults +++ /dev/null @@ -1,372 +0,0 @@ -############################################################################### -### LogWisp Configuration -### Default location: ~/.config/logwisp/logwisp.toml -### Configuration Precedence: CLI flags > Environment > File > Defaults -### Default values shown - uncommented lines represent active configuration -############################################################################### - -############################################################################### -### Global Settings -############################################################################### - -quiet = false # Enable quiet mode, suppress console output -status_reporter = true # Enable periodic status logging -auto_reload = false # Enable config auto-reload on file change - -############################################################################### -### Logging Configuration (LogWisp's internal operational logging) -############################################################################### - -[logging] -output = "stdout" # file|stdout|stderr|split|all|none -level = "info" # debug|info|warn|error - -# [logging.file] -# directory = "./log" # Log directory path -# name = "logwisp" # Base filename -# max_size_mb = 100 # Rotation threshold -# max_total_size_mb = 1000 # Total size limit -# retention_hours = 168.0 # Delete logs older than (7 days) - -[logging.console] -target = "stdout" # stdout|stderr|split -format = "txt" # txt|json - -############################################################################### -### Pipeline Configuration -### Each pipeline: sources -> rate_limit -> filters -> format -> sinks -############################################################################### - -[[pipelines]] -name = "default" # Pipeline identifier - -###============================================================================ -### Rate Limiting (Pipeline-level) -###============================================================================ - -# [pipelines.rate_limit] -# rate = 1000.0 # Entries per second (0=disabled) -# burst = 2000.0 # Burst capacity (defaults to rate) -# policy = "drop" # pass|drop -# max_entry_size_bytes = 0 # Max entry size (0=unlimited) - -###============================================================================ -### Filters (Sequential pattern matching) -###============================================================================ - -### ⚠️ Example: Include only ERROR and WARN logs -## [[pipelines.filters]] -## type = "include" # include|exclude -## logic = "or" # or|and -## patterns = [".*ERROR.*", ".*WARN.*"] - -### ⚠️ Example: Exclude debug logs -## [[pipelines.filters]] -## type = "exclude" -## patterns = [".*DEBUG.*"] - -###============================================================================ -### Format (Log transformation) -###============================================================================ - -# [pipelines.format] -# type = "raw" # raw|json|txt - -## JSON formatting -# [pipelines.format.json] -# pretty = false # Pretty-print JSON -# timestamp_field = "timestamp" # Field name for timestamp -# level_field = "level" # Field name for log level -# message_field = "message" # Field name for message -# source_field = "source" # Field name for source - -## Text templating -# [pipelines.format.txt] -# template = "{{.Timestamp | FmtTime}} [{{.Level}}] {{.Message}}" -# timestamp_format = "2006-01-02 15:04:05" - -## Raw templating -# [pipelines.format.raw] -# add_new_line = true # Preserve new line delimiter between log entries - -###============================================================================ -### SOURCES (Inputs) -### Architecture: Pipeline can have multiple sources -###============================================================================ - -###---------------------------------------------------------------------------- -### File Source (File monitoring) -[[pipelines.sources]] -type = "file" - -[pipelines.sources.file] -directory = "./" # Directory to monitor -pattern = "*.log" # Glob pattern -check_interval_ms = 100 # File check interval -recursive = false # Recursive monitoring (TODO) - -###---------------------------------------------------------------------------- -### Console Source -# [[pipelines.sources]] -# type = "console" - -# [pipelines.sources.console] -# buffer_size = 1000 - -###---------------------------------------------------------------------------- -### HTTP Source (Server mode - receives logs via HTTP POST) -# [[pipelines.sources]] -# type = "http" - -# [pipelines.sources.http] -# host = "0.0.0.0" # Listen interface -# port = 8081 # Listen port -# ingest_path = "/ingest" # Ingestion endpoint -# buffer_size = 1000 -# max_body_size = 1048576 # 1MB -# read_timeout_ms = 10000 -# write_timeout_ms = 10000 - -### Network access control -# [pipelines.sources.http.acl] -# enabled = false -# max_connections_per_ip = 10 # Max simultaneous connections from a single IP -# max_connections_total = 100 # Max simultaneous connections for this component -# requests_per_second = 100.0 # Per-IP request rate limit -# burst_size = 200 # Per-IP request burst limit -# response_message = "Rate limit exceeded" -# response_code = 429 -# ip_whitelist = ["192.168.1.0/24"] -# ip_blacklist = ["10.0.0.100"] - -### TLS configuration (mTLS support) -# [pipelines.sources.http.tls] -# enabled = false -# cert_file = "/path/to/server.pem" # Server certificate -# key_file = "/path/to/server.key" # Server private key -# client_auth = false # Enable mTLS -# client_ca_file = "/path/to/ca.pem" # CA for client verification -# verify_client_cert = true # Verify client certificates -# min_version = "TLS1.2" # TLS1.0|TLS1.1|TLS1.2|TLS1.3 -# max_version = "TLS1.3" -# cipher_suites = "" # Comma-separated cipher list - -###---------------------------------------------------------------------------- -### TCP Source (Server mode - receives logs via TCP) -# [[pipelines.sources]] -# type = "tcp" - -# [pipelines.sources.tcp] -# host = "0.0.0.0" -# port = 9091 -# buffer_size = 1000 -# read_timeout_ms = 10000 -# keep_alive = true -# keep_alive_period_ms = 30000 - -### Network access control -# [pipelines.sources.tcp.acl] -# enabled = false -# max_connections_per_ip = 10 # Max simultaneous connections from a single IP -# max_connections_total = 100 # Max simultaneous connections for this component -# requests_per_second = 100.0 # Per-IP request rate limit -# burst_size = 200 # Per-IP request burst limit -# response_message = "Rate limit exceeded" -# response_code = 429 -# ip_whitelist = ["192.168.1.0/24"] -# ip_blacklist = ["10.0.0.100"] - -### ⚠️ IMPORTANT: TCP does NOT support TLS/mTLS (gnet limitation) -### Use HTTP Source with TLS for encrypted transport - -###============================================================================ -### SINKS (Outputs) -### Architecture: Pipeline can have multiple sinks (fan-out) -###============================================================================ - -###---------------------------------------------------------------------------- -### Console Sink -# [[pipelines.sinks]] -# type = "console" - -# [pipelines.sinks.console] -# target = "stdout" # stdout|stderr|split -# colorize = false # Colorized output -# buffer_size = 100 - -###---------------------------------------------------------------------------- -### File Sink (Rotating logs) -# [[pipelines.sinks]] -# type = "file" - -# [pipelines.sinks.file] -# directory = "./logs" -# name = "output" -# max_size_mb = 100 -# max_total_size_mb = 1000 -# min_disk_free_mb = 100 -# retention_hours = 168.0 # 7 days -# buffer_size = 1000 -# flush_interval_ms = 1000 - -###---------------------------------------------------------------------------- -### HTTP Sink (Server mode - SSE streaming for clients) -[[pipelines.sinks]] -type = "http" - -[pipelines.sinks.http] -host = "0.0.0.0" -port = 8080 -stream_path = "/stream" # SSE streaming endpoint -status_path = "/status" # Status endpoint -buffer_size = 1000 -write_timeout_ms = 10000 - -### Heartbeat configuration (keep connections alive) -[pipelines.sinks.http.heartbeat] -enabled = true -interval_ms = 30000 # 30 seconds -include_timestamp = true -include_stats = false -format = "comment" # comment|event|json - -### Network access control -# [pipelines.sinks.http.acl] -# enabled = false -# max_connections_per_ip = 10 # Max simultaneous connections from a single IP -# max_connections_total = 100 # Max simultaneous connections for this component -# requests_per_second = 100.0 # Per-IP request rate limit -# burst_size = 200 # Per-IP request burst limit -# response_message = "Rate limit exceeded" -# response_code = 429 -# ip_whitelist = ["192.168.1.0/24"] -# ip_blacklist = ["10.0.0.100"] - -### TLS configuration (mTLS support) -# [pipelines.sinks.http.tls] -# enabled = false -# cert_file = "/path/to/server.pem" # Server certificate -# key_file = "/path/to/server.key" # Server private key -# client_auth = false # Enable mTLS -# client_ca_file = "/path/to/ca.pem" # CA for client verification -# verify_client_cert = true # Verify client certificates -# min_version = "TLS1.2" # TLS1.0|TLS1.1|TLS1.2|TLS1.3 -# max_version = "TLS1.3" -# cipher_suites = "" # Comma-separated cipher list - -###---------------------------------------------------------------------------- -### TCP Sink (Server mode - TCP streaming for clients) -# [[pipelines.sinks]] -# type = "tcp" - -# [pipelines.sinks.tcp] -# host = "0.0.0.0" -# port = 9090 -# buffer_size = 1000 -# write_timeout_ms = 10000 -# keep_alive = true -# keep_alive_period_ms = 30000 - -### Heartbeat configuration -# [pipelines.sinks.tcp.heartbeat] -# enabled = false -# interval_ms = 30000 -# include_timestamp = true -# include_stats = false -# format = "json" # json|txt - -### Network access control -# [pipelines.sinks.tcp.acl] -# enabled = false -# max_connections_per_ip = 10 # Max simultaneous connections from a single IP -# max_connections_total = 100 # Max simultaneous connections for this component -# requests_per_second = 100.0 # Per-IP request rate limit -# burst_size = 200 # Per-IP request burst limit -# response_message = "Rate limit exceeded" -# response_code = 429 -# ip_whitelist = ["192.168.1.0/24"] -# ip_blacklist = ["10.0.0.100"] - -### ⚠️ IMPORTANT: TCP does NOT support TLS/mTLS (gnet limitation) -### Use HTTP Sink with TLS for encrypted transport - -###---------------------------------------------------------------------------- -### HTTP Client Sink (Forward to remote HTTP endpoint) -# [[pipelines.sinks]] -# type = "http_client" - -# [pipelines.sinks.http_client] -# url = "https://logs.example.com/ingest" -# buffer_size = 1000 -# batch_size = 100 # Entries per batch -# batch_delay_ms = 1000 # Max wait before sending -# timeout_seconds = 30 -# max_retries = 3 -# retry_delay_ms = 1000 -# retry_backoff = 2.0 # Exponential backoff multiplier -# insecure_skip_verify = false # Skip TLS verification - -### TLS configuration for client -# [pipelines.sinks.http_client.tls] -# enabled = false # Enable TLS for the outgoing connection -# server_ca_file = "/path/to/ca.pem" # CA for verifying the remote server's certificate -# server_name = "logs.example.com" # For server certificate validation (SNI) -# insecure_skip_verify = false # Skip server verification, use with caution -# client_cert_file = "/path/to/client.pem" # Client's certificate to present to the server for mTLS -# client_key_file = "/path/to/client.key" # Client's private key for mTLS -# min_version = "TLS1.2" -# max_version = "TLS1.3" -# cipher_suites = "" - -### ⚠️ Example: HTTP Client Sink → HTTP Source with mTLS -## HTTP Source with mTLS: -## [pipelines.sources.http.tls] -## enabled = true -## cert_file = "/path/to/server.pem" -## key_file = "/path/to/server.key" -## client_auth = true # Enable client cert verification -## client_ca_file = "/path/to/ca.pem" -## verify_client_cert = true - -## HTTP Client with client cert: -## [pipelines.sinks.http_client.tls] -## enabled = true -## server_ca_file = "/path/to/ca.pem" # Verify server -## client_cert_file = "/path/to/client.pem" # Client certificate -## client_key_file = "/path/to/client.key" - -###---------------------------------------------------------------------------- -### TCP Client Sink (Forward to remote TCP endpoint) -# [[pipelines.sinks]] -# type = "tcp_client" - -# [pipelines.sinks.tcp_client] -# host = "logs.example.com" -# port = 9090 -# buffer_size = 1000 -# dial_timeout_seconds = 10 # Connection timeout -# write_timeout_seconds = 30 # Write timeout -# read_timeout_seconds = 10 # Read timeout -# keep_alive_seconds = 30 # TCP keep-alive -# reconnect_delay_ms = 1000 # Initial reconnect delay -# max_reconnect_delay_ms = 30000 # Max reconnect delay -# reconnect_backoff = 1.5 # Exponential backoff - -### ⚠️ WARNING: TCP Client has NO TLS support -### Use HTTP Client with TLS for encrypted transport - -############################################################################### -### Common Usage Patterns -############################################################################### - -### Pattern 1: Log Aggregation (Client → Server) -### - HTTP Client Sink → HTTP Source (with optional TLS/mTLS) -### - TCP Client Sink → TCP Source (unencrypted only) - -### Pattern 2: Live Monitoring -### - HTTP Sink: Browser-based SSE streaming (https://host:8080/stream) -### - TCP Sink: Debug interface (telnet/netcat to port 9090) - -### Pattern 3: Log Collection & Distribution -### - File Source → Multiple Sinks (fan-out) -### - Multiple Sources → Single Pipeline → Multiple Sinks \ No newline at end of file diff --git a/doc/README.md b/doc/README.md index a3d94bc..294c569 100644 --- a/doc/README.md +++ b/doc/README.md @@ -6,24 +6,20 @@ A pipeline-based log transport and processing system built in Go. LogWisp provid ### Core Capabilities - **Pipeline Architecture**: Independent processing pipelines with source(s) → filter → format → sink(s) flow -- **Multiple Input Sources**: Directory monitoring, stdin, HTTP, TCP -- **Flexible Output Sinks**: Console, file, HTTP SSE, TCP streaming, HTTP/TCP forwarding +- **Multiple Input Sources**: File monitoring, console (stdin), random log generation, null +- **Flexible Output Sinks**: Console, file, HTTP SSE, TCP streaming, null - **Real-time Processing**: Sub-millisecond latency with configurable buffering - **Hot Configuration Reload**: Update pipelines without service restart +- **Session Management**: Built-in session tracking for multiple client connections ### Data Processing - **Pattern-based Filtering**: Chainable include/exclude filters with regex support -- **Multiple Formatters**: Raw, JSON, and template-based text formatting +- **Multiple Formatters**: Raw, JSON, and text formatting with integrated sanitizer policies - **Rate Limiting**: Pipeline rate controls +- **Heartbeat Generation**: Flow-level heartbeat events for keep-alives ### Security & Reliability -- **Authentication**: mTLS support -- **Access Control**: IP whitelisting/blacklisting, connection limits -- **TLS Encryption**: Full TLS 1.2/1.3 support for HTTP connections -- **Automatic Reconnection**: Resilient client connections with exponential backoff - **File Rotation**: Size-based rotation with retention policies - -### Operational Features - **Status Monitoring**: Real-time statistics and health endpoints - **Signal Handling**: Graceful shutdown and configuration reload via signals - **Background Mode**: Daemon operation with proper signal handling @@ -38,8 +34,7 @@ A pipeline-based log transport and processing system built in Go. LogWisp provid - [Output Sinks](sinks.md) - Sink types and output options - [Filters](filters.md) - Pattern-based log filtering - [Formatters](formatters.md) - Log formatting and transformation -- [Security](security.md) - IP-based access control configuration and mTLS -- [Networking](networking.md) - TLS, rate limiting, and network features +- [Networking & Security](networking.md) - Network features (Note: TLS and Auth are currently placeholders in the new architecture) - [Command Line Interface](cli.md) - CLI flags and subcommands - [Operations Guide](operations.md) - Running and maintaining LogWisp @@ -51,15 +46,17 @@ Install LogWisp and create a basic configuration: [[pipelines]] name = "default" -[[pipelines.sources]] -type = "directory" -[pipelines.sources.directory] -path = "./" +[[pipelines.plugin_sources]] +id = "default_source" +type = "file" +[pipelines.plugin_sources.config] +directory = "./" pattern = "*.log" -[[pipelines.sinks]] +[[pipelines.plugin_sinks]] +id = "default_sink" type = "console" -[pipelines.sinks.console] +[pipelines.plugin_sinks.config] target = "stdout" ``` @@ -73,4 +70,4 @@ Run with: `logwisp -c config.toml` ## License -BSD 3-Clause License \ No newline at end of file +BSD 3-Clause License diff --git a/doc/architecture.md b/doc/architecture.md index 6a19d62..9b26436 100644 --- a/doc/architecture.md +++ b/doc/architecture.md @@ -13,13 +13,15 @@ Each pipeline operates independently with a source → filter → format → sin ``` Service (Main Process) ├── Pipeline 1 -│ ├── Sources (1 or more) -│ ├── Rate Limiter (optional) -│ ├── Filter Chain (optional) -│ ├── Formatter (optional) -│ └── Sinks (1 or more) +│ ├── Plugin Sources (1 or more) +│ ├── Flow +│ │ ├── Heartbeat Generator (optional) +│ │ ├── Rate Limiter (optional) +│ │ ├── Filter Chain (optional) +│ │ └── Formatter (optional) +│ └── Plugin Sinks (1 or more) ├── Pipeline 2 -│ └── [Same structure] +│ └── [Similar structure] └── Status Reporter (optional) ``` @@ -27,11 +29,11 @@ Service (Main Process) ### Processing Stages -1. **Source Stage**: Sources monitor inputs and generate log entries -2. **Rate Limiting**: Optional pipeline-level rate control -3. **Filtering**: Pattern-based inclusion/exclusion -4. **Formatting**: Transform entries to desired output format -5. **Distribution**: Fan-out to multiple sinks +1. **Source Stage**: Plugin sources monitor inputs and generate log entries +2. **Flow - Rate Limiting**: Optional pipeline-level rate control +3. **Flow - Filtering**: Pattern-based inclusion/exclusion +4. **Flow - Formatting**: Transform entries to desired output format with sanitization +5. **Distribution**: Fan-out to multiple plugin sinks ### Entry Lifecycle @@ -50,14 +52,16 @@ Each component maintains internal buffers to handle burst traffic: - Sinks: Independent buffers per sink - Network components: Additional TCP/HTTP buffers +*"Sink dispatch uses non-blocking sends. When a sink's input buffer is full, the event is dropped for that sink only and counted in pipeline statistics (`total_dropped_by_sink`). The policy is uniform and not configurable; a slow sink does not stall the pipeline or other sinks."* + ## Component Types ### Sources (Input) -- **Directory Source**: File system monitoring with rotation detection -- **Stdin Source**: Standard input processing -- **HTTP Source**: REST endpoint for log ingestion -- **TCP Source**: Raw TCP socket listener +- **File Source**: File system directory monitoring with rotation detection +- **Console Source**: Standard input processing (stdin) +- **Random Source**: Generates random log entries for testing +- **Null Source**: Discards logs, used for testing ### Sinks (Output) @@ -65,14 +69,13 @@ Each component maintains internal buffers to handle burst traffic: - **File Sink**: Rotating file writer - **HTTP Sink**: Server-Sent Events (SSE) streaming - **TCP Sink**: TCP server for client connections -- **HTTP Client Sink**: Forward to remote HTTP endpoints -- **TCP Client Sink**: Forward to remote TCP servers +- **Null Sink**: Discards all received events ### Processing Components - **Rate Limiter**: Token bucket algorithm for flow control - **Filter Chain**: Sequential pattern matching -- **Formatters**: Raw, JSON, or template-based text transformation +- **Formatters**: Raw, JSON, or text transformation with sanitizer policies ## Concurrency Model @@ -95,8 +98,7 @@ Each component maintains internal buffers to handle burst traffic: ### Connection Patterns **Chaining Design**: -- TCP Client Sink → TCP Source: Direct TCP forwarding -- HTTP Client Sink → HTTP Source: HTTP-based forwarding +- Future plan **Monitoring Design**: - TCP Sink: Debugging interface @@ -106,8 +108,6 @@ Each component maintains internal buffers to handle burst traffic: - HTTP/1.1 and HTTP/2 for HTTP connections - Raw TCP connections -- TLS 1.2/1.3 for HTTPS connections (HTTP only) -- Server-Sent Events for real-time streaming ## Resource Management @@ -125,6 +125,10 @@ Each component maintains internal buffers to handle burst traffic: ### Connection Management +- HTTP sink silenty drops IPv6 connections (deliberate IPv4-only enforcement) + +*Note:* Placeholder; below features are removed in restructuring and will be added in the future release. + - Per-IP connection limits - Global connection caps - Automatic reconnection with exponential backoff @@ -136,7 +140,6 @@ Each component maintains internal buffers to handle burst traffic: - Panic recovery in pipeline processing - Independent pipeline operation -- Automatic source restart on failure - Sink failure isolation ### Data Integrity @@ -165,4 +168,4 @@ Each component maintains internal buffers to handle burst traffic: - Horizontal: Multiple LogWisp instances with different configurations - Vertical: Multiple pipelines per instance - Fan-out: Multiple sinks per pipeline -- Fan-in: Multiple sources per pipeline \ No newline at end of file +- Fan-in: Multiple sources per pipeline diff --git a/doc/cli.md b/doc/cli.md index f385e77..f557372 100644 --- a/doc/cli.md +++ b/doc/cli.md @@ -15,30 +15,8 @@ logwisp [options] | Command | Description | |---------|-------------| -| `tls` | Generate TLS certificates | -| `version` | Display version information | -| `help` | Show help information | - -### tls Command - -Generate TLS certificates. - -```bash -logwisp tls [options] -``` - -**Options:** - -| Flag | Description | Default | -|------|-------------|---------| -| `-ca` | Generate CA certificate | - | -| `-server` | Generate server certificate | - | -| `-client` | Generate client certificate | - | -| `-host` | Comma-separated hosts/IPs | localhost | -| `-o` | Output file prefix | Required | -| `-ca-cert` | CA certificate file | Required for server/client | -| `-ca-key` | CA key file | Required for server/client | -| `-days` | Certificate validity days | 365 | +| `--version` | Display version information | +| `--help` | Show help information | ### version Command @@ -63,10 +41,9 @@ Output includes: | Flag | Description | Default | |------|-------------|---------| | `-c, --config` | Configuration file path | `./logwisp.toml` | -| `-b, --background` | Run as daemon | false | | `-q, --quiet` | Suppress console output | false | -| `--disable-status-reporter` | Disable status logging | false | -| `--config-auto-reload` | Enable config hot reload | false | +| `--status-reporter` | Status logging | true | +| `--auto-reload` | Enable config hot reload | false | ### Logging Options @@ -91,9 +68,9 @@ Configure pipelines via CLI (N = array index, 0-based). | Flag | Description | |------|-------------| | `--pipelines.N.name` | Pipeline name | -| `--pipelines.N.sources.N.type` | Source type | -| `--pipelines.N.filters.N.type` | Filter type | -| `--pipelines.N.sinks.N.type` | Sink type | +| `--pipelines.N.plugin_sources.N.type` | Source type | +| `--pipelines.N.flow.filters.N.type` | Filter type | +| `--pipelines.N.plugin_sinks.N.type` | Sink type | ## Flag Formats @@ -102,7 +79,7 @@ Configure pipelines via CLI (N = array index, 0-based). ```bash logwisp --quiet logwisp --quiet=true -logwisp --quiet=false +logwisp --pipelines.0.plugin_sources.0.type=console ``` ### String Flags @@ -117,13 +94,13 @@ logwisp -c config.toml ```bash logwisp --logging.level=debug logwisp --pipelines.0.name=myapp -logwisp --pipelines.0.sources.0.type=stdin +logwisp --pipelines.0.sources.0.type=console ``` ### Array Values (JSON) ```bash -logwisp --pipelines.0.filters.0.patterns='["ERROR","WARN"]' +logwisp --pipelines.0.flow.filters.0.patterns='["ERROR","WARN"]' ``` ## Environment Variables @@ -171,7 +148,7 @@ export LOGWISP_PIPELINES_0_NAME=myapp logwisp --logging.output=stderr --logging.level=debug # Quick test with stdin -logwisp --pipelines.0.sources.0.type=stdin --pipelines.0.sinks.0.type=console +logwisp --pipelines.0.plugin_sources.0.type=console --pipelines.0.plugin_sinks.0.type=console ``` ### Production Deployment @@ -194,19 +171,6 @@ logwisp --config test.toml --logging.level=debug --disable-status-reporter logwisp --config test.toml --quiet ``` -### Quick Commands - -```bash -# Generate admin password -logwisp auth -u admin -b - -# Create self-signed certs -logwisp tls -server -host localhost -o server - -# Check version -logwisp version -``` - ## Help System ### General Help @@ -217,14 +181,6 @@ logwisp -h logwisp help ``` -### Command Help - -```bash -logwisp auth --help -logwisp tls --help -logwisp help auth -``` - ## Special Flags ### Internal Flags @@ -235,6 +191,6 @@ These flags are for internal use: ### Hidden Behaviors -- SIGHUP ignored by default (nohup behavior) +- SIGHUP ignored ignored during startup (after startup triggers config reload) - Automatic panic recovery in pipelines -- Resource cleanup on shutdown \ No newline at end of file +- Resource cleanup on shutdown diff --git a/doc/configuration.md b/doc/configuration.md index 5c0642f..219a254 100644 --- a/doc/configuration.md +++ b/doc/configuration.md @@ -24,10 +24,9 @@ Top-level configuration options: | Setting | Type | Default | Description | |---------|------|---------|-------------| -| `background` | bool | false | Run as daemon process | | `quiet` | bool | false | Suppress console output | -| `disable_status_reporter` | bool | false | Disable periodic status logging | -| `config_auto_reload` | bool | false | Enable file watch for auto-reload | +| `status_reporter` | bool | true | Periodic status logging | +| `auto_reload` | bool | false | Enable file watch for auto-reload | ## Logging Configuration @@ -47,7 +46,6 @@ retention_hours = 168.0 [logging.console] target = "stdout" # stdout|stderr|split -format = "txt" # txt|json ``` ### Output Modes @@ -68,30 +66,34 @@ Each `[[pipelines]]` section defines an independent processing pipeline: name = "pipeline-name" # Rate limiting (optional) -[pipelines.rate_limit] +[pipelines.flow.rate_limit] rate = 1000.0 burst = 2000.0 policy = "drop" # pass|drop max_entry_size_bytes = 0 # 0=unlimited # Format configuration (optional) -[pipelines.format] +[pipelines.flow.format] type = "json" # raw|json|txt +sanitizer_policy = "json" -# Sources (required, 1+) -[[pipelines.sources]] -type = "directory" +[[pipelines.plugin_sources]] +id = "my_source" +type = "file" +[pipelines.plugin_sources.config] # ... source-specific config # Filters (optional) -[[pipelines.filters]] +[[pipelines.flow.filters]] type = "include" logic = "or" patterns = ["ERROR", "WARN"] # Sinks (required, 1+) -[[pipelines.sinks]] +[[pipelines.plugin_sinks]] +id = "my_sink" type = "http" +[pipelines.plugin_sinks.config] # ... sink-specific config ``` @@ -113,7 +115,7 @@ All configuration options support environment variable overrides: | `quiet` | `LOGWISP_QUIET` | | `logging.level` | `LOGWISP_LOGGING_LEVEL` | | `pipelines[0].name` | `LOGWISP_PIPELINES_0_NAME` | -| `pipelines[0].sources[0].type` | `LOGWISP_PIPELINES_0_SOURCES_0_TYPE` | +| `pipelines[0].plugin_sources[0].type` | `LOGWISP_PIPELINES_0_PLUGIN_SOURCES_0_TYPE` | ## Command-Line Overrides @@ -123,13 +125,15 @@ All configuration options can be overridden via CLI flags: logwisp --quiet \ --logging.level=debug \ --pipelines.0.name=myapp \ - --pipelines.0.sources.0.type=stdin + --pipelines.0.plugin_sources.0.type=console ``` ## Configuration Validation LogWisp validates configuration at startup: -- Required fields presence +- Rpipelines non-empty, name non-empty, ≥1 source, ≥1 sink, logging enum values.equired fields presence + +Partial check in plugin constructor: - Type correctness - Port conflicts - Path accessibility @@ -141,12 +145,12 @@ LogWisp validates configuration at startup: Enable configuration hot reload: ```toml -config_auto_reload = true +auto_reload = true ``` Or via command line: ```bash -logwisp --config-auto-reload +logwisp --auto-reload ``` Reload triggers: @@ -161,7 +165,6 @@ Reloadable items: Non-reloadable (requires restart): - Logging configuration -- Background mode - Global settings ## Default Configuration @@ -172,15 +175,17 @@ Minimal working configuration: [[pipelines]] name = "default" -[[pipelines.sources]] -type = "directory" -[pipelines.sources.directory] -path = "./" +[[pipelines.plugin_sources]] +id = "default_source" +type = "file" +[pipelines.plugin_sources.config] +directory = "./" pattern = "*.log" -[[pipelines.sinks]] +[[pipelines.plugin_sinks]] +id = "default_sink" type = "console" -[pipelines.sinks.console] +[pipelines.plugin_sinks.config] target = "stdout" ``` @@ -195,4 +200,4 @@ target = "stdout" | Float | float64 | Decimal string | | Boolean | bool | true/false | | Array | []T | JSON array string | -| Table | struct | Nested with `_` | \ No newline at end of file +| Table | struct | Nested with `_` | diff --git a/doc/filters.md b/doc/filters.md index c3710fe..961700b 100644 --- a/doc/filters.md +++ b/doc/filters.md @@ -9,7 +9,7 @@ LogWisp filters control which log entries pass through the pipeline using patter Only entries matching patterns pass through. ```toml -[[pipelines.filters]] +[[pipelines.flow.filters]] type = "include" logic = "or" # or|and patterns = [ @@ -24,7 +24,7 @@ patterns = [ Entries matching patterns are dropped. ```toml -[[pipelines.filters]] +[[pipelines.flow.filters]] type = "exclude" patterns = [ "DEBUG", @@ -89,12 +89,12 @@ Multiple filters execute sequentially: ```toml # First filter: Include errors and warnings -[[pipelines.filters]] +[[pipelines.flow.filters]] type = "include" patterns = ["ERROR", "WARN"] # Second filter: Exclude test environments -[[pipelines.filters]] +[[pipelines.flow.filters]] type = "exclude" patterns = ["test-env", "staging"] ``` @@ -137,14 +137,14 @@ patterns = ["ERROR", "WARN", "FATAL", "CRITICAL"] ### Application Filtering ```toml -[[pipelines.filters]] +[[pipelines.flow.filters]] type = "include" patterns = ["app1", "app2", "app3"] ``` ### Noise Reduction ```toml -[[pipelines.filters]] +[[pipelines.flow.filters]] type = "exclude" patterns = [ "health-check", @@ -156,7 +156,7 @@ patterns = [ ### Security Filtering ```toml -[[pipelines.filters]] +[[pipelines.flow.filters]] type = "exclude" patterns = [ "password", @@ -169,17 +169,17 @@ patterns = [ ### Multi-stage Filtering ```toml # Include production logs -[[pipelines.filters]] +[[pipelines.flow.filters]] type = "include" patterns = ["prod-", "production"] # Include only errors -[[pipelines.filters]] +[[pipelines.flow.filters]] type = "include" patterns = ["ERROR", "EXCEPTION", "FATAL"] # Exclude known issues -[[pipelines.filters]] +[[pipelines.flow.filters]] type = "exclude" patterns = ["ECONNRESET", "broken pipe"] -``` \ No newline at end of file +``` diff --git a/doc/formatters.md b/doc/formatters.md index 210a0ce..0918559 100644 --- a/doc/formatters.md +++ b/doc/formatters.md @@ -9,11 +9,10 @@ LogWisp formatters transform log entries before output to sinks. Outputs the log message as-is with optional newline. ```toml -[pipelines.format] +[pipelines.flow.format] type = "raw" - -[pipelines.format.raw] -add_new_line = true +sanitizer_policy = "raw" +flags = 1 ``` **Configuration Options:** @@ -21,6 +20,9 @@ add_new_line = true | Option | Type | Default | Description | |--------|------|---------|-------------| | `add_new_line` | bool | true | Append newline to messages | +| `type` | string | "raw" | raw, json, or txt | +| `flags` | int64 | 0 | log/formatter flags override | +| `sanitizer_policy` | string | | Sanitizer policy (e.g. "json", "raw", "txt", "shell") | ### JSON Formatter @@ -30,24 +32,11 @@ Produces structured JSON output. [pipelines.format] type = "json" -[pipelines.format.json] -pretty = false -timestamp_field = "timestamp" -level_field = "level" -message_field = "message" -source_field = "source" ++[pipelines.flow.format] + type = "json" +sanitizer_policy = "json" ``` -**Configuration Options:** - -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `pretty` | bool | false | Pretty print JSON | -| `timestamp_field` | string | "timestamp" | Field name for timestamp | -| `level_field` | string | "level" | Field name for log level | -| `message_field` | string | "message" | Field name for message | -| `source_field` | string | "source" | Field name for source | - **Output Structure:** ```json { @@ -63,11 +52,9 @@ source_field = "source" Template-based text formatting. ```toml -[pipelines.format] +[pipelines.flow.format] type = "txt" - -[pipelines.format.txt] -template = "[{{.Timestamp | FmtTime}}] [{{.Level | ToUpper}}] {{.Source}} - {{.Message}}" +sanitizer_policy = "txt" timestamp_format = "2006-01-02T15:04:05.000Z07:00" ``` @@ -75,8 +62,7 @@ timestamp_format = "2006-01-02T15:04:05.000Z07:00" | Option | Type | Default | Description | |--------|------|---------|-------------| -| `template` | string | See below | Go template string | -| `timestamp_format` | string | RFC3339 | Go time format string | +| `timestamp_format` | string | "" | Time format override | **Default Template:** ``` @@ -134,12 +120,12 @@ Each pipeline can have its own formatter: ```toml [[pipelines]] name = "json-pipeline" -[pipelines.format] +[pipelines.flow.format] type = "json" [[pipelines]] name = "text-pipeline" -[pipelines.format] +[pipelines.flow.format] type = "txt" ``` @@ -182,34 +168,13 @@ Relative performance (fastest to slowest): ### Structured Logging ```toml -[pipelines.format] +[pipelines.flow.format] type = "json" -[pipelines.format.json] -pretty = false ``` ### Human-Readable Logs ```toml -[pipelines.format] +[pipelines.flow.format] type = "txt" -[pipelines.format.txt] -template = "{{.Timestamp | FmtTime}} [{{.Level}}] {{.Message}}" timestamp_format = "15:04:05" ``` - -### Syslog Format -```toml -[pipelines.format] -type = "txt" -[pipelines.format.txt] -template = "{{.Timestamp | FmtTime}} {{.Source}} {{.Level}}: {{.Message}}" -timestamp_format = "Jan 2 15:04:05" -``` - -### Minimal Output -```toml -[pipelines.format] -type = "txt" -[pipelines.format.txt] -template = "{{.Message}}" -``` \ No newline at end of file diff --git a/doc/networking.md b/doc/networking.md index ce786b5..272a892 100644 --- a/doc/networking.md +++ b/doc/networking.md @@ -1,173 +1,40 @@ # Networking -Network configuration for LogWisp connections, including TLS, rate limiting, and access control. +*Note: Under redesign* ## TLS Configuration -### TLS Support Matrix - -| Component | TLS Support | Notes | -|-----------|-------------|-------| -| HTTP Source | ✓ | Full TLS 1.2/1.3 | -| HTTP Sink | ✓ | Full TLS 1.2/1.3 | -| HTTP Client | ✓ | Client certificates | -| TCP Source | ✗ | No encryption | -| TCP Sink | ✗ | No encryption | -| TCP Client | ✗ | No encryption | - -### Server TLS Configuration - -```toml -[pipelines.sources.http.tls] -enabled = true -cert_file = "/path/to/server.pem" -key_file = "/path/to/server.key" -min_version = "TLS1.2" # TLS1.2|TLS1.3 -client_auth = false -client_ca_file = "/path/to/client-ca.pem" -verify_client_cert = true -``` - -### Client TLS Configuration - -```toml -[pipelines.sinks.http_client.tls] -enabled = true -server_ca_file = "/path/to/ca.pem" # For server verification -server_name = "logs.example.com" -insecure_skip_verify = false -client_cert_file = "/path/to/client.pem" # For mTLS -client_key_file = "/path/to/client.key" # For mTLS -``` - -### TLS Certificate Generation - -Using the `tls` command: - -```bash -# Generate CA certificate -logwisp tls -ca -o myca - -# Generate server certificate -logwisp tls -server -ca-cert myca.pem -ca-key myca.key -host localhost,server.example.com -o server - -# Generate client certificate -logwisp tls -client -ca-cert myca.pem -ca-key myca.key -o client -``` - -Command options: - -| Flag | Description | -|------|-------------| -| `-ca` | Generate CA certificate | -| `-server` | Generate server certificate | -| `-client` | Generate client certificate | -| `-host` | Comma-separated hostnames/IPs | -| `-o` | Output file prefix | -| `-days` | Certificate validity (default: 365) | - -## Network Rate Limiting - -### Configuration Options - -```toml -[pipelines.sources.http.net_limit] -enabled = true -max_connections_per_ip = 10 -max_connections_total = 100 -requests_per_second = 100.0 -burst_size = 200 -response_code = 429 -response_message = "Rate limit exceeded" -ip_whitelist = ["192.168.1.0/24"] -ip_blacklist = ["10.0.0.0/8"] -``` - -### Rate Limiting Parameters - -| Parameter | Type | Description | -|-----------|------|-------------| -| `enabled` | bool | Enable rate limiting | -| `max_connections_per_ip` | int | Per-IP connection limit | -| `max_connections_total` | int | Global connection limit | -| `requests_per_second` | float | Request rate limit | -| `burst_size` | int | Token bucket burst capacity | -| `response_code` | int | HTTP response code when limited | -| `response_message` | string | Response message when limited | - -### IP Access Control - -**Whitelist**: Only specified IPs/networks allowed -```toml -ip_whitelist = [ - "192.168.1.0/24", # Local network - "10.0.0.0/8", # Private network - "203.0.113.5" # Specific IP -] -``` - -**Blacklist**: Specified IPs/networks denied -```toml -ip_blacklist = [ - "192.168.1.100", # Blocked host - "10.0.0.0/16" # Blocked subnet -] -``` - -Processing order: -1. Blacklist (immediate deny if matched) -2. Whitelist (must match if configured) -3. Rate limiting -4. Authentication - +*Note: As of the latest architecture updates, network TLS, mTLS, and Rate Limiting features are undergoing a redesign and are currently acting as placeholders. The documentation below details the structure for future updates.* + ## Connection Management ### TCP Keep-Alive ```toml -[pipelines.sources.tcp] +[[pipelines.plugin_sinks]] +id = "tcp_out" +type = "tcp" +[pipelines.plugin_sinks.config] keep_alive = true keep_alive_period_ms = 30000 # 30 seconds ``` -Benefits: -- Detect dead connections -- Prevent connection timeout -- Maintain NAT mappings - ### Connection Timeouts ```toml -[pipelines.sources.http] -read_timeout_ms = 10000 # 10 seconds +[[pipelines.plugin_sinks]] +id = "http_out" +type = "http" +[pipelines.plugin_sinks.config] write_timeout_ms = 10000 # 10 seconds - -[pipelines.sinks.tcp_client] -dial_timeout = 10 # Connection timeout -write_timeout = 30 # Write timeout -read_timeout = 10 # Read timeout -``` - -### Connection Limits - -Global limits: -```toml -max_connections = 100 # Total concurrent connections -``` - -Per-IP limits: -```toml -max_connections_per_ip = 10 ``` ## Heartbeat Configuration -Keep connections alive with periodic heartbeats: - -### HTTP Sink Heartbeat +Keep connections alive with periodic heartbeats. Note that Heartbeat is a flow-level feature in the new architecture. ```toml -[pipelines.sinks.http.heartbeat] +[pipelines.flow.heartbeat] enabled = true interval_ms = 30000 include_timestamp = true @@ -175,37 +42,18 @@ include_stats = false format = "comment" # comment|event|json ``` -Formats: -- **comment**: SSE comment (`: heartbeat`) -- **event**: SSE event with data -- **json**: JSON-formatted heartbeat - -### TCP Sink Heartbeat - -```toml -[pipelines.sinks.tcp.heartbeat] -enabled = true -interval_ms = 30000 -include_timestamp = true -include_stats = false -format = "json" # json|txt -``` - ## Network Protocols ### HTTP/HTTPS -- HTTP/1.1 and HTTP/2 support +- HTTP/1.1 support - Persistent connections -- Chunked transfer encoding - Server-Sent Events (SSE) ### TCP - Raw TCP sockets - Newline-delimited protocol -- Binary-safe transmission -- No encryption available ## Port Configuration @@ -213,9 +61,7 @@ format = "json" # json|txt | Service | Default Port | Protocol | |---------|--------------|----------| -| HTTP Source | 8081 | HTTP/HTTPS | -| HTTP Sink | 8080 | HTTP/HTTPS | -| TCP Source | 9091 | TCP | +| HTTP Sink | 8080 | HTTP | | TCP Sink | 9090 | TCP | ### Port Conflict Prevention @@ -223,46 +69,6 @@ format = "json" # json|txt LogWisp validates port usage at startup: - Detects port conflicts across pipelines - Prevents duplicate bindings -- Suggests alternative ports - -## Network Security - -### Best Practices - -1. **Use TLS for HTTP** connections when possible -2. **Implement rate limiting** to prevent DoS -3. **Configure IP whitelists** for restricted access -4. **Enable authentication** for all network endpoints -5. **Use non-standard ports** to reduce scanning exposure -6. **Monitor connection metrics** for anomalies -7. **Set appropriate timeouts** to prevent resource exhaustion - -### Security Warnings - -- TCP connections are **always unencrypted** -- HTTP Basic/Token auth **requires TLS** -- Avoid `skip_verify` in production -- Never expose unauthenticated endpoints publicly - -## Load Balancing - -### Client-Side Load Balancing - -Configure multiple endpoints (future feature): -```toml -[[pipelines.sinks.http_client]] -urls = [ - "https://log1.example.com/ingest", - "https://log2.example.com/ingest" -] -strategy = "round-robin" # round-robin|random|least-conn -``` - -### Server-Side Considerations - -- Use reverse proxy for load distribution -- Configure session affinity if needed -- Monitor individual instance health ## Troubleshooting @@ -286,4 +92,4 @@ strategy = "round-robin" # round-robin|random|least-conn **Connection Timeout** - Increase timeout values - Check network latency -- Verify keep-alive settings \ No newline at end of file +- Verify keep-alive settings diff --git a/doc/operations.md b/doc/operations.md index f148bbe..a08841d 100644 --- a/doc/operations.md +++ b/doc/operations.md @@ -2,6 +2,8 @@ Running, monitoring, and maintaining LogWisp in production. +*Note: TLS, acccess control under redesign* + ## Starting LogWisp ### Manual Start @@ -60,7 +62,7 @@ kill -USR1 $(pidof logwisp) Test configuration without starting: ```bash -logwisp --config test.toml --quiet --disable-status-reporter +logwisp --config test.toml --quiet --status-reporter=false ``` Check for errors: @@ -168,18 +170,10 @@ Production recommendation: `info` or `warn` Adjust buffers based on load: ```toml -# High-volume source -[[pipelines.sources]] -type = "http" -[pipelines.sources.http] -buffer_size = 5000 # Increase for burst traffic - -# Slow consumer sink -[[pipelines.sinks]] -type = "http_client" -[pipelines.sinks.http_client] -buffer_size = 10000 # Larger buffer for slow endpoints -batch_size = 500 # Larger batches +[[pipelines.plugin_sources]] +id = "file_in" +type = "file" +[pipelines.plugin_sources.config] ``` ### Rate Limiting @@ -187,22 +181,12 @@ batch_size = 500 # Larger batches Protect against overload: ```toml -[pipelines.rate_limit] +[pipelines.flow.rate_limit] rate = 1000.0 # Entries per second burst = 2000.0 # Burst capacity policy = "drop" # Drop excess entries ``` -### Connection Limits - -Prevent resource exhaustion: - -```toml -[pipelines.sources.http.net_limit] -max_connections_total = 1000 -max_connections_per_ip = 50 -``` - ## Troubleshooting ### Common Issues @@ -340,4 +324,4 @@ Data loss: - Run multiple instances for redundancy - Use load balancer for distribution - Implement monitoring alerts -- Document recovery procedures \ No newline at end of file +- Document recovery procedures diff --git a/doc/security.md b/doc/security.md index 0801fab..1a5097b 100644 --- a/doc/security.md +++ b/doc/security.md @@ -1,58 +1,4 @@ # Security -## mTLS (Mutual TLS) +*Note: Security features like mTLS and IP-Based Access Control are currently under redesign and act as placeholders in the new architecture. Future versions will reintroduce full security capabilities.* -Certificate-based authentication for HTTPS. - -### Server Configuration - -```toml -[pipelines.sources.http.tls] -enabled = true -cert_file = "/path/to/server.pem" -key_file = "/path/to/server.key" -client_auth = true -client_ca_file = "/path/to/ca.pem" -verify_client_cert = true -``` - -### Client Configuration - -```toml -[pipelines.sinks.http_client.tls] -enabled = true -cert_file = "/path/to/client.pem" -key_file = "/path/to/client.key" -``` - -### Certificate Generation - -Use the `tls` command: -```bash -# Generate CA -logwisp tls -ca -o ca - -# Generate server certificate -logwisp tls -server -ca-cert ca.pem -ca-key ca.key -host localhost -o server - -# Generate client certificate -logwisp tls -client -ca-cert ca.pem -ca-key ca.key -o client -``` - -## Access Control - -ogWisp provides IP-based access control for network connections. - -+## IP-Based Access Control - -Configure IP-based access control for sources: -```toml - [pipelines.sources.http.net_limit] - enabled = true - ip_whitelist = ["192.168.1.0/24", "10.0.0.0/8"] - ip_blacklist = ["192.168.1.100"] -``` - -Priority order: -1. Blacklist (checked first, immediate deny) -2. Whitelist (if configured, must match) \ No newline at end of file diff --git a/doc/sinks.md b/doc/sinks.md index afa503e..c792bd6 100644 --- a/doc/sinks.md +++ b/doc/sinks.md @@ -9,13 +9,12 @@ LogWisp sinks deliver processed log entries to various destinations. Output to stdout/stderr. ```toml -[[pipelines.sinks]] +[[pipelines.plugin_sinks]] +id = "console_out" type = "console" - -[pipelines.sinks.console] +[pipelines.plugin_sinks.config] target = "stdout" # stdout|stderr|split -colorize = false -buffer_size = 100 +buffer_size = 1000 ``` **Configuration Options:** @@ -23,8 +22,7 @@ buffer_size = 100 | Option | Type | Default | Description | |--------|------|---------|-------------| | `target` | string | "stdout" | Output target (stdout/stderr/split) | -| `colorize` | bool | false | Enable colored output | -| `buffer_size` | int | 100 | Internal buffer size | +| `buffer_size` | int | 1000 | Internal buffer size | **Target Modes:** - **stdout**: All output to standard output @@ -36,10 +34,10 @@ buffer_size = 100 Write logs to rotating files. ```toml -[[pipelines.sinks]] +[[pipelines.plugin_sinks]] +id = "file_out" type = "file" - -[pipelines.sinks.file] +[pipelines.plugin_sinks.config] directory = "./logs" name = "output" max_size_mb = 100 @@ -74,17 +72,15 @@ flush_interval_ms = 1000 SSE (Server-Sent Events) streaming server. ```toml -[[pipelines.sinks]] +[[pipelines.plugin_sinks]] +id = "http_out" type = "http" - -[pipelines.sinks.http] +[pipelines.plugin_sinks.config] host = "0.0.0.0" port = 8080 stream_path = "/stream" status_path = "/status" buffer_size = 1000 -max_connections = 100 -read_timeout_ms = 10000 write_timeout_ms = 10000 ``` @@ -97,34 +93,20 @@ write_timeout_ms = 10000 | `stream_path` | string | "/stream" | SSE stream endpoint | | `status_path` | string | "/status" | Status endpoint | | `buffer_size` | int | 1000 | Internal buffer size | -| `max_connections` | int | 100 | Maximum concurrent clients | -| `read_timeout_ms` | int | 10000 | Read timeout | | `write_timeout_ms` | int | 10000 | Write timeout | -**Heartbeat Configuration:** - -```toml -[pipelines.sinks.http.heartbeat] -enabled = true -interval_ms = 30000 -include_timestamp = true -include_stats = false -format = "comment" # comment|event|json -``` - ### TCP Sink TCP streaming server for debugging. ```toml -[[pipelines.sinks]] +[[pipelines.plugin_sinks]] +id = "tcp_out" type = "tcp" - -[pipelines.sinks.tcp] +[pipelines.plugin_sinks.config] host = "0.0.0.0" port = 9090 buffer_size = 1000 -max_connections = 100 keep_alive = true keep_alive_period_ms = 30000 ``` @@ -136,132 +118,21 @@ keep_alive_period_ms = 30000 | `host` | string | "0.0.0.0" | Bind address | | `port` | int | Required | Listen port | | `buffer_size` | int | 1000 | Internal buffer size | -| `max_connections` | int | 100 | Maximum concurrent clients | | `keep_alive` | bool | true | Enable TCP keep-alive | | `keep_alive_period_ms` | int | 30000 | Keep-alive interval | +| `write_timeout_ms` | int | 10000 | Write timeout | -**Note:** TCP Sink has no authentication support (debugging only). - -### HTTP Client Sink - -Forward logs to remote HTTP endpoints. +### Null Sink ```toml -[[pipelines.sinks]] -type = "http_client" - -[pipelines.sinks.http_client] -url = "https://logs.example.com/ingest" -buffer_size = 1000 -batch_size = 100 -batch_delay_ms = 1000 -timeout_seconds = 30 -max_retries = 3 -retry_delay_ms = 1000 -retry_backoff = 2.0 -insecure_skip_verify = false +[[pipelines.plugin_sinks]] +id = "null_out" +type = "null" ``` -**Configuration Options:** +## Buffer Management -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `url` | string | Required | Target URL | -| `buffer_size` | int | 1000 | Internal buffer size | -| `batch_size` | int | 100 | Logs per request | -| `batch_delay_ms` | int | 1000 | Max wait before sending | -| `timeout_seconds` | int | 30 | Request timeout | -| `max_retries` | int | 3 | Retry attempts | -| `retry_delay_ms` | int | 1000 | Initial retry delay | -| `retry_backoff` | float | 2.0 | Exponential backoff multiplier | -| `insecure_skip_verify` | bool | false | Skip TLS verification | - -### TCP Client Sink - -Forward logs to remote TCP servers. - -```toml -[[pipelines.sinks]] -type = "tcp_client" - -[pipelines.sinks.tcp_client] -host = "logs.example.com" -port = 9090 -buffer_size = 1000 -dial_timeout = 10 -write_timeout = 30 -read_timeout = 10 -keep_alive = 30 -reconnect_delay_ms = 1000 -max_reconnect_delay_ms = 30000 -reconnect_backoff = 1.5 -``` - -**Configuration Options:** - -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `host` | string | Required | Target host | -| `port` | int | Required | Target port | -| `buffer_size` | int | 1000 | Internal buffer size | -| `dial_timeout` | int | 10 | Connection timeout (seconds) | -| `write_timeout` | int | 30 | Write timeout (seconds) | -| `read_timeout` | int | 10 | Read timeout (seconds) | -| `keep_alive` | int | 30 | TCP keep-alive (seconds) | -| `reconnect_delay_ms` | int | 1000 | Initial reconnect delay | -| `max_reconnect_delay_ms` | int | 30000 | Maximum reconnect delay | -| `reconnect_backoff` | float | 1.5 | Backoff multiplier | - -## Network Sink Features - -### Network Rate Limiting - -Available for HTTP and TCP sinks: - -```toml -[pipelines.sinks.http.net_limit] -enabled = true -max_connections_per_ip = 10 -max_connections_total = 100 -ip_whitelist = ["192.168.1.0/24"] -ip_blacklist = ["10.0.0.0/8"] -``` - -### TLS Configuration (HTTP Only) - -```toml -[pipelines.sinks.http.tls] -enabled = true -cert_file = "/path/to/cert.pem" -key_file = "/path/to/key.pem" -ca_file = "/path/to/ca.pem" -min_version = "TLS1.2" -client_auth = false -``` - -HTTP Client TLS: - -```toml -[pipelines.sinks.http_client.tls] -enabled = true -server_ca_file = "/path/to/ca.pem" # For server verification -server_name = "logs.example.com" -insecure_skip_verify = false -client_cert_file = "/path/to/client.pem" # For mTLS -client_key_file = "/path/to/client.key" # For mTLS -``` - -## Sink Chaining - -Designed connection patterns: - -### Log Aggregation -- **HTTP Client Sink → HTTP Source**: HTTP/HTTPS (optional mTLS for HTTPS) -- **TCP Client Sink → TCP Source**: Raw TCP - -### Live Monitoring -- **HTTP Sink**: Browser-based SSE streaming -- **TCP Sink**: Debug interface (telnet/netcat) +- Full input buffer: entry dropped for that sink only (counted per pipeline as `total_dropped_by_sink`)" ## Sink Statistics @@ -270,4 +141,4 @@ All sinks track: - Active connections - Failed sends - Retry attempts -- Last processed timestamp \ No newline at end of file +- Last processed timestamp diff --git a/doc/sources.md b/doc/sources.md index 62de2be..422e177 100644 --- a/doc/sources.md +++ b/doc/sources.md @@ -6,31 +6,29 @@ LogWisp sources monitor various inputs and generate log entries for pipeline pro ### Directory Source -Monitors a directory for log files matching a pattern. +Monitors a directory for log files matching a pattern. (type: `file`) ```toml -[[pipelines.sources]] -type = "directory" - -[pipelines.sources.directory] -path = "/var/log/myapp" +[[pipelines.plugin_sources]] +id = "file_in" +type = "file" +[pipelines.plugin_sources.config] +directory = "/var/log/myapp" pattern = "*.log" # Glob pattern check_interval_ms = 100 # Poll interval -recursive = false # Scan subdirectories ``` **Configuration Options:** | Option | Type | Default | Description | |--------|------|---------|-------------| -| `path` | string | Required | Directory to monitor | +| `directory` | string | Required | Directory to monitor | | `pattern` | string | "*" | File pattern (glob) | | `check_interval_ms` | int | 100 | File check interval in milliseconds | -| `recursive` | bool | false | Include subdirectories | **Features:** -- Automatic file rotation detection -- Position tracking (resume after restart) +- Automatic rotation detection (inode + size tracking) +- In-memory position tracking; on restart, monitoring resumes from the current end of each file (offsets are not persisted) - Concurrent file monitoring - Pattern-based file selection @@ -39,10 +37,10 @@ recursive = false # Scan subdirectories Reads log entries from standard input. ```toml -[[pipelines.sources]] +[[pipelines.plugin_sources]] +id = "console_in" type = "console" - -[pipelines.sources.stdin] +[pipelines.plugin_sources.config] buffer_size = 1000 ``` @@ -57,107 +55,30 @@ buffer_size = 1000 - Automatic level detection - Non-blocking reads -### HTTP Source - -REST endpoint for log ingestion. +### Random Source ```toml -[[pipelines.sources]] -type = "http" - -[pipelines.sources.http] -host = "0.0.0.0" -port = 8081 -ingest_path = "/ingest" -buffer_size = 1000 -max_body_size = 1048576 # 1MB -read_timeout_ms = 10000 -write_timeout_ms = 10000 +[[pipelines.plugin_sources]] +id = "random_in" +type = "random" +[pipelines.plugin_sources.config] +interval_ms = 500 +jitter_ms = 0 +format = "txt" +length = 20 +special = false ``` -**Configuration Options:** - + **Configuration Options:** + | Option | Type | Default | Description | |--------|------|---------|-------------| -| `host` | string | "0.0.0.0" | Bind address | -| `port` | int | Required | Listen port | -| `ingest_path` | string | "/ingest" | Ingestion endpoint path | -| `buffer_size` | int | 1000 | Internal buffer size | -| `max_body_size` | int | 1048576 | Maximum request body size | -| `read_timeout_ms` | int | 10000 | Read timeout | -| `write_timeout_ms` | int | 10000 | Write timeout | - -**Input Formats:** -- Single JSON object -- JSON array -- Newline-delimited JSON (NDJSON) -- Plain text (one entry per line) - -### TCP Source - -Raw TCP socket listener for log ingestion. - -```toml -[[pipelines.sources]] -type = "tcp" - -[pipelines.sources.tcp] -host = "0.0.0.0" -port = 9091 -buffer_size = 1000 -read_timeout_ms = 10000 -keep_alive = true -keep_alive_period_ms = 30000 -``` - -**Configuration Options:** - -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `host` | string | "0.0.0.0" | Bind address | -| `port` | int | Required | Listen port | -| `buffer_size` | int | 1000 | Internal buffer size | -| `read_timeout_ms` | int | 10000 | Read timeout | -| `keep_alive` | bool | true | Enable TCP keep-alive | -| `keep_alive_period_ms` | int | 30000 | Keep-alive interval | - -**Protocol:** -- Newline-delimited JSON -- One log entry per line -- UTF-8 encoding - -## Network Source Features - -### Network Rate Limiting - -Available for HTTP and TCP sources: - -```toml -[pipelines.sources.http.net_limit] -enabled = true -max_connections_per_ip = 10 -max_connections_total = 100 -requests_per_second = 100.0 -burst_size = 200 -response_code = 429 -response_message = "Rate limit exceeded" -ip_whitelist = ["192.168.1.0/24"] -ip_blacklist = ["10.0.0.0/8"] -``` - -### TLS Configuration (HTTP Only) - -```toml -[pipelines.sources.http.tls] -enabled = true -cert_file = "/path/to/cert.pem" -key_file = "/path/to/key.pem" -min_version = "TLS1.2" -client_auth = true -client_ca_file = "/path/to/client-ca.pem" -verify_client_cert = true -``` - +| `interval_ms` | int | 500 | Generation interval | +| `jitter_ms` | int | 0 | Random jitter interval | +| `format` | string | "txt" | "txt", "json", "raw" | +| `length` | int | 20 | Log length | +| `special` | bool | false | Include special characters | + ## Source Statistics All sources track: @@ -168,10 +89,19 @@ All sources track: - Active connections (network sources) - Source-specific metrics +### Null Source + +```toml +[[pipelines.plugin_sources]] +id = "null_in" +type = "null" +[pipelines.plugin_sources.config] +``` + ## Buffer Management Each source maintains internal buffers: - Default size: 1000 entries - Drop policy when full - Configurable per source -- Non-blocking writes \ No newline at end of file +- Non-blocking writes diff --git a/go.mod b/go.mod index c15346d..cb05a2b 100644 --- a/go.mod +++ b/go.mod @@ -1,26 +1,26 @@ module logwisp -go 1.25.4 +go 1.26.0 require ( - github.com/lixenwraith/config v0.1.1-0.20251114180219-f7875023a51b - github.com/lixenwraith/log v0.1.1-0.20251115213227-55d2c92d483f - github.com/panjf2000/gnet/v2 v2.9.7 - github.com/valyala/fasthttp v1.68.0 + github.com/lixenwraith/config v0.1.1-0.20260712172228-ccd280ba6a98 + github.com/lixenwraith/log v0.1.1-0.20251117213308-9ae1b6669bf3 + github.com/panjf2000/gnet/v2 v2.10.0 + github.com/valyala/fasthttp v1.72.0 ) require ( github.com/BurntSushi/toml v1.6.0 // indirect - github.com/andybalholm/brotli v1.2.0 // indirect + github.com/andybalholm/brotli v1.2.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-viper/mapstructure/v2 v2.4.0 // indirect - github.com/klauspost/compress v1.18.2 // indirect - github.com/panjf2000/ants/v2 v2.11.4 // indirect + github.com/go-viper/mapstructure/v2 v2.5.0 // indirect + github.com/klauspost/compress v1.19.0 // indirect + github.com/panjf2000/ants/v2 v2.12.1 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.27.1 // indirect - golang.org/x/sync v0.19.0 // indirect - golang.org/x/sys v0.39.0 // indirect + go.uber.org/zap v1.28.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 4d51bdf..cf405da 100644 --- a/go.sum +++ b/go.sum @@ -2,24 +2,50 @@ github.com/BurntSushi/toml v1.6.0 h1:dRaEfpa2VI55EwlIW72hMRHdWouJeRF7TPYhI+AUQjk github.com/BurntSushi/toml v1.6.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/brotli v1.2.1 h1:R+f5xP285VArJDRgowrfb9DqL18yVK0gKAW/F+eTWro= +github.com/andybalholm/brotli v1.2.1/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/andybalholm/brotli v1.2.2 h1:HzTuoo2ErYQqf5qvcJInB8uvqSVxRttzkFexPWtnceM= +github.com/andybalholm/brotli v1.2.2/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/go-viper/mapstructure/v2 v2.4.0 h1:EBsztssimR/CONLSZZ04E8qAkxNYq4Qp9LvH92wZUgs= github.com/go-viper/mapstructure/v2 v2.4.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= +github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPEgAXnvj1Ro= +github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co= github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0= github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk= github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.18.3 h1:9PJRvfbmTabkOX8moIpXPbMMbYN60bWImDDU7L+/6zw= +github.com/klauspost/compress v1.18.3/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c= +github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/compress v1.19.0 h1:sXLILfc9jV2QYWkzFOPWStmcUVH2RHEB1JCdY2oVvCQ= +github.com/klauspost/compress v1.19.0/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/lixenwraith/config v0.1.1-0.20251114180219-f7875023a51b h1:TzTV0ArJ+nzVGPN8aiEJ2MknUqJdmHRP/0/RSfov2Qw= github.com/lixenwraith/config v0.1.1-0.20251114180219-f7875023a51b/go.mod h1:roNPTSCT5HSV9dru/zi/Catwc3FZVCFf7vob2pSlNW0= +github.com/lixenwraith/config v0.1.1-0.20260712172228-ccd280ba6a98 h1:VEFo2WsYgM5YYfzAfvXRKigCWr3zEz6h4M59N5TXMpk= +github.com/lixenwraith/config v0.1.1-0.20260712172228-ccd280ba6a98/go.mod h1:J9ydxY7he4Dz+S59xKs/kDQ2YAv+zOLF05b3Rr2/ogE= github.com/lixenwraith/log v0.1.1-0.20251115213227-55d2c92d483f h1:X2LX5FQEuWYGBS3qp5z7XxBB1sWAlqumf/oW7n/f9c0= github.com/lixenwraith/log v0.1.1-0.20251115213227-55d2c92d483f/go.mod h1:XcRPRuijAs+43Djk8VmioUJhcK8irRzUjCZaZqkd3gg= +github.com/lixenwraith/log v0.1.1-0.20251117213308-9ae1b6669bf3 h1:brSUhER7EZ28aMRFTSovZskiIoobUiAWlx+DYXYvWXQ= +github.com/lixenwraith/log v0.1.1-0.20251117213308-9ae1b6669bf3/go.mod h1:MY59N65ltw/9uTqJKwCRKjxO6w3CApXsLXCVuaekbu0= github.com/panjf2000/ants/v2 v2.11.3 h1:AfI0ngBoXJmYOpDh9m516vjqoUu2sLrIVgppI9TZVpg= github.com/panjf2000/ants/v2 v2.11.3/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= github.com/panjf2000/ants/v2 v2.11.4 h1:UJQbtN1jIcI5CYNocTj0fuAUYvsLjPoYi0YuhqV/Y48= github.com/panjf2000/ants/v2 v2.11.4/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= +github.com/panjf2000/ants/v2 v2.11.5 h1:a7LMnMEeux/ebqTux140tRiaqcFTV0q2bEHF03nl6Rg= +github.com/panjf2000/ants/v2 v2.11.5/go.mod h1:8u92CYMUc6gyvTIw8Ru7Mt7+/ESnJahz5EVtqfrilek= +github.com/panjf2000/ants/v2 v2.12.1 h1:BWvU2wHpyXWxhhNXsGB6JXLCNbshyLd1QxvoAmZnu10= +github.com/panjf2000/ants/v2 v2.12.1/go.mod h1:tSQuaNQ6r6NRhPt+IZVUevvDyFMTs+eS4ztZc52uJTY= github.com/panjf2000/gnet/v2 v2.9.7 h1:6zW7Jl3oAfXwSuh1PxHLndoL2MQRWx0AJR6aaQjxUgA= github.com/panjf2000/gnet/v2 v2.9.7/go.mod h1:WQTxDWYuQ/hz3eccH0FN32IVuvZ19HewEWx0l62fx7E= +github.com/panjf2000/gnet/v2 v2.9.8 h1:OzVnIKm4UMoGphWgypcVZMq8roVjXzXOZvLdyZHxvkE= +github.com/panjf2000/gnet/v2 v2.9.8/go.mod h1:f9wdbOFsdbZqlSvXctWbPRW5bB/W++q8Zqz+D7tQIVQ= +github.com/panjf2000/gnet/v2 v2.10.0 h1:rC4jNF+jtXj/FH+8JOIQ3XxjD+yBunYBLKg9TE3dc4g= +github.com/panjf2000/gnet/v2 v2.10.0/go.mod h1:f9wdbOFsdbZqlSvXctWbPRW5bB/W++q8Zqz+D7tQIVQ= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= @@ -28,6 +54,12 @@ github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6Kllzaw github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.68.0 h1:v12Nx16iepr8r9ySOwqI+5RBJ/DqTxhOy1HrHoDFnok= github.com/valyala/fasthttp v1.68.0/go.mod h1:5EXiRfYQAoiO/khu4oU9VISC/eVY6JqmSpPJoHCKsz4= +github.com/valyala/fasthttp v1.69.0 h1:fNLLESD2SooWeh2cidsuFtOcrEi4uB4m1mPrkJMZyVI= +github.com/valyala/fasthttp v1.69.0/go.mod h1:4wA4PfAraPlAsJ5jMSqCE2ug5tqUPwKXxVj8oNECGcw= +github.com/valyala/fasthttp v1.71.0 h1:tepR7H+Guh9VUqxxcPggYi8R3lGUu2Rsdh+z7/FCY3k= +github.com/valyala/fasthttp v1.71.0/go.mod h1:z1sDUvOShhXq/C9mwH/fSm1Vb71tUJwmQdgkBrBNwnA= +github.com/valyala/fasthttp v1.72.0 h1:R7kYdoWhn1ye1fVpP+cDHDJwYm3NkwLliwgzJ/Abg7M= +github.com/valyala/fasthttp v1.72.0/go.mod h1:zsbLTYqcpIktdQytlVBwIjY9La5d6bs990nBxWg8efk= github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= @@ -38,14 +70,30 @@ go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w= golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.21.0 h1:HLII4xRRTtCRkxYp4HNFF0Js/Og6q2i++KXbg0gHCwM= +golang.org/x/sync v0.21.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.45.0 h1:dO4czNzziLiiXplLQgBCEpCvXQ3dnkn0SdaZSYdQ+FY= +golang.org/x/sys v0.45.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.46.0 h1:noSf2Fq6F8DBgS+LysIkx7rIExoNHJsxOAtPp4rthXw= +golang.org/x/sys v0.46.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= diff --git a/src/cmd/logwisp/bootstrap.go b/src/cmd/logwisp/bootstrap.go index dcfaefa..c5acb30 100644 --- a/src/cmd/logwisp/bootstrap.go +++ b/src/cmd/logwisp/bootstrap.go @@ -6,20 +6,25 @@ import ( _ "logwisp/src/internal/source/console" _ "logwisp/src/internal/source/file" + _ "logwisp/src/internal/source/httpchain" _ "logwisp/src/internal/source/null" _ "logwisp/src/internal/source/random" + _ "logwisp/src/internal/source/tcpchain" _ "logwisp/src/internal/sink/console" _ "logwisp/src/internal/sink/file" _ "logwisp/src/internal/sink/http" + _ "logwisp/src/internal/sink/httpchain" _ "logwisp/src/internal/sink/null" _ "logwisp/src/internal/sink/tcp" + _ "logwisp/src/internal/sink/tcpchain" "logwisp/src/internal/config" "logwisp/src/internal/service" "logwisp/src/internal/version" "github.com/lixenwraith/log" + "github.com/lixenwraith/log/sanitizer" ) // bootstrapInitial handles initial service startup with status reporter @@ -132,6 +137,14 @@ func initializeLogger(cfg *config.Config) error { } logCfg.Level = levelValue + // Configure log format + if cfg.Logging.Format != "" { + logCfg.Format = cfg.Logging.Format + } + if cfg.Logging.Sanitization != "" { + logCfg.Sanitization = sanitizer.PolicyPreset(cfg.Logging.Sanitization) + } + // Configure based on output mode switch cfg.Logging.Output { case "none": @@ -176,4 +189,4 @@ func configureFileLogging(logCfg *log.Config, cfg *config.Config) { logCfg.RetentionPeriodHrs = cfg.Logging.File.RetentionHours } } -} \ No newline at end of file +} diff --git a/src/cmd/logwisp/help.go b/src/cmd/logwisp/help.go new file mode 100644 index 0000000..aacfc1f --- /dev/null +++ b/src/cmd/logwisp/help.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "logwisp/src/internal/version" + "os" +) + +// helpText is the CLI usage reference. Flags map 1:1 to TOML config paths. +const helpText = `LogWisp %s - log collection, processing, and distribution + +Usage: + logwisp [options] + logwisp help | -h | --help + logwisp --version + +Any configuration key is settable as a flag using its TOML path: + --= e.g. --logging.level=debug + +Common options: + -c, --config Configuration file (default: ./logwisp.toml) + --quiet Suppress console output + --status_reporter= Periodic status logging (default: true) + --auto_reload= Config hot reload on file change (default: false) + +Logging: + --logging.output= file|stdout|stderr|split|all|none + --logging.level= debug|info|warn|error + --logging.file.directory= + --logging.console.target= stdout|stderr|split + +Pipelines (N = 0-based index): + --pipelines.N.name= + --pipelines.N.plugin_sources.N.type= file|console|random|null + --pipelines.N.plugin_sinks.N.type= console|file|http|tcp|null + --pipelines.N.flow.filters.N.patterns='["ERROR","WARN"]' + +Environment: + LOGWISP_ Config path, '.' -> '_', uppercase + e.g. LOGWISP_LOGGING_LEVEL=debug + LOGWISP_CONFIG_FILE Configuration file path + LOGWISP_CONFIG_DIR Configuration directory + +Signals: + SIGINT, SIGTERM Graceful shutdown + SIGHUP, SIGUSR1 Reload configuration + +Exit codes: + 0 success + 1 general error + 2 configuration file not found +` + +// handleHelp prints usage and exits if a help request is present in args +func handleHelp(args []string) { + if len(args) > 0 && args[0] == "help" { + printHelp() + } + for _, arg := range args { + if arg == "--" { + break // end of flags + } + if arg == "-h" || arg == "--help" { + printHelp() + } + } +} + +// printHelp writes usage to stdout and exits with success +func printHelp() { + fmt.Printf(helpText, version.Short()) + os.Exit(0) +} diff --git a/src/cmd/logwisp/main.go b/src/cmd/logwisp/main.go index 5939cbe..776c463 100644 --- a/src/cmd/logwisp/main.go +++ b/src/cmd/logwisp/main.go @@ -7,7 +7,6 @@ import ( "os/signal" "strings" "syscall" - "time" "logwisp/src/internal/config" "logwisp/src/internal/core" @@ -25,6 +24,10 @@ func main() { // Emulates nohup signal.Ignore(syscall.SIGHUP) + // Help handled before config parsing; loader has no help flag. + // Also the future dispatch point for subcommands (tls, etc.) + handleHelp(os.Args[1:]) + // Load configuration with automatic CLI parsing cfg, err := config.Load(os.Args[1:]) if err != nil { @@ -64,8 +67,6 @@ func main() { "status_reporter", cfg.StatusReporter, "auto_reload", cfg.ConfigAutoReload) - time.Sleep(time.Second) - // Create context for shutdown ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -152,4 +153,4 @@ func shutdownLogger() { Error("Logger shutdown error: %v\n", err) } } -} \ No newline at end of file +} diff --git a/src/internal/chain/proto.go b/src/internal/chain/proto.go new file mode 100644 index 0000000..eaeecbb --- /dev/null +++ b/src/internal/chain/proto.go @@ -0,0 +1,98 @@ +package chain + +import ( + "encoding/json" + "fmt" + "logwisp/src/internal/core" + "math/rand/v2" + "time" +) + +// ProtocolVersion is declared in the hello preamble +const ProtocolVersion = 1 + +// Hello is the first NDJSON line sent by the dialing side after connect. +// Reserved for future revisions: auth credential, feature flags (ack, compression). +type Hello struct { + LogWisp int `json:"logwisp"` + Node string `json:"node,omitempty"` + // Auth string `json:"auth,omitempty"` + // Features []string `json:"features,omitempty"` +} + +// HTTP transport mapping of the chain protocol. +// Hello preamble equivalent: protocol + node carried as request headers. +// Reserved extension point: Authorization header for auth, TLS at transport. +const ( + HeaderProtocol = "X-Logwisp-Protocol" + HeaderNode = "X-Logwisp-Node" + HeaderAccepted = "X-Logwisp-Accepted" + ContentTypeNDJSON = "application/x-ndjson" +) + +// EncodeHello serializes a newline-terminated hello preamble +func EncodeHello(node string) ([]byte, error) { + b, err := json.Marshal(Hello{LogWisp: ProtocolVersion, Node: node}) + if err != nil { + return nil, err + } + return append(b, '\n'), nil +} + +// DecodeHello parses and validates a hello preamble line +func DecodeHello(line []byte) (Hello, error) { + var h Hello + if err := json.Unmarshal(line, &h); err != nil { + return h, fmt.Errorf("malformed hello: %w", err) + } + if h.LogWisp != ProtocolVersion { + return h, fmt.Errorf("unsupported protocol version: %d", h.LogWisp) + } + return h, nil +} + +// DecodeEntry parses a canonical LogEntry line and applies the node trust policy +func DecodeEntry(line []byte, connNode string, trustNode bool) (core.LogEntry, error) { + var entry core.LogEntry + if err := json.Unmarshal(line, &entry); err != nil { + return core.LogEntry{}, err + } + if entry.Time.IsZero() { + entry.Time = time.Now() + } + if entry.Node == "" || !trustNode { + entry.Node = connNode + } + entry.RawSize = int64(len(line)) + return entry, nil +} + +// EntryFromEvent extracts the structured entry, stamping node identity at +// first hop. Second return is true when synthesized from a formatted payload. +func EntryFromEvent(event core.TransportEvent, node, fallbackSource string) (core.LogEntry, bool) { + entry := event.Entry + synthesized := false + if entry.Time.IsZero() { + synthesized = true + entry = core.LogEntry{ + Time: event.Time, + Source: fallbackSource, + Message: string(event.Payload), + } + } + if entry.Node == "" { + entry.Node = node + } + return entry, synthesized +} + +// BackoffDelay computes exponential backoff with ±20% jitter +func BackoffDelay(minD, maxD time.Duration, failures int) time.Duration { + d := maxD + if failures < 63 { + if v := minD << uint(failures-1); v > 0 && v < maxD { + d = v + } + } + return d - d/5 + time.Duration(rand.Int64N(int64(2*d/5)+1)) +} diff --git a/src/internal/config/config.go b/src/internal/config/config.go index b0842da..6e494c2 100644 --- a/src/internal/config/config.go +++ b/src/internal/config/config.go @@ -205,6 +205,32 @@ type ConsoleSourceOptions struct { BufferSize int64 `toml:"buffer_size"` } +// TCPChainSourceOptions defines settings for a stdlib TCP listener ingesting +// NDJSON entries from upstream logwisp tcp_chain sinks +type TCPChainSourceOptions struct { + Host string `toml:"host"` + Port int64 `toml:"port"` + BufferSize int64 `toml:"buffer_size"` + MaxConnections int64 `toml:"max_connections"` // 0 = unlimited + ReadTimeoutMS int64 `toml:"read_timeout_ms"` // per-connection idle deadline, 0 = none + HelloTimeoutMS int64 `toml:"hello_timeout_ms"` // preamble deadline + TrustNode bool `toml:"trust_node"` // false: force node label from remote address + // Future: TLS/auth options +} + +// HTTPChainSourceOptions defines settings for a stdlib HTTP listener ingesting +// NDJSON batches from upstream logwisp http_chain sinks +type HTTPChainSourceOptions struct { + Host string `toml:"host"` + Port int64 `toml:"port"` + IngestPath string `toml:"ingest_path"` + BufferSize int64 `toml:"buffer_size"` + MaxBodyBytes int64 `toml:"max_body_bytes"` // per-request cap + ReadTimeoutMS int64 `toml:"read_timeout_ms"` // full request read deadline + TrustNode bool `toml:"trust_node"` // false: force node label from remote address + // Future: TLS/auth options +} + // --- Sink Options --- // PluginSinkConfig represents a sink plugin instance configuration @@ -251,16 +277,49 @@ type TCPSinkOptions struct { Port int64 `toml:"port"` BufferSize int64 `toml:"buffer_size"` WriteTimeout int64 `toml:"write_timeout_ms"` - KeepAlive bool `toml:"keep_alive"` KeepAlivePeriod int64 `toml:"keep_alive_period_ms"` + KeepAlive bool `toml:"keep_alive"` } // HTTPSinkOptions defines settings for an HTTP SSE server sink type HTTPSinkOptions struct { - Host string `toml:"host"` - Port int64 `toml:"port"` StreamPath string `toml:"stream_path"` StatusPath string `toml:"status_path"` + Host string `toml:"host"` + Port int64 `toml:"port"` BufferSize int64 `toml:"buffer_size"` WriteTimeout int64 `toml:"write_timeout_ms"` -} \ No newline at end of file +} + +// TCPChainSinkOptions defines settings for a stdlib TCP client forwarding +// entries to a downstream logwisp tcp_chain source +type TCPChainSinkOptions struct { + Node string `toml:"node"` // origin label, default: os.Hostname() + Host string `toml:"host"` + Port int64 `toml:"port"` + BufferSize int64 `toml:"buffer_size"` + DialTimeoutMS int64 `toml:"dial_timeout_ms"` + WriteTimeoutMS int64 `toml:"write_timeout_ms"` + BackoffMinMS int64 `toml:"backoff_min_ms"` + BackoffMaxMS int64 `toml:"backoff_max_ms"` + KeepAlivePeriodMS int64 `toml:"keep_alive_period_ms"` + KeepAlive bool `toml:"keep_alive"` + // Future: TLS/auth options +} + +// HTTPChainSinkOptions defines settings for a stdlib HTTP client posting +// NDJSON batches to a downstream logwisp http_chain source +type HTTPChainSinkOptions struct { + Node string `toml:"node"` // origin label, default: os.Hostname() + Host string `toml:"host"` + Port int64 `toml:"port"` + IngestPath string `toml:"ingest_path"` + BufferSize int64 `toml:"buffer_size"` + MaxBatchCount int64 `toml:"max_batch_count"` + MaxBatchBytes int64 `toml:"max_batch_bytes"` + FlushIntervalMS int64 `toml:"flush_interval_ms"` + RequestTimeoutMS int64 `toml:"request_timeout_ms"` // covers dial + write + response + BackoffMinMS int64 `toml:"backoff_min_ms"` + BackoffMaxMS int64 `toml:"backoff_max_ms"` + // Future: TLS/auth options +} diff --git a/src/internal/config/loader.go b/src/internal/config/loader.go index 2a1fd24..7bad3e5 100644 --- a/src/internal/config/loader.go +++ b/src/internal/config/loader.go @@ -65,6 +65,12 @@ func Load(args []string) (*Config, error) { // Store the manager for hot reload configManager = cfg + // Surface typo'd flags (e.g. --status-reporter vs --status_reporter); + // pre-logger phase, stderr only, suppressed in quiet mode + if unknown := cfg.UnknownCLIKeys(); len(unknown) > 0 && !finalConfig.Quiet { + fmt.Fprintf(os.Stderr, "Warning: unrecognized flags ignored: %v\n", unknown) + } + // Start watcher if auto-reload is enabled if finalConfig.ConfigAutoReload { watchOpts := lconfig.WatchOptions{ @@ -99,6 +105,7 @@ func defaults() *Config { Logging: &LogConfig{ Output: "stdout", Level: "info", + Format: "txt", File: &LogFileConfig{ Directory: "./log", Name: "logwisp", @@ -195,4 +202,4 @@ func customEnvTransform(path string) string { env = strings.ToUpper(env) // env = "LOGWISP_" + env // already added by WithEnvPrefix return env -} \ No newline at end of file +} diff --git a/src/internal/config/validate.go b/src/internal/config/validate.go index c45a9b0..e976a99 100644 --- a/src/internal/config/validate.go +++ b/src/internal/config/validate.go @@ -17,6 +17,15 @@ func ValidateConfig(cfg *Config) error { return fmt.Errorf("no pipelines configured") } + // Reject duplicate pipeline names (service map is keyed by name) + names := make(map[string]struct{}, len(cfg.Pipelines)) + for i, p := range cfg.Pipelines { + if _, dup := names[p.Name]; dup { + return fmt.Errorf("pipeline[%d]: duplicate name %q", i, p.Name) + } + names[p.Name] = struct{}{} + } + if err := validateLogConfig(cfg.Logging); err != nil { return fmt.Errorf("logging: %w", err) } @@ -52,6 +61,17 @@ func validateLogConfig(cfg *LogConfig) error { return fmt.Errorf("level: %w", err) } + if cfg.Format != "" { + if err := lconfig.OneOf("raw", "txt", "json")(cfg.Format); err != nil { + return fmt.Errorf("format: %w", err) + } + } + if cfg.Sanitization != "" { + if err := lconfig.OneOf("raw", "json", "txt", "shell")(cfg.Sanitization); err != nil { + return fmt.Errorf("sanitization: %w", err) + } + } + if cfg.Console != nil { validateTarget := lconfig.OneOf("stdout", "stderr", "split") if err := validateTarget(cfg.Console.Target); err != nil { @@ -60,4 +80,4 @@ func validateLogConfig(cfg *LogConfig) error { } return nil -} \ No newline at end of file +} diff --git a/src/internal/core/flow.go b/src/internal/core/flow.go index 0ada160..0103762 100644 --- a/src/internal/core/flow.go +++ b/src/internal/core/flow.go @@ -5,9 +5,10 @@ import ( "time" ) -// Represents a single log record flowing through the pipeline +// LogEntry represents a single log record flowing through the pipeline type LogEntry struct { Time time.Time `json:"time"` + Node string `json:"node,omitempty"` // origin node identity for chained topologies; first hop stamps, relays preserve Source string `json:"source"` Level string `json:"level,omitempty"` Message string `json:"message"` @@ -20,4 +21,7 @@ type TransportEvent struct { Time time.Time // Formatted, serialized log payload Payload []byte -} \ No newline at end of file + // Structured entry for re-serializing sinks (chain links). Zero Time => absent + Entry LogEntry +} + diff --git a/src/internal/flow/flow.go b/src/internal/flow/flow.go index b97fa59..348b8d8 100644 --- a/src/internal/flow/flow.go +++ b/src/internal/flow/flow.go @@ -119,6 +119,7 @@ func (f *Flow) Process(entry core.LogEntry) (core.TransportEvent, bool) { event := core.TransportEvent{ Time: entry.Time, Payload: formatted, + Entry: entry, // Carry structured entry so chain sinks are format-independent } return event, true @@ -159,4 +160,5 @@ func (f *Flow) GetStats() map[string]any { } return stats -} \ No newline at end of file +} + diff --git a/src/internal/flow/heartbeat.go b/src/internal/flow/heartbeat.go index 7d34fbf..c7821f7 100644 --- a/src/internal/flow/heartbeat.go +++ b/src/internal/flow/heartbeat.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "strconv" "sync/atomic" "time" @@ -127,7 +128,8 @@ func (hg *HeartbeatGenerator) generateHeartbeat(t time.Time) core.TransportEvent // SSE comment format - bypass formatter for this special case if hg.config.IncludeStats { beatNum := hg.beatCount.Load() - payload = []byte(": heartbeat " + t.Format(time.RFC3339) + " [#" + string(beatNum) + "]\n") + payload = []byte(": heartbeat " + t.Format(time.RFC3339) + + " [#" + strconv.FormatUint(beatNum, 10) + "]\n") } else { payload = []byte(": heartbeat " + t.Format(time.RFC3339) + "\n") } @@ -159,10 +161,11 @@ func (hg *HeartbeatGenerator) generateHeartbeat(t time.Time) core.TransportEvent return core.TransportEvent{ Time: t, Payload: payload, + Entry: entry, // heartbeats traverse chain links as structured entries } } // IntervalMS returns the heartbeat interval in milliseconds func (hg *HeartbeatGenerator) IntervalMS() int64 { return hg.config.IntervalMS -} \ No newline at end of file +} diff --git a/src/internal/format/adapter.go b/src/internal/format/adapter.go index 373c90e..f817f94 100644 --- a/src/internal/format/adapter.go +++ b/src/internal/format/adapter.go @@ -89,6 +89,8 @@ func NewFormatterAdapter(cfg *config.FormatConfig) (*FormatterAdapter, error) { func (a *FormatterAdapter) Format(entry core.LogEntry) ([]byte, error) { // Map logwisp LogEntry to formatter args level := mapLevel(entry.Level) + // syslog-style origin prefix for chained entries + src := sourceLabel(entry) // Build args based on whether we have structured fields var args []any @@ -101,18 +103,19 @@ func (a *FormatterAdapter) Format(entry core.LogEntry) ([]byte, error) { args = []any{entry.Message, fields} // Add structured flag to properly format fields as JSON object effectiveFlags := a.flags | formatter.FlagStructuredJSON - return a.formatter.Format(effectiveFlags, entry.Time, level, entry.Source, args), nil + return a.formatter.Format(effectiveFlags, entry.Time, level, src, args), nil } } // Simple message without fields args = []any{entry.Message} - return a.formatter.Format(a.flags, entry.Time, level, entry.Source, args), nil + return a.formatter.Format(a.flags, entry.Time, level, src, args), nil } // FormatWithFlags allows custom flags for specific formatting needs func (a *FormatterAdapter) FormatWithFlags(entry core.LogEntry, customFlags int64) ([]byte, error) { level := mapLevel(entry.Level) + src := sourceLabel(entry) var args []any if len(entry.Fields) > 0 { @@ -127,7 +130,7 @@ func (a *FormatterAdapter) FormatWithFlags(entry core.LogEntry, customFlags int6 args = []any{entry.Message} } - return a.formatter.Format(customFlags, entry.Time, level, entry.Source, args), nil + return a.formatter.Format(customFlags, entry.Time, level, src, args), nil } // Name returns formatter type @@ -149,4 +152,13 @@ func mapLevel(level string) int64 { default: return 0 } -} \ No newline at end of file +} + +// sourceLabel prefixes origin node onto source (syslog HOSTNAME + TAG convention) +func sourceLabel(entry core.LogEntry) string { + if entry.Node == "" { + return entry.Source + } + return entry.Node + "/" + entry.Source +} + diff --git a/src/internal/pipeline/pipeline.go b/src/internal/pipeline/pipeline.go index 667ac37..06265fc 100644 --- a/src/internal/pipeline/pipeline.go +++ b/src/internal/pipeline/pipeline.go @@ -41,13 +41,11 @@ type Pipeline struct { // PipelineStats contains runtime statistics for a pipeline type PipelineStats struct { - StartTime time.Time - TotalEntriesProcessed atomic.Uint64 - TotalEntriesDroppedByRateLimit atomic.Uint64 - TotalEntriesFiltered atomic.Uint64 - SourceStats []source.SourceStats - SinkStats []sink.SinkStats - FlowStats map[string]any + StartTime time.Time + TotalEntriesDroppedBySink atomic.Uint64 + SourceStats []source.SourceStats + SinkStats []sink.SinkStats + FlowStats map[string]any } // NewPipeline creates a new pipeline with registry support @@ -74,7 +72,6 @@ func NewPipeline( cancel: pipelineCancel, } - // Create flow processor // Create flow processor flowProcessor, err := flow.NewFlow(cfg.Flow, logger) if err != nil { @@ -177,7 +174,7 @@ func (p *Pipeline) initSourceCapabilities(s source.Source, cfg config.PluginSour // initSinkCapabilities checks and injects optional capabilities func (p *Pipeline) initSinkCapabilities(s sink.Sink, cfg config.PluginSinkConfig) error { - // Initiate and activate source capabilities + // Initiate and activate sink capabilities for _, c := range s.Capabilities() { switch c { // Network capabilities @@ -203,48 +200,36 @@ func (p *Pipeline) run() { defer p.logger.Info("msg", "Pipeline processing loop stopped", "pipeline", p.Config.Name) var componentWg sync.WaitGroup - // Start a goroutine for each source to fan-in data for _, src := range p.Sources { componentWg.Add(1) go func(s source.Source) { defer componentWg.Done() ch := s.Subscribe() - for { - select { - case entry, ok := <-ch: - if !ok { - return - } - // Process and distribute the log entry - if event, passed := p.Flow.Process(entry); passed { - // Fan-out to all sinks - for _, snk := range p.Sinks { - snk.Input() <- event - } - } - case <-p.ctx.Done(): - return + // Range allows in-flight data to drain cleanly once Source.Stop() closes the channel + for entry := range ch { + if event, passed := p.Flow.Process(entry); passed { + // Use non-blocking dispatcher + p.dispatch(event) } } }(src) } + var hbWg sync.WaitGroup // Start heartbeat generator if enabled if heartbeatCh := p.Flow.StartHeartbeat(p.ctx); heartbeatCh != nil { - componentWg.Add(1) + hbWg.Add(1) go func() { - defer componentWg.Done() + defer hbWg.Done() for { select { case event, ok := <-heartbeatCh: if !ok { return } - // Fan-out heartbeat to all sinks - for _, snk := range p.Sinks { - snk.Input() <- event - } + // Use non-blocking dispatcher + p.dispatch(event) case <-p.ctx.Done(): return } @@ -253,6 +238,23 @@ func (p *Pipeline) run() { } componentWg.Wait() + + // Terminate internal contexts (heartbeat) once flow is complete + p.cancel() + hbWg.Wait() +} + +// dispatch performs a non-blocking send to all sinks. +// A full/stalled sink must never block the run loop or starve sibling sinks. +func (p *Pipeline) dispatch(event core.TransportEvent) { + for _, snk := range p.Sinks { + select { + case snk.Input() <- event: + default: + // Buffer full - drop to avoid deadlocking the pipeline + p.Stats.TotalEntriesDroppedBySink.Add(1) + } + } } // Start starts the pipeline operation and all its components including flow, sources, and sinks @@ -294,10 +296,7 @@ func (p *Pipeline) Stop() error { p.logger.Info("msg", "Stopping pipeline", "pipeline", p.Config.Name) - // Signal all components and the run loop to stop - p.cancel() - - // Stop all sources concurrently to halt new data ingress + // 1. Stop all sources concurrently to halt new data ingress and close their channels var sourceWg sync.WaitGroup for _, src := range p.Sources { sourceWg.Add(1) @@ -308,10 +307,11 @@ func (p *Pipeline) Stop() error { } sourceWg.Wait() - // Wait for the run loop to finish processing and sending all in-flight data + // 2. Wait for the run loop to finish processing and sending all in-flight data + // run() inherently calls p.cancel() when the source channels are empty p.wg.Wait() - // Stop all sinks concurrently now that no new data will be sent + // 3. Stop all sinks concurrently now that no new data will be sent var sinkWg sync.WaitGroup for _, s := range p.Sinks { sinkWg.Add(1) @@ -361,92 +361,82 @@ func (p *Pipeline) GetStats() map[string]any { } }() - // Collect source stats - sourceStats := make([]map[string]any, 0, len(p.Sources)) + // 1. Live collect source stats + sources := make([]map[string]any, 0, len(p.Sources)) for _, src := range p.Sources { if src == nil { - continue // Skip nil sources + continue } - - stats := src.GetStats() - sourceStats = append(sourceStats, map[string]any{ - "id": stats.ID, - "type": stats.Type, - "total_entries": stats.TotalEntries, - "dropped_entries": stats.DroppedEntries, - "start_time": stats.StartTime, - "last_entry_time": stats.LastEntryTime, - "details": stats.Details, + s := src.GetStats() + sources = append(sources, map[string]any{ + "id": s.ID, + "type": s.Type, + "total_entries": s.TotalEntries, + "dropped_entries": s.DroppedEntries, + "start_time": s.StartTime, + "last_entry_time": s.LastEntryTime, + "details": s.Details, }) } - // Collect sink stats - sinkStats := make([]map[string]any, 0, len(p.Sinks)) - for _, s := range p.Sinks { - if s == nil { - continue // Skip nil sinks + // 2. Live collect sink stats + sinks := make([]map[string]any, 0, len(p.Sinks)) + for _, snk := range p.Sinks { + if snk == nil { + continue } - - stats := s.GetStats() - sinkStats = append(sinkStats, map[string]any{ - "id": stats.ID, - "type": stats.Type, - "total_processed": stats.TotalProcessed, - "active_connections": stats.ActiveConnections, - "start_time": stats.StartTime, - "last_processed": stats.LastProcessed, - "details": stats.Details, + s := snk.GetStats() + sinks = append(sinks, map[string]any{ + "id": s.ID, + "type": s.Type, + "total_processed": s.TotalProcessed, + "active_connections": s.ActiveConnections, + "start_time": s.StartTime, + "last_processed": s.LastProcessed, + "details": s.Details, }) } - // Get flow stats + // 3. Collect flow stats and calculate filtered total var flowStats map[string]any var totalFiltered uint64 + var totalProcessed uint64 + if p.Flow != nil { flowStats = p.Flow.GetStats() - // Extract total_filtered from flow for top-level visibility + + // Map the top-level processed counter directly from Flow's source of truth + if tp, ok := flowStats["total_processed"].(uint64); ok { + totalProcessed = tp + } + + // Calculate total dropped specifically by the filter chain if filters, ok := flowStats["filters"].(map[string]any); ok { if totalPassed, ok := filters["total_passed"].(uint64); ok { - if totalProcessed, ok := filters["total_processed"].(uint64); ok { - totalFiltered = totalProcessed - totalPassed + if tProc, ok := filters["total_processed"].(uint64); ok { + totalFiltered = tProc - totalPassed } } } } + // 4. Calculate Uptime var uptime int if p.running.Load() && !p.Stats.StartTime.IsZero() { uptime = int(time.Since(p.Stats.StartTime).Seconds()) } return map[string]any{ - "name": p.Config.Name, - "running": p.running.Load(), - "uptime_seconds": uptime, - "total_processed": p.Stats.TotalEntriesProcessed.Load(), - "total_filtered": totalFiltered, - "source_count": len(p.Sources), - "sources": sourceStats, - "sink_count": len(p.Sinks), - "sinks": sinkStats, - "flow": flowStats, + "name": p.Config.Name, + "running": p.running.Load(), + "uptime_seconds": uptime, + "total_processed": totalProcessed, + "total_filtered": totalFiltered, + "total_dropped_by_sink": p.Stats.TotalEntriesDroppedBySink.Load(), + "source_count": len(p.Sources), + "sources": sources, + "sink_count": len(p.Sinks), + "sinks": sinks, + "flow": flowStats, } } - -// TODO: incomplete implementation -// startStatsUpdater runs a periodic stats updater -func (p *Pipeline) startStatsUpdater(ctx context.Context) { - go func() { - ticker := time.NewTicker(core.ServiceStatsUpdateInterval) - defer ticker.Stop() - - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - // Periodic stats updates if needed - } - } - }() -} \ No newline at end of file diff --git a/src/internal/plugin/factory.go b/src/internal/plugin/factory.go index 075840c..7ea1287 100644 --- a/src/internal/plugin/factory.go +++ b/src/internal/plugin/factory.go @@ -34,16 +34,6 @@ type PluginMetadata struct { MaxInstances int // 0 = unlimited, 1 = single instance only } -// // global variables holding available source and sink plugins -// var ( -// sourceFactories map[string]SourceFactory -// sinkFactories map[string]SinkFactory -// sourceMetadata map[string]*PluginMetadata -// sinkMetadata map[string]*PluginMetadata -// mu sync.RWMutex -// // once sync.Once -// ) - // registry encapsulates all plugin factories with lazy initialization type registry struct { sourceFactories map[string]SourceFactory @@ -71,11 +61,6 @@ func getRegistry() *registry { return globalRegistry } -// func init() { -// sourceFactories = make(map[string]SourceFactory) -// sinkFactories = make(map[string]SinkFactory) -// } - // RegisterSource registers a source factory function func RegisterSource(name string, constructor SourceFactory) error { r := getRegistry() diff --git a/src/internal/sink/http/http.go b/src/internal/sink/http/http.go index ba05954..9d204c1 100644 --- a/src/internal/sink/http/http.go +++ b/src/internal/sink/http/http.go @@ -170,16 +170,16 @@ func (h *HTTPSink) Start(ctx context.Context) error { addr := fmt.Sprintf("%s:%d", h.config.Host, h.config.Port) - errChan := make(chan error, 1) + ln, err := net.Listen("tcp4", addr) + if err != nil { + return fmt.Errorf("http sink bind %s: %w", addr, err) + } go func() { - h.logger.Info("msg", "HTTP server starting", - "component", "http_sink", - "instance_id", h.id, - "address", addr) - - err := h.server.ListenAndServe(addr) - if err != nil { - errChan <- err + if err := h.server.Serve(ln); err != nil { + h.logger.Error("msg", "HTTP server terminated", + "component", "http_sink", + "instance_id", h.id, + "error", err) } }() @@ -193,18 +193,12 @@ func (h *HTTPSink) Start(ctx context.Context) error { } }() - // Check if server started - select { - case err := <-errChan: - return err - case <-time.After(HttpServerStartTimeout): - h.logger.Info("msg", "HTTP server started", - "component", "http_sink", - "instance_id", h.id, - "host", h.config.Host, - "port", h.config.Port) - return nil - } + h.logger.Info("msg", "HTTP server started", + "component", "http_sink", + "instance_id", h.id, + "host", h.config.Host, + "port", h.config.Port) + return nil } // Stop gracefully shuts down the HTTP server and all client connections @@ -356,6 +350,8 @@ func (h *HTTPSink) requestHandler(ctx *fasthttp.RequestCtx) { remoteAddr := ctx.RemoteAddr() if tcpAddr, ok := remoteAddr.(*net.TCPAddr); ok { if tcpAddr.IP.To4() == nil { + h.logger.Debug("msg", "IPv6 connection rejected", + "component", "http_sink", "remote_addr", remoteAddr.String()) ctx.SetConnectionClose() return } @@ -414,8 +410,6 @@ func (h *HTTPSink) handleStream(ctx *fasthttp.RequestCtx) { "client_id", clientID, "active_clients", connectCount) - h.wg.Add(1) - defer func() { disconnectCount := h.activeClients.Add(-1) h.logger.Debug("msg", "HTTP client disconnected", @@ -431,7 +425,6 @@ func (h *HTTPSink) handleStream(ctx *fasthttp.RequestCtx) { } h.proxy.RemoveSession(sess.ID) - h.wg.Done() }() // Send connected event with metadata @@ -549,4 +542,4 @@ func splitLines(data []byte) [][]byte { return [][]byte{data} } return lines -} \ No newline at end of file +} diff --git a/src/internal/sink/httpchain/httpchain.go b/src/internal/sink/httpchain/httpchain.go new file mode 100644 index 0000000..3bd065a --- /dev/null +++ b/src/internal/sink/httpchain/httpchain.go @@ -0,0 +1,422 @@ +package httpchain + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "logwisp/src/internal/chain" + "logwisp/src/internal/config" + "logwisp/src/internal/core" + "logwisp/src/internal/plugin" + "logwisp/src/internal/session" + "logwisp/src/internal/sink" + + lconfig "github.com/lixenwraith/config" + "github.com/lixenwraith/log" +) + +func init() { + if err := plugin.RegisterSink("http_chain", NewHTTPChainSinkPlugin); err != nil { + panic(fmt.Sprintf("failed to register http_chain sink: %v", err)) + } +} + +const ( + DefaultHTTPChainSinkBufferSize = 1000 + DefaultHTTPChainSinkIngestPath = "/ingest" + DefaultHTTPChainSinkMaxBatchCount = 100 + DefaultHTTPChainSinkMaxBatchBytes = 1024 * 1024 + DefaultHTTPChainSinkFlushIntervalMS = 1000 + DefaultHTTPChainSinkRequestTimeoutMS = 10000 + DefaultHTTPChainSinkBackoffMinMS = 500 + DefaultHTTPChainSinkBackoffMaxMS = 30000 +) + +// HTTPChainSink batches structured entries and posts NDJSON to a downstream +// http_chain source. Delivery is at-least-once per batch. +type HTTPChainSink struct { + id string + proxy *session.Proxy + session *session.Session + config *config.HTTPChainSinkOptions + + node string + url string + + client *http.Client + input chan core.TransportEvent + logger *log.Logger + + // Batch state owned exclusively by run loop goroutine + batch bytes.Buffer + batchCount int64 + + reqTimeout time.Duration + done chan struct{} + wg sync.WaitGroup + startTime time.Time + + totalProcessed atomic.Uint64 + batchesSent atomic.Uint64 + requestErrors atomic.Uint64 + droppedBatches atomic.Uint64 + synthesized atomic.Uint64 + lastProcessed atomic.Value // time.Time +} + +// NewHTTPChainSinkPlugin creates an http_chain sink through plugin factory +func NewHTTPChainSinkPlugin( + id string, + configMap map[string]any, + logger *log.Logger, + proxy *session.Proxy, +) (sink.Sink, error) { + opts := &config.HTTPChainSinkOptions{} + if err := lconfig.ScanMap(configMap, opts); err != nil { + return nil, fmt.Errorf("failed to parse config: %w", err) + } + if err := lconfig.NonEmpty(opts.Host); err != nil { + return nil, fmt.Errorf("host: %w", err) + } + if err := lconfig.Port(opts.Port); err != nil { + return nil, fmt.Errorf("port: %w", err) + } + if opts.IngestPath == "" { + opts.IngestPath = DefaultHTTPChainSinkIngestPath + } else if !strings.HasPrefix(opts.IngestPath, "/") { + return nil, fmt.Errorf("ingest_path: must start with '/'") + } + if opts.BufferSize <= 0 { + opts.BufferSize = DefaultHTTPChainSinkBufferSize + } + if opts.MaxBatchCount <= 0 { + opts.MaxBatchCount = DefaultHTTPChainSinkMaxBatchCount + } + if opts.MaxBatchBytes <= 0 { + opts.MaxBatchBytes = DefaultHTTPChainSinkMaxBatchBytes + } + if opts.FlushIntervalMS <= 0 { + opts.FlushIntervalMS = DefaultHTTPChainSinkFlushIntervalMS + } + if opts.RequestTimeoutMS <= 0 { + opts.RequestTimeoutMS = DefaultHTTPChainSinkRequestTimeoutMS + } + if opts.BackoffMinMS <= 0 { + opts.BackoffMinMS = DefaultHTTPChainSinkBackoffMinMS + } + if opts.BackoffMaxMS < opts.BackoffMinMS { + opts.BackoffMaxMS = DefaultHTTPChainSinkBackoffMaxMS + } + + node := opts.Node + if node == "" { + if hn, err := os.Hostname(); err == nil { + node = hn + } else { + node = "unknown" + } + } + + addr := net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)) + + transport := &http.Transport{ + DialContext: func(ctx context.Context, network, address string) (net.Conn, error) { + // IPv4-only, aligns with tcp/http sinks + d := net.Dialer{} + return d.DialContext(ctx, "tcp4", address) + }, + MaxIdleConnsPerHost: 2, + IdleConnTimeout: 90 * time.Second, + DisableCompression: true, + // Future: TLSClientConfig; HTTP/2 via ALPN once TLS lands + } + + t := &HTTPChainSink{ + id: id, + proxy: proxy, + config: opts, + node: node, + // Future: "https" scheme with TLS + url: "http://" + addr + opts.IngestPath, + client: &http.Client{Transport: transport}, + input: make(chan core.TransportEvent, opts.BufferSize), + done: make(chan struct{}), + logger: logger, + reqTimeout: time.Duration(opts.RequestTimeoutMS) * time.Millisecond, + } + t.lastProcessed.Store(time.Time{}) + + t.session = proxy.CreateSession( + "http_chain://"+addr, + map[string]any{ + "instance_id": id, + "type": "http_chain", + "target": t.url, + "node": node, + }, + ) + + logger.Info("msg", "HTTP chain sink initialized", + "component", "http_chain_sink", + "instance_id", id, + "target", t.url, + "node", node) + return t, nil +} + +// Capabilities returns supported capabilities +func (t *HTTPChainSink) Capabilities() []core.Capability { + // CapTLS/CapAuth added when transport security lands + return []core.Capability{ + core.CapSessionAware, + } +} + +// Input returns the channel for sending transport events +func (t *HTTPChainSink) Input() chan<- core.TransportEvent { + return t.input +} + +// Start launches the batching loop; downstream availability is not required +func (t *HTTPChainSink) Start(ctx context.Context) error { + t.startTime = time.Now() + t.wg.Add(1) + go t.runLoop(ctx) + + t.logger.Info("msg", "HTTP chain sink started", + "component", "http_chain_sink", + "instance_id", t.id, + "target", t.url) + return nil +} + +// Stop terminates the loop. Worst case: one in-flight request timeout plus +// one final-flush request timeout. +func (t *HTTPChainSink) Stop() { + t.logger.Info("msg", "Stopping HTTP chain sink", + "component", "http_chain_sink", + "instance_id", t.id) + + close(t.done) + t.wg.Wait() + t.client.CloseIdleConnections() + + if t.session != nil { + t.proxy.RemoveSession(t.session.ID) + } + + t.logger.Info("msg", "HTTP chain sink stopped", + "component", "http_chain_sink", + "instance_id", t.id, + "total_processed", t.totalProcessed.Load()) +} + +// GetStats returns sink statistics +func (t *HTTPChainSink) GetStats() sink.SinkStats { + lastProc, _ := t.lastProcessed.Load().(time.Time) + return sink.SinkStats{ + ID: t.id, + Type: "http_chain", + TotalProcessed: t.totalProcessed.Load(), + StartTime: t.startTime, + LastProcessed: lastProc, + Details: map[string]any{ + "target": t.url, + "node": t.node, + "batches_sent": t.batchesSent.Load(), + "request_errors": t.requestErrors.Load(), + "dropped_batches": t.droppedBatches.Load(), + "synthesized": t.synthesized.Load(), + }, + } +} + +// runLoop batches events and flushes on size or interval +func (t *HTTPChainSink) runLoop(ctx context.Context) { + defer t.wg.Done() + + // Fold done channel into a context for request/backoff interruption + runCtx, cancel := context.WithCancel(ctx) + defer cancel() + go func() { + select { + case <-t.done: + cancel() + case <-runCtx.Done(): + } + }() + + ticker := time.NewTicker(time.Duration(t.config.FlushIntervalMS) * time.Millisecond) + defer ticker.Stop() + + for { + select { + case <-runCtx.Done(): + t.finalFlush() + return + case <-ticker.C: + if t.batchCount > 0 && !t.flush(runCtx) { + t.finalFlush() + return + } + case event, ok := <-t.input: + if !ok { + t.finalFlush() + return + } + t.append(event) + if t.batchCount >= t.config.MaxBatchCount || + int64(t.batch.Len()) >= t.config.MaxBatchBytes { + if !t.flush(runCtx) { + t.finalFlush() + return + } + } + } + } +} + +// append serializes one event into the pending batch +func (t *HTTPChainSink) append(event core.TransportEvent) { + entry, synthesized := chain.EntryFromEvent(event, t.node, t.id) + if synthesized { + t.synthesized.Add(1) + } + line, err := json.Marshal(entry) + if err != nil { + // Non-transient: drop entry + t.logger.Error("msg", "Failed to marshal chain entry", + "component", "http_chain_sink", + "error", err) + return + } + t.batch.Write(line) + t.batch.WriteByte('\n') + t.batchCount++ +} + +// flush delivers the pending batch, retrying transient failures with backoff. +// Returns false when shutdown interrupts delivery; undelivered batch is dropped +// by finalFlush semantics (batch already consumed here). +func (t *HTTPChainSink) flush(ctx context.Context) bool { + body := bytes.Clone(t.batch.Bytes()) + count := t.batchCount + t.batch.Reset() + t.batchCount = 0 + + failures := 0 + for { + if failures > 0 && !t.waitBackoff(ctx, failures) { + t.droppedBatches.Add(1) + return false + } + transient, err := t.post(ctx, body) + if err == nil { + t.batchesSent.Add(1) + t.totalProcessed.Add(uint64(count)) + t.lastProcessed.Store(time.Now()) + t.proxy.UpdateActivity(t.session.ID) + return true + } + t.requestErrors.Add(1) + if !transient { + t.droppedBatches.Add(1) + t.logger.Error("msg", "Chain batch rejected, dropping", + "component", "http_chain_sink", + "target", t.url, + "entries", count, + "error", err) + return true + } + if ctx.Err() != nil { + t.droppedBatches.Add(1) + return false + } + failures++ + t.logger.Warn("msg", "Chain batch delivery failed", + "component", "http_chain_sink", + "target", t.url, + "attempt", failures, + "error", err) + } +} + +// finalFlush best-effort delivers the pending batch during shutdown (single attempt) +func (t *HTTPChainSink) finalFlush() { + if t.batchCount == 0 { + return + } + fctx, cancel := context.WithTimeout(context.Background(), t.reqTimeout) + defer cancel() + + count := t.batchCount + if _, err := t.post(fctx, t.batch.Bytes()); err != nil { + t.droppedBatches.Add(1) + t.logger.Warn("msg", "Final chain batch dropped on shutdown", + "component", "http_chain_sink", + "entries", count, + "error", err) + return + } + t.batchesSent.Add(1) + t.totalProcessed.Add(uint64(count)) +} + +// post sends one NDJSON batch; transient=true marks retryable failures +func (t *HTTPChainSink) post(ctx context.Context, body []byte) (transient bool, err error) { + reqCtx, cancel := context.WithTimeout(ctx, t.reqTimeout) + defer cancel() + + req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, t.url, bytes.NewReader(body)) + if err != nil { + return false, err + } + req.Header.Set("Content-Type", chain.ContentTypeNDJSON) + req.Header.Set(chain.HeaderProtocol, strconv.Itoa(chain.ProtocolVersion)) + req.Header.Set(chain.HeaderNode, t.node) + // Future: Authorization header for auth + + resp, err := t.client.Do(req) + if err != nil { + return true, err + } + defer resp.Body.Close() + // Drain for connection reuse + io.Copy(io.Discard, resp.Body) + + switch { + case resp.StatusCode >= 200 && resp.StatusCode < 300: + return false, nil + case resp.StatusCode == http.StatusRequestTimeout, + resp.StatusCode == http.StatusTooManyRequests, + resp.StatusCode >= 500: + return true, fmt.Errorf("status %s", resp.Status) + default: + return false, fmt.Errorf("status %s", resp.Status) + } +} + +// waitBackoff sleeps for the computed delay, interruptible by shutdown +func (t *HTTPChainSink) waitBackoff(ctx context.Context, failures int) bool { + minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond + maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond + timer := time.NewTimer(chain.BackoffDelay(minD, maxD, failures)) + defer timer.Stop() + select { + case <-timer.C: + return true + case <-ctx.Done(): + return false + } +} diff --git a/src/internal/sink/tcp/tcp.go b/src/internal/sink/tcp/tcp.go index 9436399..22683f5 100644 --- a/src/internal/sink/tcp/tcp.go +++ b/src/internal/sink/tcp/tcp.go @@ -40,6 +40,7 @@ type TCPSink struct { server *tcpServer engine *gnet.Engine engineMu sync.Mutex + booted chan struct{} // Application input chan core.TransportEvent @@ -63,7 +64,7 @@ type TCPSink struct { const ( // Server lifecycle - TCPServerStartTimeout = 100 * time.Millisecond + TCPServerStartTimeout = 2 * time.Second TCPServerShutdownTimeout = 2 * time.Second // Connection management @@ -151,6 +152,8 @@ func (t *TCPSink) Start(ctx context.Context) error { sink: t, clients: make(map[gnet.Conn]*tcpClient), } + // Fresh channel per Start + t.booted = make(chan struct{}) t.startTime = time.Now() @@ -213,12 +216,25 @@ func (t *TCPSink) Start(ctx context.Context) error { close(t.done) t.wg.Wait() return err - case <-time.After(TCPServerStartTimeout): + // Bind confirmation via OnBoot + case <-t.booted: t.logger.Info("msg", "TCP server started", "component", "tcp_sink", "instance_id", t.id, "port", t.config.Port) return nil + // Timeout failure + case <-time.After(TCPServerStartTimeout): + t.engineMu.Lock() + if t.engine != nil { + stopCtx, cancel := context.WithTimeout(context.Background(), TCPServerShutdownTimeout) + (*t.engine).Stop(stopCtx) + cancel() + } + t.engineMu.Unlock() + close(t.done) + t.wg.Wait() + return fmt.Errorf("tcp sink start timeout on %s", addr) } } @@ -309,6 +325,9 @@ func (s *tcpServer) OnBoot(eng gnet.Engine) gnet.Action { s.sink.engine = &eng s.sink.engineMu.Unlock() + // Listener is bound at this point; unblock Start + close(s.sink.booted) + s.sink.logger.Debug("msg", "TCP server booted", "component", "tcp_sink", "instance_id", s.sink.id) @@ -409,8 +428,10 @@ func (s *tcpServer) OnTraffic(c gnet.Conn) gnet.Action { s.sink.proxy.UpdateActivity(client.sessionID) } - // TCP sink doesn't expect data from clients, discard - c.Discard(-1) + // TCP sink doesn't expect data from clients, discard safely + if bufLen := c.InboundBuffered(); bufLen > 0 { + c.Next(bufLen) + } return gnet.None } @@ -469,4 +490,4 @@ func (t *TCPSink) handleWriteError(c gnet.Conn, err error) { delete(t.consecutiveWriteErrors, c) c.Close() } -} \ No newline at end of file +} diff --git a/src/internal/sink/tcpchain/tcpchain.go b/src/internal/sink/tcpchain/tcpchain.go new file mode 100644 index 0000000..334af5a --- /dev/null +++ b/src/internal/sink/tcpchain/tcpchain.go @@ -0,0 +1,398 @@ +package tcpchain + +import ( + "context" + "encoding/json" + "fmt" + "math/rand/v2" + "net" + "os" + "strconv" + "sync" + "sync/atomic" + "time" + + "logwisp/src/internal/chain" + "logwisp/src/internal/config" + "logwisp/src/internal/core" + "logwisp/src/internal/plugin" + "logwisp/src/internal/session" + "logwisp/src/internal/sink" + + lconfig "github.com/lixenwraith/config" + "github.com/lixenwraith/log" +) + +func init() { + if err := plugin.RegisterSink("tcp_chain", NewTCPChainSinkPlugin); err != nil { + panic(fmt.Sprintf("failed to register tcp_chain sink: %v", err)) + } +} + +const ( + DefaultChainSinkBufferSize = 1000 + DefaultChainSinkDialTimeoutMS = 5000 + DefaultChainSinkWriteTimeoutMS = 5000 + DefaultChainSinkBackoffMinMS = 500 + DefaultChainSinkBackoffMaxMS = 30000 + DefaultChainSinkKeepAlivePeriodMS = 30000 +) + +// TCPChainSink forwards structured entries to a downstream tcp_chain source +type TCPChainSink struct { + id string + proxy *session.Proxy + session *session.Session + config *config.TCPChainSinkOptions + + node string + addr string + helloLine []byte + + input chan core.TransportEvent + logger *log.Logger + + // conn owned exclusively by run loop goroutine + conn net.Conn + everConnected bool + dialTimeout time.Duration + writeTimeout time.Duration + + done chan struct{} + wg sync.WaitGroup + startTime time.Time + + totalProcessed atomic.Uint64 + writeErrors atomic.Uint64 + reconnects atomic.Uint64 + synthesized atomic.Uint64 + connected atomic.Bool + lastProcessed atomic.Value // time.Time +} + +// NewTCPChainSinkPlugin creates a tcp_chain sink through plugin factory +func NewTCPChainSinkPlugin( + id string, + configMap map[string]any, + logger *log.Logger, + proxy *session.Proxy, +) (sink.Sink, error) { + opts := &config.TCPChainSinkOptions{ + KeepAlive: true, + } + if err := lconfig.ScanMap(configMap, opts); err != nil { + return nil, fmt.Errorf("failed to parse config: %w", err) + } + if err := lconfig.NonEmpty(opts.Host); err != nil { + return nil, fmt.Errorf("host: %w", err) + } + if err := lconfig.Port(opts.Port); err != nil { + return nil, fmt.Errorf("port: %w", err) + } + + if opts.BufferSize <= 0 { + opts.BufferSize = DefaultChainSinkBufferSize + } + if opts.DialTimeoutMS <= 0 { + opts.DialTimeoutMS = DefaultChainSinkDialTimeoutMS + } + if opts.WriteTimeoutMS <= 0 { + opts.WriteTimeoutMS = DefaultChainSinkWriteTimeoutMS + } + if opts.BackoffMinMS <= 0 { + opts.BackoffMinMS = DefaultChainSinkBackoffMinMS + } + if opts.BackoffMaxMS < opts.BackoffMinMS { + opts.BackoffMaxMS = DefaultChainSinkBackoffMaxMS + } + if opts.KeepAlivePeriodMS <= 0 { + opts.KeepAlivePeriodMS = DefaultChainSinkKeepAlivePeriodMS + } + + node := opts.Node + if node == "" { + if hn, err := os.Hostname(); err == nil { + node = hn + } else { + node = "unknown" + } + } + + helloLine, err := chain.EncodeHello(node) + if err != nil { + return nil, fmt.Errorf("hello: %w", err) + } + + t := &TCPChainSink{ + id: id, + proxy: proxy, + config: opts, + node: node, + addr: net.JoinHostPort(opts.Host, strconv.FormatInt(opts.Port, 10)), + helloLine: helloLine, + input: make(chan core.TransportEvent, opts.BufferSize), + done: make(chan struct{}), + logger: logger, + dialTimeout: time.Duration(opts.DialTimeoutMS) * time.Millisecond, + writeTimeout: time.Duration(opts.WriteTimeoutMS) * time.Millisecond, + } + t.lastProcessed.Store(time.Time{}) + + t.session = proxy.CreateSession( + "tcp_chain://"+t.addr, + map[string]any{ + "instance_id": id, + "type": "tcp_chain", + "target": t.addr, + "node": node, + }, + ) + + logger.Info("msg", "TCP chain sink initialized", + "component", "tcp_chain_sink", + "instance_id", id, + "target", t.addr, + "node", node) + return t, nil +} + +// Capabilities returns supported capabilities +func (t *TCPChainSink) Capabilities() []core.Capability { + // CapTLS/CapAuth added when transport security lands + return []core.Capability{ + core.CapSessionAware, + } +} + +// Input returns the channel for sending transport events +func (t *TCPChainSink) Input() chan<- core.TransportEvent { + return t.input +} + +// Start launches the forwarding loop; connection is established lazily so +// pipeline start does not depend on downstream availability +func (t *TCPChainSink) Start(ctx context.Context) error { + t.startTime = time.Now() + t.wg.Add(1) + go t.runLoop(ctx) + + t.logger.Info("msg", "TCP chain sink started", + "component", "tcp_chain_sink", + "instance_id", t.id, + "target", t.addr) + return nil +} + +// Stop terminates the forwarding loop. Worst-case latency: one write timeout +// plus one backoff wait (both interruptible or bounded). +func (t *TCPChainSink) Stop() { + t.logger.Info("msg", "Stopping TCP chain sink", + "component", "tcp_chain_sink", + "instance_id", t.id) + + close(t.done) + t.wg.Wait() + + if t.session != nil { + t.proxy.RemoveSession(t.session.ID) + } + + t.logger.Info("msg", "TCP chain sink stopped", + "component", "tcp_chain_sink", + "instance_id", t.id, + "total_processed", t.totalProcessed.Load()) +} + +// GetStats returns sink statistics +func (t *TCPChainSink) GetStats() sink.SinkStats { + lastProc, _ := t.lastProcessed.Load().(time.Time) + var active int64 + if t.connected.Load() { + active = 1 + } + return sink.SinkStats{ + ID: t.id, + Type: "tcp_chain", + TotalProcessed: t.totalProcessed.Load(), + ActiveConnections: active, + StartTime: t.startTime, + LastProcessed: lastProc, + Details: map[string]any{ + "target": t.addr, + "node": t.node, + "connected": t.connected.Load(), + "reconnects": t.reconnects.Load(), + "write_errors": t.writeErrors.Load(), + "synthesized": t.synthesized.Load(), + }, + } +} + +// runLoop consumes transport events and forwards them downstream +func (t *TCPChainSink) runLoop(ctx context.Context) { + defer t.wg.Done() + defer t.closeConn() + + for { + select { + case <-ctx.Done(): + return + case <-t.done: + return + case event, ok := <-t.input: + if !ok { + return + } + entry, synthesized := chain.EntryFromEvent(event, t.node, t.id) + if synthesized { + t.synthesized.Add(1) + } + line, err := json.Marshal(entry) + if err != nil { + // Non-transient: drop + t.logger.Error("msg", "Failed to marshal chain entry", + "component", "tcp_chain_sink", + "error", err) + continue + } + if !t.deliver(ctx, append(line, '\n')) { + return // shutdown during retry + } + t.totalProcessed.Add(1) + t.lastProcessed.Store(time.Now()) + t.proxy.UpdateActivity(t.session.ID) + } + } +} + +// toEntry extracts the structured entry, stamping node identity at first hop +func (t *TCPChainSink) toEntry(event core.TransportEvent) core.LogEntry { + entry := event.Entry + if entry.Time.IsZero() { + // Defensive: event without structured entry, wrap formatted payload + t.synthesized.Add(1) + entry = core.LogEntry{ + Time: event.Time, + Source: t.id, + Message: string(event.Payload), + } + } + if entry.Node == "" { + entry.Node = t.node + } + return entry +} + +// deliver writes one line, holding it across reconnects until sent or shutdown. +// Backpressure during outage propagates to the pipeline dispatch drop counter. +func (t *TCPChainSink) deliver(ctx context.Context, line []byte) bool { + failures := 0 + for { + if t.conn == nil { + if failures > 0 && !t.waitBackoff(ctx, failures) { + return false + } + if err := t.connect(ctx); err != nil { + if ctx.Err() != nil { + return false + } + failures++ + t.logger.Debug("msg", "Chain connect failed", + "component", "tcp_chain_sink", + "target", t.addr, + "attempt", failures, + "error", err) + continue + } + } + + t.conn.SetWriteDeadline(time.Now().Add(t.writeTimeout)) + if _, err := t.conn.Write(line); err != nil { + t.writeErrors.Add(1) + failures++ + t.logger.Warn("msg", "Chain write failed", + "component", "tcp_chain_sink", + "target", t.addr, + "error", err) + t.closeConn() + continue + } + return true + } +} + +// connect performs a single dial + hello attempt +func (t *TCPChainSink) connect(ctx context.Context) error { + d := net.Dialer{Timeout: t.dialTimeout} + if t.config.KeepAlive { + d.KeepAliveConfig = net.KeepAliveConfig{ + Enable: true, + Idle: time.Duration(t.config.KeepAlivePeriodMS) * time.Millisecond, + } + } + + // IPv4-only + conn, err := d.DialContext(ctx, "tcp4", t.addr) + if err != nil { + return err + } + + conn.SetWriteDeadline(time.Now().Add(t.writeTimeout)) + if _, err := conn.Write(t.helloLine); err != nil { + conn.Close() + return fmt.Errorf("hello: %w", err) + } + + t.conn = conn + t.connected.Store(true) + if t.everConnected { + t.reconnects.Add(1) + } + t.everConnected = true + + t.logger.Info("msg", "Chain link established", + "component", "tcp_chain_sink", + "target", t.addr, + "node", t.node) + return nil +} + +// closeConn tears down the current connection (run loop goroutine only) +func (t *TCPChainSink) closeConn() { + if t.conn != nil { + t.conn.Close() + t.conn = nil + } + t.connected.Store(false) +} + +// waitBackoff sleeps for the computed delay, interruptible by shutdown +func (t *TCPChainSink) waitBackoff(ctx context.Context, failures int) bool { + minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond + maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond + timer := time.NewTimer(chain.BackoffDelay(minD, maxD, failures)) + defer timer.Stop() + + select { + case <-timer.C: + return true + case <-ctx.Done(): + return false + case <-t.done: + return false + } +} + +// backoffDelay computes exponential backoff with ±20% jitter +func (t *TCPChainSink) backoffDelay(failures int) time.Duration { + minD := time.Duration(t.config.BackoffMinMS) * time.Millisecond + maxD := time.Duration(t.config.BackoffMaxMS) * time.Millisecond + + d := maxD + if failures < 63 { + if v := minD << uint(failures-1); v > 0 && v < maxD { + d = v + } + } + return d - d/5 + time.Duration(rand.Int64N(int64(2*d/5)+1)) +} diff --git a/src/internal/source/file/file.go b/src/internal/source/file/file.go index aa5f69f..9f6b753 100644 --- a/src/internal/source/file/file.go +++ b/src/internal/source/file/file.go @@ -161,7 +161,7 @@ func (fs *FileSource) Stop() { } fs.wg.Wait() - fs.proxy.RemoveSession(fs.id) + fs.proxy.RemoveSession(fs.session.ID) fs.mu.Lock() for _, w := range fs.watchers { @@ -360,4 +360,5 @@ func globToRegex(glob string) string { regex = strings.ReplaceAll(regex, `\*`, `.*`) regex = strings.ReplaceAll(regex, `\?`, `.`) return "^" + regex + "$" -} \ No newline at end of file +} + diff --git a/src/internal/source/httpchain/httpchain.go b/src/internal/source/httpchain/httpchain.go new file mode 100644 index 0000000..8e8cd86 --- /dev/null +++ b/src/internal/source/httpchain/httpchain.go @@ -0,0 +1,324 @@ +package httpchain + +import ( + "bufio" + "context" + "errors" + "fmt" + "net" + "net/http" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "logwisp/src/internal/chain" + "logwisp/src/internal/config" + "logwisp/src/internal/core" + "logwisp/src/internal/plugin" + "logwisp/src/internal/session" + "logwisp/src/internal/source" + + lconfig "github.com/lixenwraith/config" + "github.com/lixenwraith/log" +) + +func init() { + if err := plugin.RegisterSource("http_chain", NewHTTPChainSourcePlugin); err != nil { + panic(fmt.Sprintf("failed to register http_chain source: %v", err)) + } +} + +const ( + DefaultHTTPChainSourceBufferSize = 1000 + DefaultHTTPChainSourceIngestPath = "/ingest" + DefaultHTTPChainSourceMaxBodyBytes = 8 * 1024 * 1024 + DefaultHTTPChainSourceReadTimeoutMS = 30000 + HTTPChainReadHeaderTimeout = 10 * time.Second + HTTPChainServerShutdownTimeout = 2 * time.Second +) + +// HTTPChainSource accepts NDJSON batches from upstream http_chain sinks +type HTTPChainSource struct { + id string + proxy *session.Proxy + config *config.HTTPChainSourceOptions + + subscribers []chan core.LogEntry + server *http.Server + logger *log.Logger + + // Session cache: one session per remote host + declared node + sessions map[string]string // key -> sessionID + sessionsMu sync.Mutex + + mu sync.RWMutex + + startTime time.Time + totalEntries atomic.Uint64 + droppedEntries atomic.Uint64 + parseErrors atomic.Uint64 + totalRequests atomic.Uint64 + rejectedRequests atomic.Uint64 + lastEntryTime atomic.Value // time.Time +} + +// NewHTTPChainSourcePlugin creates an http_chain source through plugin factory +func NewHTTPChainSourcePlugin( + id string, + configMap map[string]any, + logger *log.Logger, + proxy *session.Proxy, +) (source.Source, error) { + opts := &config.HTTPChainSourceOptions{ + Host: "0.0.0.0", + TrustNode: true, + } + if err := lconfig.ScanMap(configMap, opts); err != nil { + return nil, fmt.Errorf("failed to parse config: %w", err) + } + if err := lconfig.Port(opts.Port); err != nil { + return nil, fmt.Errorf("port: %w", err) + } + if opts.IngestPath == "" { + opts.IngestPath = DefaultHTTPChainSourceIngestPath + } else if !strings.HasPrefix(opts.IngestPath, "/") { + return nil, fmt.Errorf("ingest_path: must start with '/'") + } + if opts.BufferSize <= 0 { + opts.BufferSize = DefaultHTTPChainSourceBufferSize + } + if opts.MaxBodyBytes <= 0 { + opts.MaxBodyBytes = DefaultHTTPChainSourceMaxBodyBytes + } + if opts.ReadTimeoutMS <= 0 { + opts.ReadTimeoutMS = DefaultHTTPChainSourceReadTimeoutMS + } + + s := &HTTPChainSource{ + id: id, + proxy: proxy, + config: opts, + subscribers: make([]chan core.LogEntry, 0), + sessions: make(map[string]string), + logger: logger, + } + s.lastEntryTime.Store(time.Time{}) + + logger.Info("msg", "HTTP chain source initialized", + "component", "http_chain_source", + "instance_id", id, + "host", opts.Host, + "port", opts.Port, + "ingest_path", opts.IngestPath) + return s, nil +} + +// Capabilities returns supported capabilities +func (s *HTTPChainSource) Capabilities() []core.Capability { + // CapTLS/CapAuth added when transport security lands + return []core.Capability{ + core.CapSessionAware, + core.CapMultiSession, + } +} + +// Subscribe returns a channel for receiving log entries +func (s *HTTPChainSource) Subscribe() <-chan core.LogEntry { + s.mu.Lock() + defer s.mu.Unlock() + ch := make(chan core.LogEntry, s.config.BufferSize) + s.subscribers = append(s.subscribers, ch) + return ch +} + +// Start binds the listener and serves the ingest endpoint +func (s *HTTPChainSource) Start() error { + addr := net.JoinHostPort(s.config.Host, strconv.FormatInt(s.config.Port, 10)) + // IPv4-only, aligns with tcp/http sinks + ln, err := net.Listen("tcp4", addr) + if err != nil { + return fmt.Errorf("listen %s: %w", addr, err) + } + + mux := http.NewServeMux() + // Method-scoped pattern: mux answers 405 with Allow header on non-POST + mux.HandleFunc(http.MethodPost+" "+s.config.IngestPath, s.handleIngest) + + s.server = &http.Server{ + Handler: mux, + ReadTimeout: time.Duration(s.config.ReadTimeoutMS) * time.Millisecond, + ReadHeaderTimeout: HTTPChainReadHeaderTimeout, + // Future: TLSConfig for transport security + } + s.startTime = time.Now() + + go func() { + if err := s.server.Serve(ln); err != nil && !errors.Is(err, http.ErrServerClosed) { + s.logger.Error("msg", "HTTP chain server terminated", + "component", "http_chain_source", + "instance_id", s.id, + "error", err) + } + }() + + s.logger.Info("msg", "HTTP chain source started", + "component", "http_chain_source", + "instance_id", s.id, + "addr", addr) + return nil +} + +// Stop shuts down the server, sessions, and subscriber channels +func (s *HTTPChainSource) Stop() { + if s.server != nil { + ctx, cancel := context.WithTimeout(context.Background(), HTTPChainServerShutdownTimeout) + defer cancel() + s.server.Shutdown(ctx) + } + + s.sessionsMu.Lock() + for _, id := range s.sessions { + s.proxy.RemoveSession(id) + } + s.sessions = make(map[string]string) + s.sessionsMu.Unlock() + + s.mu.Lock() + for _, ch := range s.subscribers { + close(ch) + } + s.mu.Unlock() + + s.logger.Info("msg", "HTTP chain source stopped", + "component", "http_chain_source", + "instance_id", s.id) +} + +// GetStats returns the source's statistics +func (s *HTTPChainSource) GetStats() source.SourceStats { + lastEntry, _ := s.lastEntryTime.Load().(time.Time) + + s.sessionsMu.Lock() + cachedSessions := len(s.sessions) + s.sessionsMu.Unlock() + + return source.SourceStats{ + ID: s.id, + Type: "http_chain", + TotalEntries: s.totalEntries.Load(), + DroppedEntries: s.droppedEntries.Load(), + StartTime: s.startTime, + LastEntryTime: lastEntry, + Details: map[string]any{ + "host": s.config.Host, + "port": s.config.Port, + "ingest_path": s.config.IngestPath, + "total_requests": s.totalRequests.Load(), + "rejected_requests": s.rejectedRequests.Load(), + "parse_errors": s.parseErrors.Load(), + "cached_sessions": cachedSessions, + "trust_node": s.config.TrustNode, + }, + } +} + +// handleIngest validates protocol headers and ingests one NDJSON batch. +// Batch acceptance is atomic: entries publish only after a clean full read. +func (s *HTTPChainSource) handleIngest(w http.ResponseWriter, r *http.Request) { + s.totalRequests.Add(1) + + if r.Header.Get(chain.HeaderProtocol) != strconv.Itoa(chain.ProtocolVersion) { + s.rejectedRequests.Add(1) + http.Error(w, "unsupported protocol version", http.StatusBadRequest) + return + } + + remoteHost := r.RemoteAddr + if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil { + remoteHost = host + } + connNode := r.Header.Get(chain.HeaderNode) + if connNode == "" || !s.config.TrustNode { + connNode = remoteHost + } + + body := http.MaxBytesReader(w, r.Body, s.config.MaxBodyBytes) + scanner := bufio.NewScanner(body) + scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes) + + entries := make([]core.LogEntry, 0, 128) + for scanner.Scan() { + line := scanner.Bytes() + if len(line) == 0 { + continue + } + entry, err := chain.DecodeEntry(line, connNode, s.config.TrustNode) + if err != nil { + // Content error within a clean transfer: skip line, keep batch + s.parseErrors.Add(1) + continue + } + entries = append(entries, entry) + } + if err := scanner.Err(); err != nil { + // Transfer error: reject batch without partial ingestion, sender retries + s.rejectedRequests.Add(1) + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + http.Error(w, "body too large", http.StatusRequestEntityTooLarge) + return + } + s.logger.Debug("msg", "Chain batch read failed", + "component", "http_chain_source", + "remote_addr", r.RemoteAddr, + "error", err) + http.Error(w, "malformed body", http.StatusBadRequest) + return + } + + for _, entry := range entries { + s.publish(entry) + } + s.proxy.UpdateActivity(s.sessionFor(remoteHost, connNode)) + + w.Header().Set(chain.HeaderAccepted, strconv.Itoa(len(entries))) + w.WriteHeader(http.StatusNoContent) +} + +// sessionFor returns the cached session for a remote+node, recreating after idle expiry +func (s *HTTPChainSource) sessionFor(remoteHost, node string) string { + key := remoteHost + "|" + node + s.sessionsMu.Lock() + defer s.sessionsMu.Unlock() + + if id, ok := s.sessions[key]; ok { + if _, exists := s.proxy.GetSession(id); exists { + return id + } + } + sess := s.proxy.CreateSession(remoteHost, map[string]any{ + "type": "http_chain", + "node": node, + }) + s.sessions[key] = sess.ID + return sess.ID +} + +// publish sends a log entry to all subscribers +func (s *HTTPChainSource) publish(entry core.LogEntry) { + s.mu.RLock() + defer s.mu.RUnlock() + + s.totalEntries.Add(1) + s.lastEntryTime.Store(entry.Time) + + for _, ch := range s.subscribers { + select { + case ch <- entry: + default: + s.droppedEntries.Add(1) + } + } +} diff --git a/src/internal/source/tcpchain/tcpchain.go b/src/internal/source/tcpchain/tcpchain.go new file mode 100644 index 0000000..f10af83 --- /dev/null +++ b/src/internal/source/tcpchain/tcpchain.go @@ -0,0 +1,353 @@ +package tcpchain + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "net" + "strconv" + "sync" + "sync/atomic" + "time" + + "logwisp/src/internal/chain" + "logwisp/src/internal/config" + "logwisp/src/internal/core" + "logwisp/src/internal/plugin" + "logwisp/src/internal/session" + "logwisp/src/internal/source" + + lconfig "github.com/lixenwraith/config" + "github.com/lixenwraith/log" +) + +func init() { + if err := plugin.RegisterSource("tcp_chain", NewTCPChainSourcePlugin); err != nil { + panic(fmt.Sprintf("failed to register tcp_chain source: %v", err)) + } +} + +const ( + DefaultChainSourceBufferSize = 1000 + DefaultChainSourceHelloTimeoutMS = 10000 +) + +// TCPChainSource accepts connections from upstream tcp_chain sinks and ingests NDJSON entries +type TCPChainSource struct { + id string + proxy *session.Proxy + config *config.TCPChainSourceOptions + + subscribers []chan core.LogEntry + listener net.Listener + conns map[net.Conn]struct{} + logger *log.Logger + + mu sync.RWMutex + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + + startTime time.Time + totalEntries atomic.Uint64 + droppedEntries atomic.Uint64 + parseErrors atomic.Uint64 + rejectedConns atomic.Uint64 + activeConns atomic.Int64 + lastEntryTime atomic.Value // time.Time +} + +// NewTCPChainSourcePlugin creates a tcp_chain source through plugin factory +func NewTCPChainSourcePlugin( + id string, + configMap map[string]any, + logger *log.Logger, + proxy *session.Proxy, +) (source.Source, error) { + opts := &config.TCPChainSourceOptions{ + Host: "0.0.0.0", + TrustNode: true, + } + if err := lconfig.ScanMap(configMap, opts); err != nil { + return nil, fmt.Errorf("failed to parse config: %w", err) + } + if err := lconfig.Port(opts.Port); err != nil { + return nil, fmt.Errorf("port: %w", err) + } + if opts.BufferSize <= 0 { + opts.BufferSize = DefaultChainSourceBufferSize + } + if opts.HelloTimeoutMS <= 0 { + opts.HelloTimeoutMS = DefaultChainSourceHelloTimeoutMS + } + + s := &TCPChainSource{ + id: id, + proxy: proxy, + config: opts, + subscribers: make([]chan core.LogEntry, 0), + conns: make(map[net.Conn]struct{}), + logger: logger, + } + s.lastEntryTime.Store(time.Time{}) + + logger.Info("msg", "TCP chain source initialized", + "component", "tcp_chain_source", + "instance_id", id, + "host", opts.Host, + "port", opts.Port) + return s, nil +} + +// Capabilities returns supported capabilities +func (s *TCPChainSource) Capabilities() []core.Capability { + // CapTLS/CapAuth added when transport security lands + return []core.Capability{ + core.CapSessionAware, + core.CapMultiSession, + } +} + +// Subscribe returns a channel for receiving log entries +func (s *TCPChainSource) Subscribe() <-chan core.LogEntry { + s.mu.Lock() + defer s.mu.Unlock() + ch := make(chan core.LogEntry, s.config.BufferSize) + s.subscribers = append(s.subscribers, ch) + return ch +} + +// Start binds the listener and begins accepting connections +func (s *TCPChainSource) Start() error { + addr := net.JoinHostPort(s.config.Host, strconv.FormatInt(s.config.Port, 10)) + // IPv4-only + ln, err := net.Listen("tcp4", addr) + if err != nil { + return fmt.Errorf("listen %s: %w", addr, err) + } + s.listener = ln + s.ctx, s.cancel = context.WithCancel(context.Background()) + s.startTime = time.Now() + + s.wg.Add(1) + go s.acceptLoop() + + s.logger.Info("msg", "TCP chain source started", + "component", "tcp_chain_source", + "instance_id", s.id, + "addr", addr) + return nil +} + +// Stop closes the listener, all connections, and subscriber channels +func (s *TCPChainSource) Stop() { + if s.cancel != nil { + s.cancel() + } + if s.listener != nil { + s.listener.Close() + } + + s.mu.Lock() + for conn := range s.conns { + conn.Close() // unblocks per-connection reads + } + s.mu.Unlock() + + s.wg.Wait() + + s.mu.Lock() + for _, ch := range s.subscribers { + close(ch) + } + s.mu.Unlock() + + s.logger.Info("msg", "TCP chain source stopped", + "component", "tcp_chain_source", + "instance_id", s.id) +} + +// GetStats returns the source's statistics +func (s *TCPChainSource) GetStats() source.SourceStats { + lastEntry, _ := s.lastEntryTime.Load().(time.Time) + return source.SourceStats{ + ID: s.id, + Type: "tcp_chain", + TotalEntries: s.totalEntries.Load(), + DroppedEntries: s.droppedEntries.Load(), + StartTime: s.startTime, + LastEntryTime: lastEntry, + Details: map[string]any{ + "host": s.config.Host, + "port": s.config.Port, + "active_connections": s.activeConns.Load(), + "rejected_conns": s.rejectedConns.Load(), + "parse_errors": s.parseErrors.Load(), + "trust_node": s.config.TrustNode, + }, + } +} + +// acceptLoop accepts upstream connections until listener close +func (s *TCPChainSource) acceptLoop() { + defer s.wg.Done() + for { + conn, err := s.listener.Accept() + if err != nil { + if errors.Is(err, net.ErrClosed) || s.ctx.Err() != nil { + return + } + s.logger.Warn("msg", "Accept error", + "component", "tcp_chain_source", + "error", err) + continue + } + + if s.config.MaxConnections > 0 && s.activeConns.Load() >= s.config.MaxConnections { + s.rejectedConns.Add(1) + conn.Close() + continue + } + + s.mu.Lock() + s.conns[conn] = struct{}{} + s.mu.Unlock() + + s.wg.Add(1) + go s.handleConn(conn) + } +} + +// handleConn validates the hello preamble, then streams entries until EOF/error +func (s *TCPChainSource) handleConn(conn net.Conn) { + defer s.wg.Done() + remote := conn.RemoteAddr().String() + s.activeConns.Add(1) + + var sessID string + defer func() { + conn.Close() + s.mu.Lock() + delete(s.conns, conn) + s.mu.Unlock() + if sessID != "" { + s.proxy.RemoveSession(sessID) + } + s.activeConns.Add(-1) + }() + + scanner := bufio.NewScanner(conn) + // Oversized line (> MaxLogEntryBytes) is a protocol violation; scanner is + // unrecoverable after ErrTooLong, connection terminates + scanner.Buffer(make([]byte, 0, 64*1024), core.MaxLogEntryBytes) + + // Hello preamble + conn.SetReadDeadline(time.Now().Add(time.Duration(s.config.HelloTimeoutMS) * time.Millisecond)) + if !scanner.Scan() { + s.logger.Warn("msg", "Connection closed before hello", + "component", "tcp_chain_source", + "remote_addr", remote, + "error", scanner.Err()) + return + } + hello, err := chain.DecodeHello(scanner.Bytes()) + if err != nil { + s.logger.Warn("msg", "Rejected chain connection", + "component", "tcp_chain_source", + "remote_addr", remote, + "error", err) + return + } + + connNode := hello.Node + if connNode == "" || !s.config.TrustNode { + if host, _, splitErr := net.SplitHostPort(remote); splitErr == nil { + connNode = host + } else { + connNode = remote + } + } + + sess := s.proxy.CreateSession(remote, map[string]any{ + "type": "tcp_chain", + "node": connNode, + }) + sessID = sess.ID + + s.logger.Info("msg", "Chain connection established", + "component", "tcp_chain_source", + "remote_addr", remote, + "node", connNode) + + idle := time.Duration(s.config.ReadTimeoutMS) * time.Millisecond + for { + if idle > 0 { + conn.SetReadDeadline(time.Now().Add(idle)) + } else { + conn.SetReadDeadline(time.Time{}) + } + if !scanner.Scan() { + if err := scanner.Err(); err != nil && !errors.Is(err, net.ErrClosed) { + s.logger.Debug("msg", "Chain read terminated", + "component", "tcp_chain_source", + "remote_addr", remote, + "error", err) + } + return + } + line := scanner.Bytes() + if len(line) == 0 { + continue + } + s.proxy.UpdateActivity(sessID) + + entry, err := chain.DecodeEntry(line, connNode, s.config.TrustNode) + if err != nil { + s.parseErrors.Add(1) + s.logger.Debug("msg", "Dropped malformed chain entry", + "component", "tcp_chain_source", + "error", err) + continue + } + s.publish(entry) + } +} + +// parseEntry decodes a canonical LogEntry line and applies the node policy +func (s *TCPChainSource) parseEntry(line []byte, connNode string) (core.LogEntry, bool) { + var entry core.LogEntry + if err := json.Unmarshal(line, &entry); err != nil { + s.parseErrors.Add(1) + s.logger.Debug("msg", "Dropped malformed chain entry", + "component", "tcp_chain_source", + "error", err) + return core.LogEntry{}, false + } + if entry.Time.IsZero() { + entry.Time = time.Now() + } + if entry.Node == "" || !s.config.TrustNode { + entry.Node = connNode + } + entry.RawSize = int64(len(line)) + return entry, true +} + +// publish sends a log entry to all subscribers +func (s *TCPChainSource) publish(entry core.LogEntry) { + s.mu.RLock() + defer s.mu.RUnlock() + + s.totalEntries.Add(1) + s.lastEntryTime.Store(entry.Time) + + for _, ch := range s.subscribers { + select { + case ch <- entry: + default: + s.droppedEntries.Add(1) + } + } +}