v0.3.2 doc update

2025-07-12 23:24:41 -04:00
parent e31591ac8d
commit 0accb5f2d3
8 changed files with 648 additions and 183 deletions
--- a/doc/architecture.md
+++ b/doc/architecture.md
@ -14,21 +14,24 @@ LogWisp implements a flexible pipeline architecture for real-time log processing
 │  │  Sources           Filters              Sinks                    │   │
 │  │  ┌──────┐        ┌────────┐          ┌──────┐                    │   │
 │  │  │ Dir  │──┐     │Include │     ┌────│ HTTP │←── Client 1        │   │
-│  │  └──────┘  │     │ ERROR  │     │    └──────┘                    │   │
-│  │            ├────▶│  WARN  │────▶├────┌──────┐                    │   │
-│  │  ┌──────┐  │     └────────┘     │    │ File │                    │   │
-│  │  │ File │──┘          ▼         │    └──────┘                    │   │
-│  │  └──────┘        ┌────────┐     │    ┌──────┐                    │   │
-│  │                  │Exclude │     └────│ TCP  │←── Client 2        │   │
-│  │                  │ DEBUG  │          └──────┘                    │   │
-│  │                  └────────┘                                      │   │
+│  │  └──────┘  ├────▶│ ERROR  │     │    └──────┘                    │   │
+│  │            │     │  WARN  │────▶├────┌──────┐                    │   │
+│  │  ┌──────┐  │     └────┬───┘     │    │ File │                    │   │
+│  │  │ HTTP │──┤          ▼         │    └──────┘                    │   │
+│  │  └──────┘  │     ┌────────┐     │    ┌──────┐                    │   │
+│  │  ┌──────┐  │     │Exclude │     └────│ TCP  │←── Client 2        │   │
+│  │  │ TCP  │──┘     │ DEBUG  │          └──────┘                    │   │
+│  │  └──────┘        └────────┘                                      │   │
 │  └──────────────────────────────────────────────────────────────────┘   │
 │                                                                         │
 │  ┌─────────────────────────── Pipeline 2 ───────────────────────────┐   │
 │  │                                                                  │   │
-│  │  ┌──────┐                            ┌──────┐                    │   │
-│  │  │Stdin │────────────────────────────│Stdout│                    │   │
-│  │  └──────┘         (No Filters)       └──────┘                    │   │
+│  │  ┌──────┐                            ┌───────────┐               │   │
+│  │  │Stdin │───────────────────────┬───▶│HTTP Client│──► Remote     │   │
+│  │  └──────┘         (No Filters)  │    └───────────┘               │   │
+│  │                                 │    ┌───────────┐               │   │
+│  │                                 └────│TCP Client │──► Remote     │   │
+│  │                                      └───────────┘               │   │
 │  └──────────────────────────────────────────────────────────────────┘   │
 │                                                                         │
 │  ┌─────────────────────────── Pipeline N ───────────────────────────┐   │
@ -43,14 +46,13 @@ LogWisp implements a flexible pipeline architecture for real-time log processing
 Log Entry Flow:

 ┌─────────┐     ┌─────────┐     ┌─────────┐     ┌─────────┐
-│  File   │     │  Parse  │     │ Filter  │     │ Format  │
-│ Watcher │────▶│  Entry  │────▶│  Chain  │────▶│  Send   │
+│  Source │     │  Parse  │     │ Filter  │     │  Sink   │
+│ Monitor │────▶│  Entry  │────▶│  Chain  │────▶│ Deliver │
 └─────────┘     └─────────┘     └─────────┘     └─────────┘
     │               │               │               │
     ▼               ▼               ▼               ▼
-  Detect         Extract         Include/        Deliver to
-  Changes        Timestamp       Exclude         Clients
-                 & Level         Patterns
+  Detect         Extract         Include/        Send to
+  Input          & Format        Exclude         Clients


 Entry Processing:
@ -70,7 +72,11 @@ Entry Processing:
   │   TCP    │◀───┼────────── Entry ◀──────────────────┘
   └──────────┘    │           (if passed)
   ┌──────────┐    │
-   │   File   │◀───┘
+   │   File   │◀───┤
+   └──────────┘    │
+   ┌──────────┐    │
+   │ HTTP/TCP │◀───┘
+   │  Client  │
   └──────────┘
 ```

@ -99,6 +105,16 @@ Directory Source:
    │ • Track Pos  │
    │ • Detect Rot │
    └──────────────┘
+
+HTTP/TCP Sources:
+┌─────────────────────────────────┐
+│      Network Listener           │
+├─────────────────────────────────┤
+│ • JSON Parsing                  │
+│ • Rate Limiting                 │
+│ • Connection Management         │
+│ • Input Validation              │
+└─────────────────────────────────┘
 ```

 ### Filters
@ -162,6 +178,20 @@ TCP Sink:
 │    │ • Rate Limiting        │     │
 │    └────────────────────────┘     │
 └───────────────────────────────────┘
+
+Client Sinks:
+┌───────────────────────────────────┐
+│       HTTP/TCP Client             │
+├───────────────────────────────────┤
+│    ┌────────────────────────┐     │
+│    │    Output Manager      │     │
+│    ├────────────────────────┤     │
+│    │ • Batching             │     │
+│    │ • Retry Logic          │     │
+│    │ • Connection Pooling   │     │
+│    │ • Failover             │     │
+│    └────────────────────────┘     │
+└───────────────────────────────────┘
 ```

 ## Router Mode
@ -179,10 +209,10 @@ Router Architecture:
        │                    │                    │
   /app/stream          /db/stream         /sys/stream
        │                    │                    │
-   ┌────▼────┐         ┌────▼────┐         ┌────▼────┐
-   │Pipeline │         │Pipeline │         │Pipeline │
-   │  "app"  │         │  "db"   │         │  "sys"  │
-   └─────────┘         └─────────┘         └─────────┘
+   ┌────▼────┐          ┌────▼────┐          ┌────▼────┐
+   │Pipeline │          │Pipeline │          │Pipeline │
+   │  "app"  │          │  "db"   │          │  "sys"  │
+   └─────────┘          └─────────┘          └─────────┘

 Path Routing:
 Client Request ──▶ Router ──▶ Parse Path ──▶ Find Pipeline ──▶ Route
@ -205,6 +235,13 @@ Buffer Flow:
     ▼                ▼                 ▼
 Drop if full    Backpressure      Drop if full
 (counted)        (blocking)        (counted)
+
+Client Sinks:
+┌──────────┐     ┌──────────┐     ┌──────────┐
+│  Entry   │     │  Batch   │     │  Send    │
+│  Buffer  │────▶│  Buffer  │────▶│  Queue   │
+│ (1000)   │     │ (100)    │     │  (retry) │
+└──────────┘     └──────────┘     └──────────┘
 ```

 ## Rate Limiting
@ -241,9 +278,11 @@ Goroutine Structure:

 Main ────┬──── Pipeline 1 ────┬──── Source Reader 1
         │                    ├──── Source Reader 2
-         │                    ├──── Filter Processor
         │                    ├──── HTTP Server
-         │                    └──── TCP Server
+         │                    ├──── TCP Server
+         │                    ├──── Filter Processor
+         │                    ├──── HTTP Client Writer
+         │                    └──── TCP Client Writer
         │
         ├──── Pipeline 2 ────┬──── Source Reader
         │                    └──── Sink Writers
@ -267,7 +306,7 @@ Priority Order:
 4. Defaults ──────────┘

 Example:
-CLI:     --log-level debug
+CLI:     --logging.level debug
 Env:     LOGWISP_PIPELINES_0_NAME=app
 File:    pipelines.toml
 Default: buffer_size = 1000
--- a/doc/cli.md
+++ b/doc/cli.md
@ -12,7 +12,7 @@ logwisp [options]

 ### `--config <path>`
 Configuration file location.
- **Default**: `~/.config/logwisp.toml`
+- **Default**: `~/.config/logwisp/logwisp.toml`
 - **Example**: `logwisp --config /etc/logwisp/production.toml`

 ### `--router`
@ -27,32 +27,89 @@ Display version information.
 Run as background process.
 - **Example**: `logwisp --background`

+### `--quiet`
+Suppress all output (overrides logging configuration) except sinks.
+- **Example**: `logwisp --quiet`
+
+### `--disable-status-reporter`
+Disable periodic status reporting.
+- **Example**: `logwisp --disable-status-reporter`
+
 ## Logging Options

 Override configuration file settings:

-### `--log-output <mode>`
+### `--logging.output <mode>`
 LogWisp's operational log output.
 - **Values**: `file`, `stdout`, `stderr`, `both`, `none`
- **Example**: `logwisp --log-output both`
+- **Example**: `logwisp --logging.output both`

-### `--log-level <level>`
+### `--logging.level <level>`
 Minimum log level.
 - **Values**: `debug`, `info`, `warn`, `error`
- **Example**: `logwisp --log-level debug`
+- **Example**: `logwisp --logging.level debug`

-### `--log-file <path>`
-Log file path (with file output).
- **Example**: `logwisp --log-file /var/log/logwisp/app.log`
-
-### `--log-dir <directory>`
+### `--logging.file.directory <path>`
 Log directory (with file output).
- **Example**: `logwisp --log-dir /var/log/logwisp`
+- **Example**: `logwisp --logging.file.directory /var/log/logwisp`

-### `--log-console <target>`
+### `--logging.file.name <name>`
+Log file name (with file output).
+- **Example**: `logwisp --logging.file.name app`
+
+### `--logging.file.max_size_mb <size>`
+Maximum log file size in MB.
+- **Example**: `logwisp --logging.file.max_size_mb 200`
+
+### `--logging.file.max_total_size_mb <size>`
+Maximum total log size in MB.
+- **Example**: `logwisp --logging.file.max_total_size_mb 2000`
+
+### `--logging.file.retention_hours <hours>`
+Log retention period in hours.
+- **Example**: `logwisp --logging.file.retention_hours 336`
+
+### `--logging.console.target <target>`
 Console output destination.
 - **Values**: `stdout`, `stderr`, `split`
- **Example**: `logwisp --log-console split`
+- **Example**: `logwisp --logging.console.target split`
+
+### `--logging.console.format <format>`
+Console output format.
+- **Values**: `txt`, `json`
+- **Example**: `logwisp --logging.console.format json`
+
+## Pipeline Options
+
+Configure pipelines via CLI (N = array index, 0-based):
+
+### `--pipelines.N.name <name>`
+Pipeline name.
+- **Example**: `logwisp --pipelines.0.name myapp`
+
+### `--pipelines.N.sources.N.type <type>`
+Source type.
+- **Example**: `logwisp --pipelines.0.sources.0.type directory`
+
+### `--pipelines.N.sources.N.options.<key> <value>`
+Source options.
+- **Example**: `logwisp --pipelines.0.sources.0.options.path /var/log`
+
+### `--pipelines.N.filters.N.type <type>`
+Filter type.
+- **Example**: `logwisp --pipelines.0.filters.0.type include`
+
+### `--pipelines.N.filters.N.patterns <json>`
+Filter patterns (JSON array).
+- **Example**: `logwisp --pipelines.0.filters.0.patterns '["ERROR","WARN"]'`
+
+### `--pipelines.N.sinks.N.type <type>`
+Sink type.
+- **Example**: `logwisp --pipelines.0.sinks.0.type http`
+
+### `--pipelines.N.sinks.N.options.<key> <value>`
+Sink options.
+- **Example**: `logwisp --pipelines.0.sinks.0.options.port 8080`

 ## Examples

@ -68,19 +125,39 @@ logwisp --config /etc/logwisp/production.toml
 ### Development
 ```bash
 # Debug mode
-logwisp --log-output stderr --log-level debug
+logwisp --logging.output stderr --logging.level debug

 # With file output
-logwisp --log-output both --log-level debug --log-dir ./debug-logs
+logwisp --logging.output both --logging.level debug --logging.file.directory ./debug-logs
 ```

 ### Production
 ```bash
 # File logging
-logwisp --log-output file --log-dir /var/log/logwisp
+logwisp --logging.output file --logging.file.directory /var/log/logwisp

 # Background with router
 logwisp --background --router --config /etc/logwisp/prod.toml
+
+# Quiet mode for cron
+logwisp --quiet --config /etc/logwisp/batch.toml
+```
+
+### Pipeline Configuration via CLI
+```bash
+# Simple pipeline
+logwisp --pipelines.0.name app \
+        --pipelines.0.sources.0.type directory \
+        --pipelines.0.sources.0.options.path /var/log/app \
+        --pipelines.0.sinks.0.type http \
+        --pipelines.0.sinks.0.options.port 8080
+
+# With filters
+logwisp --pipelines.0.name filtered \
+        --pipelines.0.sources.0.type stdin \
+        --pipelines.0.filters.0.type include \
+        --pipelines.0.filters.0.patterns '["ERROR","CRITICAL"]' \
+        --pipelines.0.sinks.0.type stdout
 ```

 ## Priority Order
@ -94,9 +171,10 @@ logwisp --background --router --config /etc/logwisp/prod.toml

 - `0`: Success
 - `1`: General error
- `2`: Invalid arguments
+- `2`: Configuration file not found

 ## Signals

 - `SIGINT` (Ctrl+C): Graceful shutdown
 - `SIGTERM`: Graceful shutdown
+- `SIGKILL`: Immediate shutdown (exit code 137)
--- a/doc/configuration.md
+++ b/doc/configuration.md
@ -2,20 +2,86 @@

 LogWisp uses TOML format with a flexible **source → filter → sink** pipeline architecture.

+## Configuration Methods
+
+LogWisp supports three configuration methods with the following precedence:
+
+1. **Command-line flags** (highest priority)
+2. **Environment variables**
+3. **Configuration file** (lowest priority)
+
+### Complete Configuration Reference
+
+| Category | CLI Flag | Environment Variable | TOML File |
+|----------|----------|---------------------|-----------|
+| **Top-level** |
+| Router mode | `--router` | `LOGWISP_ROUTER` | `router = true` |
+| Background mode | `--background` | `LOGWISP_BACKGROUND` | `background = true` |
+| Show version | `--version` | `LOGWISP_VERSION` | `version = true` |
+| Quiet mode | `--quiet` | `LOGWISP_QUIET` | `quiet = true` |
+| Disable status reporter | `--disable-status-reporter` | `LOGWISP_DISABLE_STATUS_REPORTER` | `disable_status_reporter = true` |
+| Config file | `--config <path>` | `LOGWISP_CONFIG_FILE` | N/A |
+| Config directory | N/A | `LOGWISP_CONFIG_DIR` | N/A |
+| **Logging** |
+| Output mode | `--logging.output <mode>` | `LOGWISP_LOGGING_OUTPUT` | `[logging]`<br>`output = "stderr"` |
+| Log level | `--logging.level <level>` | `LOGWISP_LOGGING_LEVEL` | `[logging]`<br>`level = "info"` |
+| File directory | `--logging.file.directory <path>` | `LOGWISP_LOGGING_FILE_DIRECTORY` | `[logging.file]`<br>`directory = "./logs"` |
+| File name | `--logging.file.name <name>` | `LOGWISP_LOGGING_FILE_NAME` | `[logging.file]`<br>`name = "logwisp"` |
+| Max file size | `--logging.file.max_size_mb <size>` | `LOGWISP_LOGGING_FILE_MAX_SIZE_MB` | `[logging.file]`<br>`max_size_mb = 100` |
+| Max total size | `--logging.file.max_total_size_mb <size>` | `LOGWISP_LOGGING_FILE_MAX_TOTAL_SIZE_MB` | `[logging.file]`<br>`max_total_size_mb = 1000` |
+| Retention hours | `--logging.file.retention_hours <hours>` | `LOGWISP_LOGGING_FILE_RETENTION_HOURS` | `[logging.file]`<br>`retention_hours = 168` |
+| Console target | `--logging.console.target <target>` | `LOGWISP_LOGGING_CONSOLE_TARGET` | `[logging.console]`<br>`target = "stderr"` |
+| Console format | `--logging.console.format <format>` | `LOGWISP_LOGGING_CONSOLE_FORMAT` | `[logging.console]`<br>`format = "txt"` |
+| **Pipelines** |
+| Pipeline name | `--pipelines.N.name <name>` | `LOGWISP_PIPELINES_N_NAME` | `[[pipelines]]`<br>`name = "default"` |
+| Source type | `--pipelines.N.sources.N.type <type>` | `LOGWISP_PIPELINES_N_SOURCES_N_TYPE` | `[[pipelines.sources]]`<br>`type = "directory"` |
+| Source options | `--pipelines.N.sources.N.options.<key> <value>` | `LOGWISP_PIPELINES_N_SOURCES_N_OPTIONS_<KEY>` | `[[pipelines.sources]]`<br>`options = { ... }` |
+| Filter type | `--pipelines.N.filters.N.type <type>` | `LOGWISP_PIPELINES_N_FILTERS_N_TYPE` | `[[pipelines.filters]]`<br>`type = "include"` |
+| Filter logic | `--pipelines.N.filters.N.logic <logic>` | `LOGWISP_PIPELINES_N_FILTERS_N_LOGIC` | `[[pipelines.filters]]`<br>`logic = "or"` |
+| Filter patterns | `--pipelines.N.filters.N.patterns <json>` | `LOGWISP_PIPELINES_N_FILTERS_N_PATTERNS` | `[[pipelines.filters]]`<br>`patterns = [...]` |
+| Sink type | `--pipelines.N.sinks.N.type <type>` | `LOGWISP_PIPELINES_N_SINKS_N_TYPE` | `[[pipelines.sinks]]`<br>`type = "http"` |
+| Sink options | `--pipelines.N.sinks.N.options.<key> <value>` | `LOGWISP_PIPELINES_N_SINKS_N_OPTIONS_<KEY>` | `[[pipelines.sinks]]`<br>`options = { ... }` |
+| Auth type | `--pipelines.N.auth.type <type>` | `LOGWISP_PIPELINES_N_AUTH_TYPE` | `[pipelines.auth]`<br>`type = "none"` |
+
+Note: `N` represents array indices (0-based).
+
 ## Configuration File Location

 1. Command line: `--config /path/to/config.toml`
-2. Environment: `$LOGWISP_CONFIG_FILE`
-3. User config: `~/.config/logwisp.toml`
+2. Environment: `$LOGWISP_CONFIG_FILE` and `$LOGWISP_CONFIG_DIR`
+3. User config: `~/.config/logwisp/logwisp.toml`
 4. Current directory: `./logwisp.toml`

 ## Configuration Structure

 ```toml
+# Optional: Enable router mode
+router = false
+
+# Optional: Background mode
+background = false
+
+# Optional: Quiet mode
+quiet = false
+
+# Optional: Disable status reporter
+disable_status_reporter = false
+
 # Optional: LogWisp's own logging
 [logging]
-output = "stderr"
-level = "info"
+output = "stderr"  # file, stdout, stderr, both, none
+level = "info"     # debug, info, warn, error
+
+[logging.file]
+directory = "./logs"
+name = "logwisp"
+max_size_mb = 100
+max_total_size_mb = 1000
+retention_hours = 168
+
+[logging.console]
+target = "stderr"  # stdout, stderr, split
+format = "txt"     # txt or json

 # Required: At least one pipeline
 [[pipelines]]
@ -37,27 +103,6 @@ type = "http"
 options = { ... }
 ```

-## Logging Configuration
-
-Controls LogWisp's operational logging:
-
-```toml
-[logging]
-output = "stderr"  # file, stdout, stderr, both, none
-level = "info"     # debug, info, warn, error
-
-[logging.file]
-directory = "./logs"
-name = "logwisp"
-max_size_mb = 100
-max_total_size_mb = 1000
-retention_hours = 168
-
-[logging.console]
-target = "stderr"  # stdout, stderr, split
-format = "txt"     # txt or json
-```
-
 ## Pipeline Configuration

 Each `[[pipelines]]` section defines an independent processing pipeline.
@ -93,6 +138,39 @@ type = "stdin"
 options = {}
 ```

+#### HTTP Source
+```toml
+[[pipelines.sources]]
+type = "http"
+options = {
+    port = 8081,                  # Port to listen on
+    ingest_path = "/ingest",      # Path for POST requests
+    buffer_size = 1000,           # Input buffer size
+    rate_limit = {                # Optional rate limiting
+        enabled = true,
+        requests_per_second = 10.0,
+        burst_size = 20,
+        limit_by = "ip"
+    }
+}
+```
+
+#### TCP Source
+```toml
+[[pipelines.sources]]
+type = "tcp"
+options = {
+    port = 9091,                  # Port to listen on
+    buffer_size = 1000,           # Input buffer size
+    rate_limit = {                # Optional rate limiting
+        enabled = true,
+        requests_per_second = 5.0,
+        burst_size = 10,
+        limit_by = "ip"
+    }
+}
+```
+
 ### Filters

 Control which log entries pass through:
@ -133,7 +211,8 @@ options = {
        enabled = true,
        interval_seconds = 30,
        format = "comment",  # comment or json
-        include_timestamp = true
+        include_timestamp = true,
+        include_stats = false
    },
    
    # Rate limiting
@ -142,7 +221,10 @@ options = {
        requests_per_second = 10.0,
        burst_size = 20,
        limit_by = "ip",  # ip or global
-        max_connections_per_ip = 5
+        max_connections_per_ip = 5,
+        max_total_connections = 100,
+        response_code = 429,
+        response_message = "Rate limit exceeded"
    }
 }
 ```
@ -154,8 +236,45 @@ type = "tcp"
 options = {
    port = 9090,
    buffer_size = 5000,
-    heartbeat = { enabled = true, interval_seconds = 60 },
-    rate_limit = { enabled = true, requests_per_second = 5.0 }
+    heartbeat = { enabled = true, interval_seconds = 60, format = "json" },
+    rate_limit = { enabled = true, requests_per_second = 5.0, burst_size = 10 }
+}
+```
+
+#### HTTP Client Sink
+```toml
+[[pipelines.sinks]]
+type = "http_client"
+options = {
+    url = "https://remote-log-server.com/ingest",
+    buffer_size = 1000,
+    batch_size = 100,
+    batch_delay_ms = 1000,
+    timeout_seconds = 30,
+    max_retries = 3,
+    retry_delay_ms = 1000,
+    retry_backoff = 2.0,
+    headers = {
+        "Authorization" = "Bearer <API_KEY_HERE>",
+        "X-Custom-Header" = "value"
+    },
+    insecure_skip_verify = false
+}
+```
+
+#### TCP Client Sink
+```toml
+[[pipelines.sinks]]
+type = "tcp_client"
+options = {
+    address = "remote-server.com:9090",
+    buffer_size = 1000,
+    dial_timeout_seconds = 10,
+    write_timeout_seconds = 30,
+    keep_alive_seconds = 30,
+    reconnect_delay_ms = 1000,
+    max_reconnect_delay_seconds = 30,
+    reconnect_backoff = 1.5
 }
 ```

@ -167,7 +286,9 @@ options = {
    directory = "/var/log/logwisp",
    name = "app",
    max_size_mb = 100,
+    max_total_size_mb = 1000,
    retention_hours = 168.0,
+    min_disk_free_mb = 1000,
    buffer_size = 2000
 }
 ```
@ -176,7 +297,10 @@ options = {
 ```toml
 [[pipelines.sinks]]
 type = "stdout"  # or "stderr"
-options = { buffer_size = 500 }
+options = { 
+    buffer_size = 500,
+    target = "stdout"  # stdout, stderr, or split
+}
 ```

 ## Complete Examples
@ -196,7 +320,7 @@ type = "http"
 options = { port = 8080 }
 ```

-### Production with Filtering
+### Filtering

 ```toml
 [logging]
@ -248,6 +372,10 @@ options = { path = "/var/log/app", pattern = "*.log" }
 type = "stdin"
 options = {}

+[[pipelines.sources]]
+type = "http"
+options = { port = 8081, ingest_path = "/logs" }
+
 [[pipelines.sinks]]
 type = "tcp"
 options = { port = 9090 }
@ -257,6 +385,7 @@ options = { port = 9090 }

 ```toml
 # Run with: logwisp --router
+router = true

 [[pipelines]]
 name = "api"
@ -282,11 +411,33 @@ options = { port = 8080 }  # Shared port
 # http://localhost:8080/status
 ```

-## Validation
+### Remote Log Forwarding

-LogWisp validates on startup:
- Required fields (name, sources, sinks)
- Port conflicts between pipelines
- Pattern syntax
- Path accessibility
- Rate limit values
+```toml
+[[pipelines]]
+name = "forwarder"
+
+[[pipelines.sources]]
+type = "directory"
+options = { path = "/var/log/app", pattern = "*.log" }
+
+[[pipelines.filters]]
+type = "include"
+patterns = ["ERROR", "WARN"]
+
+[[pipelines.sinks]]
+type = "http_client"
+options = {
+    url = "https://log-aggregator.example.com/ingest",
+    batch_size = 100,
+    batch_delay_ms = 5000,
+    headers = { "Authorization" = "Bearer <API_KEY_HERE>" }
+}
+
+[[pipelines.sinks]]
+type = "tcp_client"
+options = {
+    address = "backup-logger.example.com:9090",
+    reconnect_delay_ms = 5000
+}
+```
--- a/doc/environment.md
+++ b/doc/environment.md
@ -28,10 +28,28 @@ export LOGWISP_CONFIG_DIR=/etc/logwisp
 export LOGWISP_CONFIG_FILE=production.toml
 ```

+### `LOGWISP_ROUTER`
+Enable router mode.
+```bash
+export LOGWISP_ROUTER=true
+```
+
+### `LOGWISP_BACKGROUND`
+Run in background.
+```bash
+export LOGWISP_BACKGROUND=true
+```
+
+### `LOGWISP_QUIET`
+Suppress all output.
+```bash
+export LOGWISP_QUIET=true
+```
+
 ### `LOGWISP_DISABLE_STATUS_REPORTER`
 Disable periodic status reporting.
 ```bash
-export LOGWISP_DISABLE_STATUS_REPORTER=1
+export LOGWISP_DISABLE_STATUS_REPORTER=true
 ```

 ## Logging Variables
@ -47,11 +65,15 @@ LOGWISP_LOGGING_LEVEL=debug
 LOGWISP_LOGGING_FILE_DIRECTORY=/var/log/logwisp
 LOGWISP_LOGGING_FILE_NAME=logwisp
 LOGWISP_LOGGING_FILE_MAX_SIZE_MB=100
+LOGWISP_LOGGING_FILE_MAX_TOTAL_SIZE_MB=1000
 LOGWISP_LOGGING_FILE_RETENTION_HOURS=168

 # Console logging
 LOGWISP_LOGGING_CONSOLE_TARGET=stderr
 LOGWISP_LOGGING_CONSOLE_FORMAT=json
+
+# Special console target override
+LOGWISP_CONSOLE_TARGET=split  # Overrides sink console targets
 ```

 ## Pipeline Configuration
@ -85,6 +107,25 @@ LOGWISP_PIPELINES_0_FILTERS_1_TYPE=exclude
 LOGWISP_PIPELINES_0_FILTERS_1_PATTERNS='["DEBUG"]'
 ```

+### HTTP Source
+```bash
+LOGWISP_PIPELINES_0_SOURCES_0_TYPE=http
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_PORT=8081
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_INGEST_PATH=/ingest
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_BUFFER_SIZE=1000
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_RATE_LIMIT_ENABLED=true
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_RATE_LIMIT_REQUESTS_PER_SECOND=10.0
+```
+
+### TCP Source
+```bash
+LOGWISP_PIPELINES_0_SOURCES_0_TYPE=tcp
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_PORT=9091
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_BUFFER_SIZE=1000
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_RATE_LIMIT_ENABLED=true
+LOGWISP_PIPELINES_0_SOURCES_0_OPTIONS_RATE_LIMIT_REQUESTS_PER_SECOND=5.0
+```
+
 ### HTTP Sink Options
 ```bash
 # Basic
@ -95,18 +136,69 @@ LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_STATUS_PATH=/status
 LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_HEARTBEAT_ENABLED=true
 LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_HEARTBEAT_INTERVAL_SECONDS=30
 LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_HEARTBEAT_FORMAT=comment
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_HEARTBEAT_INCLUDE_TIMESTAMP=true
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_HEARTBEAT_INCLUDE_STATS=false

 # Rate Limiting
 LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_ENABLED=true
 LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_REQUESTS_PER_SECOND=10.0
 LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_BURST_SIZE=20
 LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_LIMIT_BY=ip
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_MAX_CONNECTIONS_PER_IP=5
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RATE_LIMIT_MAX_TOTAL_CONNECTIONS=100
+```
+
+### HTTP Client Sink
+```bash
+LOGWISP_PIPELINES_0_SINKS_0_TYPE=http_client
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_URL=https://log-server.com/ingest
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_BATCH_SIZE=100
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_BATCH_DELAY_MS=5000
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_TIMEOUT_SECONDS=30
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_MAX_RETRIES=3
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RETRY_DELAY_MS=1000
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RETRY_BACKOFF=2.0
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_HEADERS='{"Authorization":"Bearer <API_KEY_HERE>"}'
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_INSECURE_SKIP_VERIFY=false
+```
+
+### TCP Client Sink
+```bash
+LOGWISP_PIPELINES_0_SINKS_0_TYPE=tcp_client
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_ADDRESS=remote-server.com:9090
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_DIAL_TIMEOUT_SECONDS=10
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_WRITE_TIMEOUT_SECONDS=30
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_KEEP_ALIVE_SECONDS=30
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RECONNECT_DELAY_MS=1000
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_MAX_RECONNECT_DELAY_SECONDS=30
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RECONNECT_BACKOFF=1.5
+```
+
+### File Sink
+```bash
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_DIRECTORY=/var/log/logwisp
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_NAME=app
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_MAX_SIZE_MB=100
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_MAX_TOTAL_SIZE_MB=1000
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_RETENTION_HOURS=168
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_MIN_DISK_FREE_MB=1000
+```
+
+### Console Sinks
+```bash
+LOGWISP_PIPELINES_0_SINKS_0_TYPE=stdout
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_BUFFER_SIZE=500
+LOGWISP_PIPELINES_0_SINKS_0_OPTIONS_TARGET=stdout
 ```

 ## Example

 ```bash
-#!/bin/bash
+#!/usr/bin/env bash
+
+# General settings
+export LOGWISP_ROUTER=true
+export LOGWISP_DISABLE_STATUS_REPORTER=false

 # Logging
 export LOGWISP_LOGGING_OUTPUT=both
@ -137,6 +229,18 @@ export LOGWISP_PIPELINES_1_SOURCES_0_OPTIONS_PATH=/var/log/syslog
 export LOGWISP_PIPELINES_1_SINKS_0_TYPE=tcp
 export LOGWISP_PIPELINES_1_SINKS_0_OPTIONS_PORT=9090

+# Pipeline 2: Remote forwarding
+export LOGWISP_PIPELINES_2_NAME=forwarder
+export LOGWISP_PIPELINES_2_SOURCES_0_TYPE=http
+export LOGWISP_PIPELINES_2_SOURCES_0_OPTIONS_PORT=8081
+export LOGWISP_PIPELINES_2_SOURCES_0_OPTIONS_INGEST_PATH=/logs
+
+# HTTP client sink
+export LOGWISP_PIPELINES_2_SINKS_0_TYPE=http_client
+export LOGWISP_PIPELINES_2_SINKS_0_OPTIONS_URL=https://log-aggregator.example.com/ingest
+export LOGWISP_PIPELINES_2_SINKS_0_OPTIONS_BATCH_SIZE=100
+export LOGWISP_PIPELINES_2_SINKS_0_OPTIONS_HEADERS='{"Authorization":"Bearer <API_KEY_HERE>"}'
+
 logwisp
 ```

--- a/doc/filters.md
+++ b/doc/filters.md
@ -173,3 +173,96 @@ logwisp --log-level debug
 # Check output
 curl -N http://localhost:8080/stream
 ```
+
+## Regex Pattern Guide
+
+LogWisp uses Go's standard regex engine (RE2). It includes most common features but omits backtracking-heavy syntax.
+
+For complex logic, chain multiple filters (e.g., an `include` followed by an `exclude`) rather than writing one complex regex.
+
+### Basic Matching
+
+| Pattern | Description | Example |
+| :--- | :--- | :--- |
+| `literal` | Matches the exact text. | `"ERROR"` matches any log with "ERROR". |
+| `.` | Matches any single character (except newline). | `"user."` matches "userA", "userB", etc. |
+| `a\|b` | Matches expression `a` OR expression `b`. | `"error\|fail"` matches lines with "error" or "fail". |
+
+### Anchors and Boundaries
+
+Anchors tie your pattern to a specific position in the line.
+
+| Pattern | Description | Example |
+| :--- | :--- | :--- |
+| `^` | Matches the beginning of the line. | `"^ERROR"` matches lines *starting* with "ERROR". |
+| `$` | Matches the end of the line. | `"crashed$"` matches lines *ending* with "crashed". |
+| `\b` | Matches a word boundary. | `"\berror\b"` matches "error" but not "terrorist". |
+
+### Character Classes
+
+| Pattern | Description | Example |
+| :--- | :--- | :--- |
+| `[abc]` | Matches `a`, `b`, or `c`. | `"[aeiou]"` matches any vowel. |
+| `[^abc]` | Matches any character *except* `a`, `b`, or `c`. | `"[^0-9]"` matches any non-digit. |
+| `[a-z]` | Matches any character in the range `a` to `z`. | `"[a-zA-Z]"` matches any letter. |
+| `\d` | Matches any digit (`[0-9]`). | `\d{3}` matches three digits, like "123". |
+| `\w` | Matches any word character (`[a-zA-Z0-9_]`). | `\w+` matches one or more word characters. |
+| `\s` | Matches any whitespace character. | `\s+` matches one or more spaces or tabs. |
+
+### Quantifiers
+
+Quantifiers specify how many times a character or group must appear.
+
+| Pattern | Description | Example |
+| :--- | :--- | :--- |
+| `*` | Zero or more times. | `"a*"` matches "", "a", "aa". |
+| `+` | One or more times. | `"a+"` matches "a", "aa", but not "". |
+| `?` | Zero or one time. | `"colou?r"` matches "color" and "colour". |
+| `{n}` | Exactly `n` times. | `\d{4}` matches a 4-digit number. |
+| `{n,}` | `n` or more times. | `\d{2,}` matches numbers with 2 or more digits. |
+| `{n,m}` | Between `n` and `m` times. | `\d{1,3}` matches numbers with 1 to 3 digits. |
+
+### Grouping
+
+| Pattern | Description | Example |
+| :--- | :--- | :--- |
+| `(...)` | Groups an expression and captures the match. | `(ERROR|WARN)` captures "ERROR" or "WARN". |
+| `(?:...)`| Groups an expression *without* capturing. Faster. | `(?:ERROR|WARN)` is more efficient if you just need to group. |
+
+### Flags and Modifiers
+
+Flags are placed at the beginning of a pattern to change its behavior.
+
+| Pattern | Description |
+| :--- | :--- |
+| `(?i)` | Case-insensitive matching. |
+| `(?m)` | Multi-line mode (`^` and `$` match start/end of lines). |
+
+**Example:** `"(?i)error"` matches "error", "ERROR", and "Error".
+
+### Practical Examples for Logging
+
+*   **Match an IP Address:**
+    ```
+    \b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b
+    ```
+
+*   **Match HTTP 4xx or 5xx Status Codes:**
+    ```
+    "status[= ](4|5)\d{2}"
+    ```
+
+*   **Match a slow database query (>100ms):**
+    ```
+    "Query took [1-9]\d{2,}ms"
+    ```
+
+*   **Match key-value pairs:**
+    ```
+    "user=(admin|guest)"
+    ```
+
+*   **Match Java exceptions:**
+    ```
+    "Exception:|at .+\.java:\d+"
+    ```
--- a/doc/installation.md
+++ b/doc/installation.md
@ -6,7 +6,7 @@ Installation process on tested platforms.

 - **OS**: Linux, FreeBSD
 - **Architecture**: amd64
- **Go**: 1.23+ (for building)
+- **Go**: 1.24+ (for building)

 ## Installation

@ -14,15 +14,10 @@ Installation process on tested platforms.

 ```bash
 # Linux amd64
-wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-linux-amd64
+wget https://github.com/lixenwraith/logwisp/releases/latest/download/logwisp-linux-amd64
 chmod +x logwisp-linux-amd64
 sudo mv logwisp-linux-amd64 /usr/local/bin/logwisp

-# macOS Intel
-wget https://github.com/yourusername/logwisp/releases/latest/download/logwisp-darwin-amd64
-chmod +x logwisp-darwin-amd64
-sudo mv logwisp-darwin-amd64 /usr/local/bin/logwisp
-
 # Verify
 logwisp --version
 ```
@ -30,7 +25,7 @@ logwisp --version
 ### From Source

 ```bash
-git clone https://github.com/yourusername/logwisp.git
+git clone https://github.com/lixenwraith/logwisp.git
 cd logwisp
 make build
 sudo make install
--- a/doc/quickstart.md
+++ b/doc/quickstart.md
@ -1,6 +1,6 @@
 # Quick Start Guide

-Get LogWisp up and running in 5 minutes with the new pipeline architecture!
+Get LogWisp up and running in minutes:

 ## Installation

@ -51,7 +51,7 @@ echo "[WARN] Low memory warning" >> test.log

 ### Monitor Specific Directory

-Create `~/.config/logwisp.toml`:
+Create `~/.config/logwisp/logwisp.toml`:

 ```toml
 [[pipelines]]
@ -143,11 +143,46 @@ logwisp --router
 # http://localhost:8080/status (global)
 ```

+### Remote Log Collection
+
+Receive logs via HTTP/TCP and forward to remote servers:
+
+```toml
+[[pipelines]]
+name = "collector"
+
+# Receive logs via HTTP POST
+[[pipelines.sources]]
+type = "http"
+options = { port = 8081, ingest_path = "/ingest" }
+
+# Forward to remote server
+[[pipelines.sinks]]
+type = "http_client"
+options = {
+    url = "https://log-server.com/ingest",
+    batch_size = 100,
+    headers = { "Authorization" = "Bearer <API_KEY_HERE>" }
+}
+```
+
+Send logs to collector:
+```bash
+curl -X POST http://localhost:8081/ingest \
+  -H "Content-Type: application/json" \
+  -d '{"message": "Test log", "level": "INFO"}'
+```
+
 ## Quick Tips

 ### Enable Debug Logging
 ```bash
-logwisp --log-level debug --log-output stderr
+logwisp --logging.level debug --logging.output stderr
+```
+
+### Quiet Mode
+```bash
+logwisp --quiet
 ```

 ### Rate Limiting
@ -170,3 +205,11 @@ options = {
 type = "stdout"  # or "stderr"
 options = {}
 ```
+
+### Split Console Output
+```toml
+# INFO/DEBUG to stdout, ERROR/WARN to stderr
+[[pipelines.sinks]]
+type = "stdout"
+options = { target = "split" }
+```
--- a/doc/status.md
+++ b/doc/status.md
@ -24,7 +24,8 @@ Example response:
    "port": 8080,
    "active_clients": 5,
    "buffer_size": 1000,
-    "uptime_seconds": 3600
+    "uptime_seconds": 3600,
+    "mode": {"standalone": true, "router": false}
  },
  "sources": [{
    "type": "directory",
@ -40,8 +41,36 @@ Example response:
  "sinks": [{
    "type": "http",
    "total_processed": 48234,
-    "active_connections": 5
-  }]
+    "active_connections": 5,
+    "details": {
+      "port": 8080,
+      "buffer_size": 1000,
+      "rate_limit": {
+        "enabled": true,
+        "total_requests": 98234,
+        "blocked_requests": 234
+      }
+    }
+  }],
+  "endpoints": {
+    "transport": "/stream",
+    "status": "/status"
+  },
+  "features": {
+    "heartbeat": {
+      "enabled": true,
+      "interval": 30,
+      "format": "comment"
+    },
+    "ssl": {
+      "enabled": false
+    },
+    "rate_limit": {
+      "enabled": true,
+      "requests_per_second": 10.0,
+      "burst_size": 20
+    }
+  }
 }
 ```

@ -53,18 +82,29 @@ Example response:
 | `active_watchers` | Files being watched | 1-1000 |
 | `total_entries` | Entries processed | Increasing |
 | `dropped_entries` | Buffer overflows | < 1% of total |
+| `active_connections` | Network connections (HTTP/TCP sources) | Within limits |

 ### Sink Metrics
 | Metric | Description | Warning Signs |
 |--------|-------------|---------------|
 | `active_connections` | Current clients | Near limit |
 | `total_processed` | Entries sent | Should match filter output |
+| `total_batches` | Batches sent (client sinks) | Increasing |
+| `failed_batches` | Failed sends (client sinks) | > 0 indicates issues |

 ### Filter Metrics
 | Metric | Description | Notes |
 |--------|-------------|-------|
 | `total_processed` | Entries checked | All entries |
 | `total_passed` | Passed filters | Check if too low/high |
+| `total_matched` | Pattern matches | Per filter stats |
+
+### Rate Limit Metrics
+| Metric | Description | Action |
+|--------|-------------|--------|
+| `blocked_requests` | Rejected requests | Increase limits if high |
+| `active_ips` | Unique IPs tracked | Monitor for attacks |
+| `total_connections` | Current connections | Check against limits |

 ## Operational Logging

@ -74,32 +114,11 @@ Example response:
 level = "info"  # debug, info, warn, error
 ```

-### Important Messages
-
-**Startup**:
-```
-LogWisp starting version=1.0.0
-Pipeline created successfully pipeline=app
-HTTP server started port=8080
-```
-
-**Connections**:
-```
-HTTP client connected remote_addr=192.168.1.100 active_clients=6
-TCP connection opened active_connections=3
-```
-
-**Errors**:
-```
-Failed to open file path=/var/log/app.log error=permission denied
-Request rate limited ip=192.168.1.100
-```
-
 ## Health Checks

 ### Basic Check
 ```bash
-#!/bin/bash
+#!/usr/bin/env bash
 if curl -s -f http://localhost:8080/status > /dev/null; then
    echo "Healthy"
 else
@ -110,7 +129,7 @@ fi

 ### Advanced Check
 ```bash
-#!/bin/bash
+#!/usr/bin/env bash
 STATUS=$(curl -s http://localhost:8080/status)
 DROPPED=$(echo "$STATUS" | jq '.sources[0].dropped_entries')
 TOTAL=$(echo "$STATUS" | jq '.sources[0].total_entries')
@ -119,68 +138,11 @@ if [ $((DROPPED * 100 / TOTAL)) -gt 5 ]; then
    echo "High drop rate"
    exit 1
 fi
-```
-
-### Docker
-```dockerfile
-HEALTHCHECK --interval=30s --timeout=3s \
-    CMD curl -f http://localhost:8080/status || exit 1
-```
-
-## Integration
-
-### Prometheus Export
-```bash
-#!/bin/bash
-STATUS=$(curl -s http://localhost:8080/status)
-
-cat << EOF
-# HELP logwisp_active_clients Active streaming clients
-# TYPE logwisp_active_clients gauge
-logwisp_active_clients $(echo "$STATUS" | jq '.server.active_clients')
-
-# HELP logwisp_total_entries Total log entries
-# TYPE logwisp_total_entries counter
-logwisp_total_entries $(echo "$STATUS" | jq '.sources[0].total_entries')
-EOF
-```
-
-### Alerts
-
-| Alert | Condition | Severity |
-|-------|-----------|----------|
-| Service Down | Status fails | Critical |
-| High Drops | >10% dropped | Warning |
-| No Activity | 0 entries/min | Warning |
-| Rate Limited | >20% blocked | Warning |
-
-## Performance Monitoring
-
-### CPU Usage
-```bash
-top -p $(pgrep logwisp)
-```
-
-### Memory Usage
-```bash
-ps aux | grep logwisp
-```
-
-### Connections
-```bash
-ss -tan | grep :8080 | wc -l
-```
-
-## Troubleshooting
-
-Enable debug logging:
-```bash
-logwisp --log-level debug --log-output stderr
-```
-
-Check specific components:
-```bash
-curl -s http://localhost:8080/status | jq '.sources'
-curl -s http://localhost:8080/status | jq '.filters'
-curl -s http://localhost:8080/status | jq '.sinks'
+
+# Check client sink failures
+FAILED=$(echo "$STATUS" | jq '.sinks[] | select(.type=="http_client") | .details.failed_batches // 0' | head -1)
+if [ "$FAILED" -gt 10 ]; then
+    echo "High failure rate"
+    exit 1
+fi
 ```