v0.3.3 pipeline rate limiter added

src/internal/service/service.go

@@ -4,6 +4,7 @@ package service
 import (
 	"context"
 	"fmt"
+	"logwisp/src/internal/ratelimit"
 	"sync"
 	"time"
 
@@ -77,6 +78,16 @@ func (s *Service) NewPipeline(cfg config.PipelineConfig) error {
 		pipeline.Sources = append(pipeline.Sources, src)
 	}
 
+	// Create pipeline rate limiter
+	if cfg.RateLimit != nil {
+		limiter, err := ratelimit.New(*cfg.RateLimit, s.logger)
+		if err != nil {
+			pipelineCancel()
+			return fmt.Errorf("failed to create pipeline rate limiter: %w", err)
+		}
+		pipeline.RateLimiter = limiter
+	}
+
 	// Create filter chain
 	if len(cfg.Filters) > 0 {
 		chain, err := filter.NewChain(cfg.Filters, s.logger)
@@ -175,6 +186,14 @@ func (s *Service) wirePipeline(p *Pipeline) {
 
 					p.Stats.TotalEntriesProcessed.Add(1)
 
+					// Apply pipeline rate limiter
+					if p.RateLimiter != nil {
+						if !p.RateLimiter.Allow(entry) {
+							p.Stats.TotalEntriesDroppedByRateLimit.Add(1)
+							continue // Drop the entry
+						}
+					}
+
 					// Apply filters if configured
 					if p.FilterChain != nil {
 						if !p.FilterChain.Apply(entry) {