Merge pull request #164 from askui/feat/filter-logs

feat(chat): filter logs using `ASKUI__CHAT_API__TELEMETRY__LOG__FILTERS`

Author: adi-wan-askui

src/askui/chat/api/settings.py

@@ -6,6 +6,7 @@
 
 from askui.chat.api.mcp_configs.models import McpConfig
 from askui.chat.api.telemetry.integrations.fastapi.settings import TelemetrySettings
+from askui.chat.api.telemetry.logs.settings import LogFilter, LogSettings
 from askui.utils.datetime_utils import now
 
 
@@ -66,5 +67,13 @@ class Settings(BaseSettings):
         ),
     )
     telemetry: TelemetrySettings = Field(
-        default_factory=TelemetrySettings,
+        default_factory=lambda: TelemetrySettings(
+            log=LogSettings(
+                filters=[
+                    LogFilter(type="equals", key="path", value="/v1/health"),
+                    LogFilter(type="equals", key="path", value="/v1/metrics"),
+                    LogFilter(type="equals", key="method", value="OPTIONS"),
+                ],
+            ),
+        ),
     )

src/askui/chat/api/telemetry/integrations/fastapi/__init__.py

@@ -32,4 +32,4 @@ def instrument(
     app.add_middleware(AccessLoggingMiddleware)
     app.add_middleware(CorrelationIdMiddleware)
     app.add_middleware(RawContextMiddleware)
-    Instrumentator().instrument(app).expose(app)
+    Instrumentator().instrument(app).expose(app, endpoint="/v1/metrics")

src/askui/chat/api/telemetry/logs/settings.py

@@ -1,4 +1,5 @@
 import enum
+from typing import Literal
 
 from pydantic import BaseModel
 
@@ -16,6 +17,16 @@ class LogFormat(str, enum.Enum):
     LOGFMT = "logfmt"
 
 
+class EqualsLogFilter(BaseModel):
+    type: Literal["equals"]
+    key: str
+    value: str
+
+
+LogFilter = EqualsLogFilter
+
+
 class LogSettings(BaseModel):
     format: LogFormat = LogFormat.LOGFMT
     level: LogLevel = LogLevel.INFO
+    filters: list[LogFilter] | None = None

src/askui/chat/api/telemetry/logs/structlog.py

@@ -4,6 +4,7 @@
 
 from .settings import LogFormat, LogLevel, LogSettings
 from .structlog_processors import (
+    create_filter_processor,
     drop_color_message_key_processor,
     flatten_dict_processor,
 )
@@ -47,6 +48,7 @@ def get_shared_processors(settings: LogSettings) -> list[structlog.types.Process
     """Returns a list of processors, i.e., a processor chain, that can be shared between
     structlog and stdlib loggers so that their content is consistent."""
     format_dependent_processors = get_format_dependent_processors(settings.format)
+    filter_processor = create_filter_processor(settings.filters)
     return [
         structlog.contextvars.merge_contextvars,
         structlog.stdlib.add_logger_name,
@@ -58,6 +60,7 @@ def get_shared_processors(settings: LogSettings) -> list[structlog.types.Process
         structlog.processors.StackInfoRenderer(),
         *format_dependent_processors,
         structlog.processors.EventRenamer(EVENT_KEY),
+        filter_processor,
     ]
 
 

src/askui/chat/api/telemetry/logs/structlog_processors.py

@@ -1,6 +1,9 @@
 import logging
 
-from structlog.types import EventDict
+from structlog import DropEvent
+from structlog.types import EventDict, Processor
+
+from askui.chat.api.telemetry.logs.settings import LogFilter
 
 from .utils import flatten_dict
 
@@ -27,3 +30,49 @@ def drop_color_message_key_processor(
     """
     event_dict.pop("color_message", None)
     return event_dict
+
+
+def null_processor(
+    logger: logging.Logger,  # noqa: ARG001
+    method_name: str,  # noqa: ARG001
+    event_dict: EventDict,
+) -> EventDict:
+    """
+    A processor that does nothing.
+    """
+    return event_dict
+
+
+def create_filter_processor(filters: list[LogFilter] | None) -> Processor:
+    """
+    Creates a structlog processor that filters out log lines based on field matches.
+
+    Args:
+        filters (dict[str, Any] | None): Dictionary of field names to values to filter out.
+            If a log line has a field with a matching value, it will be filtered out.
+
+    Returns:
+        A structlog processor function that filters log lines.
+    """
+    if not filters:
+        return null_processor
+
+    def filter_processor(
+        logger: logging.Logger,  # noqa: ARG001
+        method_name: str,  # noqa: ARG001
+        event_dict: EventDict,
+    ) -> EventDict:
+        """
+        Filters out log lines where any field matches the filter values.
+        Returns None to drop the log line, or the event_dict to keep it.
+        """
+        for filter_ in filters:
+            if filter_.type == "equals":
+                if (
+                    filter_.key in event_dict
+                    and event_dict[filter_.key] == filter_.value
+                ):
+                    raise DropEvent
+        return event_dict
+
+    return filter_processor