diff --git a/src/content/docs/explanations/configuration.mdx b/src/content/docs/explanations/configuration.mdx index 21a32fb14..c4edf82a6 100644 --- a/src/content/docs/explanations/configuration.mdx +++ b/src/content/docs/explanations/configuration.mdx @@ -141,8 +141,8 @@ Tenzir provides node-level TLS configuration that applies to all operators and connectors using TLS/HTTPS connections. These settings are used by operators that make outbound connections (e.g., to_opensearch, to_splunk, save_email) -and those that accept inbound connections (e.g., load_tcp, -save_tcp). +and those that accept inbound connections (e.g., accept_tcp, +serve_tcp). :::note[Use Only When Required] We do not recommend manually configuring TLS settings unless required for @@ -192,8 +192,9 @@ configuration: - to_opensearch: Applies min version and ciphers to HTTPS connections - to_splunk: Applies min version and ciphers to Splunk HEC connections - save_email: Applies min version and ciphers to SMTP connections -- load_tcp: Applies min version and ciphers to TLS server mode -- save_tcp: Applies min version and ciphers to TLS client and server modes +- accept_tcp: Applies min version and ciphers to TLS server mode +- from_tcp: Applies min version and ciphers to TLS client mode +- serve_tcp: Applies min version and ciphers to TLS server mode - from_opensearch: Applies min version and ciphers to HTTPS connections ## Plugins diff --git a/src/content/docs/guides/collecting/get-data-from-the-network.mdx b/src/content/docs/guides/collecting/get-data-from-the-network.mdx index f20c9a073..d5007f5fa 100644 --- a/src/content/docs/guides/collecting/get-data-from-the-network.mdx +++ b/src/content/docs/guides/collecting/get-data-from-the-network.mdx @@ -8,16 +8,16 @@ capture raw packets from network interfaces. ## TCP sockets -The [Transmission Control Protocol (TCP)](/integrations/tcp) provides reliable, -ordered byte streams. Use TCP when you need guaranteed delivery and message -ordering. +tcp provides reliable, ordered byte streams. Use TCP +when you need guaranteed delivery and message ordering. ### Listen for connections -Start a TCP server that accepts incoming connections: +Use accept_tcp to start a TCP server that accepts incoming +connections: ```tql -from "tcp://0.0.0.0:9000" { +accept_tcp "0.0.0.0:9000" { read_json } ``` @@ -27,10 +27,10 @@ pipeline to convert incoming bytes to events. ### Connect to a remote server -Act as a TCP client by connecting to an existing server: +Use from_tcp to connect to an existing server: ```tql -from "tcp://192.168.1.100:9000", connect=true { +from_tcp "192.168.1.100:9000" { read_json } ``` @@ -40,7 +40,7 @@ from "tcp://192.168.1.100:9000", connect=true { Secure your TCP connections with TLS by passing a `tls` record: ```tql -from "tcp://0.0.0.0:9443", tls={certfile: "cert.pem", keyfile: "key.pem"} { +accept_tcp "0.0.0.0:9443", tls={certfile: "cert.pem", keyfile: "key.pem"} { read_json } ``` @@ -56,9 +56,8 @@ For production TLS configuration, including mutual TLS and cipher settings, see ## UDP sockets -The [User Datagram Protocol (UDP)](/integrations/udp) is a connectionless -protocol ideal for high-volume, loss-tolerant data like syslog messages or -metrics. +udp is a connectionless protocol ideal for +high-volume, loss-tolerant data like syslog messages or metrics. ### Receive UDP datagrams @@ -99,8 +98,8 @@ this = { ## Packet capture -Capture raw network packets from a [network interface card (NIC)](/integrations/nic) -for deep packet inspection or network forensics. +Capture raw network packets with nic for deep +packet inspection or network forensics. ### List available interfaces diff --git a/src/content/docs/guides/node-setup/configure-tls.mdx b/src/content/docs/guides/node-setup/configure-tls.mdx index dde429be4..498841099 100644 --- a/src/content/docs/guides/node-setup/configure-tls.mdx +++ b/src/content/docs/guides/node-setup/configure-tls.mdx @@ -21,8 +21,8 @@ tenzir: ``` These settings apply automatically to operators like from_http, -load_tcp, save_tcp, -to_opensearch, from_opensearch, +accept_tcp, from_tcp, +serve_tcp, to_opensearch, from_opensearch, to_splunk, save_email, and to_fluent_bit. diff --git a/src/content/docs/guides/routing/load-balance-pipelines.mdx b/src/content/docs/guides/routing/load-balance-pipelines.mdx index 094fc4004..1b859a988 100644 --- a/src/content/docs/guides/routing/load-balance-pipelines.mdx +++ b/src/content/docs/guides/routing/load-balance-pipelines.mdx @@ -17,8 +17,7 @@ nested pipelines, enabling you to spread load across multiple destinations. let $endpoints = ["host1:8080", "host2:8080", "host3:8080"] subscribe "events" load_balance $endpoints { - write_json - save_tcp $endpoints + to_tcp $endpoints { write_json } } ``` @@ -36,8 +35,7 @@ let $cfg = ["192.168.0.30:8080", "192.168.0.31:8080"] subscribe "input" load_balance $cfg { - write_json - save_tcp $cfg + to_tcp $cfg { write_json } } ``` diff --git a/src/content/docs/guides/routing/send-to-destinations.mdx b/src/content/docs/guides/routing/send-to-destinations.mdx index 171f54727..5e0deb9f4 100644 --- a/src/content/docs/guides/routing/send-to-destinations.mdx +++ b/src/content/docs/guides/routing/send-to-destinations.mdx @@ -57,8 +57,8 @@ to_opensearch "https://opensearch.example.com:9200", ### Cloud services -Route events to cloud destinations like [Amazon SQS](/integrations/amazon/sqs) -and [Google Cloud Pub/Sub](/integrations/google/cloud-pubsub). +Route events to cloud destinations like amazon/sqs +and google/cloud-pubsub. Send to SQS: @@ -105,8 +105,7 @@ save_file "s3://bucket/logs/events.jsonl" Send NDJSON over tcp: ```tql -write_json -save_tcp "collector.example.com:5044" +to_tcp "collector.example.com:5044" { write_json } ``` ## Expression-based serialization diff --git a/src/content/docs/integrations/index.mdx b/src/content/docs/integrations/index.mdx index 80d263c57..251fc4867 100644 --- a/src/content/docs/integrations/index.mdx +++ b/src/content/docs/integrations/index.mdx @@ -9,43 +9,33 @@ packages at the top to native protocol connectors at the core. ## Packages -packages are 1-click deployable integrations that deliver instant value. -They bundle pipelines, [enrichment contexts](/explanations/enrichment/), and -configurations for common security tools like Splunk, CrowdStrike, Elastic, -SentinelOne, Palo Alto, and many more. +packages are 1-click deployable integrations that +deliver instant value. They bundle pipelines, +enrichment, and configurations for common security +tools like Splunk, CrowdStrike, Elastic, SentinelOne, Palo Alto, and many more. -Browse our freely available [package library on -GitHub](https://github.com/tenzir/library). +Browse our freely available [package library on GitHub](https://github.com/tenzir/library). You can also use ai-workbench/use-agent-skills to generate custom packages with AI assistance. ## Core Integrations Core integrations are native connectors to the ecosystem, enabling communication over numerous protocols and APIs: -- **Cloud storage**: amazon/s3, - [GCS](/integrations/google/cloud-storage), - microsoft/azure-blob-storage -- **Message queues**: kafka, - amazon/sqs, amqp -- **Databases**: snowflake, - clickhouse -- **Network protocols**: tcp, udp, - http, syslog +- **Cloud storage**: amazon/s3, google/cloud-storage, microsoft/azure-blob-storage +- **Message queues**: kafka, amazon/sqs, amqp +- **Databases**: snowflake, clickhouse, mysql +- **Network protocols**: tcp, udp, http, syslog Under the hood, core integrations use a C++ plugin abstraction to provide an [operator](/reference/operators/), [function](/reference/functions/), or [context](/explanations/enrichment/) that you can use in TQL to directly interface with the respective resource, such as a TCP socket or cloud storage -bucket. We typically implement this functionality using the respective SDK, such as the -[AWS SDK](https://aws.amazon.com/sdk-for-cpp/), [Google Cloud +bucket. We typically implement this functionality using the respective SDK, such +as the [AWS SDK](https://aws.amazon.com/sdk-for-cpp/), [Google Cloud SDK](https://cloud.google.com/cpp), or [librdkafka](https://github.com/confluentinc/librdkafka), though some integrations require a custom implementation. -:::note[Dedicated Operators] -For some applications, we provide a **dedicated operator** that dramatically -simplifies the user experience. For example, -to_splunk and -from_opensearch offer a -streamlined interface compared to composing generic HTTP or protocol operators. +:::note[Dedicated operators] +For some applications, we provide a **dedicated operator** that dramatically simplifies the user experience. For example, to_splunk and from_opensearch offer a streamlined interface compared to composing generic HTTP or protocol operators. ::: diff --git a/src/content/docs/integrations/microsoft/windows-event-logs.mdx b/src/content/docs/integrations/microsoft/windows-event-logs.mdx index 4fedc2268..83101e2a8 100644 --- a/src/content/docs/integrations/microsoft/windows-event-logs.mdx +++ b/src/content/docs/integrations/microsoft/windows-event-logs.mdx @@ -221,10 +221,8 @@ configuration: Import the logs via TCP: ```tql -load_tcp "127.0.0.1:4000", - tls=true, - certfile="key_and_cert.pem", - keyfile="key_and_cert.pem" { +accept_tcp "127.0.0.1:4000", + tls={certfile: "key_and_cert.pem", keyfile: "key_and_cert.pem"} { read_json } import @@ -251,7 +249,7 @@ configuration to publish to the `nxlog` topic: ``` -Then use our [Kafka integration](/integrations/kafka) to read from the topic: +Then use kafka to read from the topic: ```tql from_kafka "nxlog" @@ -589,7 +587,7 @@ Accept the logs sent with the configuration above into Tenzir via tcp: ```tql -load_tcp "10.0.0.1:1514" { +accept_tcp "10.0.0.1:1514" { read_json } publish "wec" @@ -618,7 +616,7 @@ Security monitoring often focuses on specific event types. Filter for logon events (Event ID 4624) and failed logon attempts (Event ID 4625): ```tql -load_tcp "10.0.0.1:1514" { +accept_tcp "10.0.0.1:1514" { read_delimited "\n", include_separator=true } this = data.parse_winlog() @@ -631,7 +629,7 @@ The `EventData` section contains event-specific fields. For a successful logon event, extract the relevant information: ```tql -load_tcp "10.0.0.1:1514" { +accept_tcp "10.0.0.1:1514" { read_delimited "\n", include_separator=true } this = data.parse_winlog() diff --git a/src/content/docs/integrations/mysql.excalidraw b/src/content/docs/integrations/mysql.excalidraw new file mode 100644 index 000000000..2beae0e56 --- /dev/null +++ b/src/content/docs/integrations/mysql.excalidraw @@ -0,0 +1,947 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://app.excalidraw.com", + "elements": [ + { + "id": "8MN5I4hT3wVen_QUHAUnu", + "type": "rectangle", + "x": 1029.4206230693742, + "y": 462.25159339855855, + "width": 150.57937693062584, + "height": 143.23101048605469, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "Zy", + "roundness": { + "type": 3 + }, + "seed": 239993862, + "version": 526, + "versionNonce": 175282956, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "Q90BHXvcB9LwBRiLiSjA9" + } + ], + "updated": 1770653676903, + "link": null, + "locked": false + }, + { + "id": "Q90BHXvcB9LwBRiLiSjA9", + "type": "text", + "x": 1067.1423107412302, + "y": 467.25159339855855, + "width": 75.13600158691406, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "Zz", + "roundness": null, + "seed": 1597739846, + "version": 506, + "versionNonce": 1537026956, + "isDeleted": false, + "boundElements": [], + "updated": 1770653655303, + "link": null, + "locked": false, + "text": "Database", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "top", + "containerId": "8MN5I4hT3wVen_QUHAUnu", + "originalText": "Database", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "type": "rectangle", + "version": 481, + "versionNonce": 1842653364, + "index": "aU", + "isDeleted": false, + "id": "MdBLs5DrRJv7pPr1ECBdP", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dotted", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 860, + "y": 380, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "width": 480.0000000000001, + "height": 260, + "seed": 1012386970, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [ + { + "id": "anAP1e8hQ4MPrGd5mJIPw", + "type": "arrow" + } + ], + "updated": 1770653666972, + "link": null, + "locked": true + }, + { + "type": "arrow", + "version": 1418, + "versionNonce": 1665469708, + "isDeleted": true, + "id": "g_xmr_rAUoQx04wSxq2yQ", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 764.4167779398937, + "y": 525.2324875938532, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "width": 231.80542997696136, + "height": 0, + "seed": 61286746, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1770653628213, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "triangle", + "points": [ + [ + 0, + 0 + ], + [ + 231.80542997696136, + 0 + ] + ], + "index": "aV" + }, + { + "type": "line", + "version": 1403, + "versionNonce": 690428340, + "isDeleted": true, + "id": "K1eRli9Ss4OCth8Rxisx0", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 584.4167779398937, + "y": 485.22529758183066, + "strokeColor": "#000000", + "backgroundColor": "#e9ecef", + "width": 160, + "height": 80, + "seed": 174911002, + "groupIds": [ + "nS55pT0iEmYN13lrATL_r" + ], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1770653628213, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": null, + "points": [ + [ + 0, + 0 + ], + [ + 40, + 39.67197250080198 + ], + [ + -1.4210854715202004e-14, + 80 + ], + [ + 160, + 80 + ], + [ + 160, + 0 + ], + [ + 0, + 0 + ] + ], + "index": "aW", + "polygon": false + }, + { + "type": "image", + "version": 3810, + "versionNonce": 867666828, + "index": "aX", + "isDeleted": true, + "id": "Ob5zHrhyWqtZkihkGCreM", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 650.5291677541359, + "y": 499.0482924112763, + "strokeColor": "transparent", + "backgroundColor": "#ffc9c9", + "width": 61.54214099463984, + "height": 50.681763172056336, + "seed": 1704729306, + "groupIds": [ + "fCtmhUGKWexr3OXqoPqQN", + "nS55pT0iEmYN13lrATL_r" + ], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1770653628213, + "link": null, + "locked": false, + "status": "saved", + "fileId": "d1ee05fd41e4fad60a7372901fb1a4d206070a057f9b3a48768235e2c5775230a90d16ba6804ee1fbd79befd1ec8cbbf", + "scale": [ + 1, + 1 + ], + "crop": null + }, + { + "id": "tYpVzQAJgHPTVmA_sLOkS", + "type": "text", + "x": 886.072345512997, + "y": 537.0310114160328, + "width": 108.73332977294922, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "ae", + "roundness": null, + "seed": 1953084442, + "version": 241, + "versionNonce": 771867444, + "isDeleted": true, + "boundElements": [], + "updated": 1770653628213, + "link": null, + "locked": false, + "text": "clickhouse-cpp", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "clickhouse-cpp", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "NsxTI00fJ7Tk4JFzEOGL_", + "type": "image", + "x": 994.4490034404632, + "y": 385.84069071345743, + "width": 221.3677416168746, + "height": 67.5359211712499, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 4, + "strokeStyle": "solid", + "roughness": 0, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "ah", + "roundness": null, + "seed": 135755354, + "version": 509, + "versionNonce": 1203241908, + "isDeleted": true, + "boundElements": [], + "updated": 1770653606682, + "link": null, + "locked": false, + "status": "saved", + "fileId": "74235aaa80d5377e7af618a88eaf3165fae35889019cba2a4a62571f5ce6a7f9ad2989f927cd6093f58b899f2e11d669", + "scale": [ + 1, + 1 + ], + "crop": null + }, + { + "id": "grimbjvvQaXcX0fZ1QGkH", + "type": "rectangle", + "x": 1065.2209485133449, + "y": 508.5832717947994, + "width": 79.72278435602915, + "height": 56.34336263774654, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "3KLhhHW84bohbevIB1Hzc" + ], + "frameId": null, + "index": "amd", + "roundness": null, + "seed": 1503711494, + "version": 762, + "versionNonce": 932232454, + "isDeleted": false, + "boundElements": [], + "updated": 1741098374272, + "link": null, + "locked": false + }, + { + "id": "jDCSnbgRZUSNDrfjEd8Fc", + "type": "line", + "x": 1065.581736075349, + "y": 533.2097230148814, + "width": 78.67259902626967, + "height": 0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "3KLhhHW84bohbevIB1Hzc" + ], + "frameId": null, + "index": "aml", + "roundness": { + "type": 2 + }, + "seed": 816627782, + "version": 333, + "versionNonce": 566176838, + "isDeleted": false, + "boundElements": [], + "updated": 1741098374272, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 78.67259902626967, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null, + "polygon": false + }, + { + "id": "x1F877POVQOiJKca5RDRq", + "type": "line", + "x": 1081.316255880603, + "y": 533.2097230148814, + "width": 0.12437256409457782, + "height": 31.7169114176644, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "3KLhhHW84bohbevIB1Hzc" + ], + "frameId": null, + "index": "amt", + "roundness": { + "type": 2 + }, + "seed": 1996015494, + "version": 330, + "versionNonce": 1927132038, + "isDeleted": false, + "boundElements": [], + "updated": 1741098374272, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -0.12437256409457782, + 31.7169114176644 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null, + "polygon": false + }, + { + "id": "okS3KPOvVBZwY3ofsj0Uc", + "type": "line", + "x": 1097.3058925944924, + "y": 533.52609872761, + "width": 0.17604688976762406, + "height": 31.400535704935898, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "3KLhhHW84bohbevIB1Hzc" + ], + "frameId": null, + "index": "an", + "roundness": { + "type": 2 + }, + "seed": 2093082310, + "version": 341, + "versionNonce": 86594246, + "isDeleted": false, + "boundElements": [], + "updated": 1741098374272, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -0.17604688976762406, + 31.400535704935898 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null, + "polygon": false + }, + { + "id": "ZJrOj4tP-Rs6DlBgjxBwu", + "type": "line", + "x": 1113.4949144809275, + "y": 533.5112964970239, + "width": 0.42710638798645123, + "height": 31.415337935521947, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "3KLhhHW84bohbevIB1Hzc" + ], + "frameId": null, + "index": "anV", + "roundness": { + "type": 2 + }, + "seed": 44375558, + "version": 337, + "versionNonce": 1665075718, + "isDeleted": false, + "boundElements": [], + "updated": 1741098374272, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -0.42710638798645123, + 31.415337935521947 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null, + "polygon": false + }, + { + "id": "vYTHyZXj8YpcxC4gKL6cW", + "type": "line", + "x": 1128.9366108631093, + "y": 533.4277221087583, + "width": 0.06915961804813092, + "height": 31.498912323787575, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "3KLhhHW84bohbevIB1Hzc" + ], + "frameId": null, + "index": "ao", + "roundness": { + "type": 2 + }, + "seed": 765422918, + "version": 337, + "versionNonce": 1008458054, + "isDeleted": false, + "boundElements": [], + "updated": 1741098374272, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.06915961804813092, + 31.498912323787575 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null, + "polygon": false + }, + { + "id": "AL2_d8YDszRMsO8BuMuM5", + "type": "text", + "x": 1084.4878148765415, + "y": 510.21968757478163, + "width": 42.39994812011719, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "3KLhhHW84bohbevIB1Hzc" + ], + "frameId": null, + "index": "ap", + "roundness": null, + "seed": 2064503942, + "version": 222, + "versionNonce": 1461502086, + "isDeleted": false, + "boundElements": [], + "updated": 1741098374272, + "link": null, + "locked": false, + "text": "Table", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Table", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "sr_2aHlCdgC88c4nWzhF_", + "type": "image", + "x": 1040, + "y": 390, + "width": 140, + "height": 70, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "transparent", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aq", + "roundness": null, + "seed": 1850443956, + "version": 24, + "versionNonce": 500443700, + "isDeleted": false, + "boundElements": [], + "updated": 1770653635369, + "link": null, + "locked": false, + "status": "saved", + "fileId": "2e07883d1d9f64a8e2b93ea39a266e26f42f707d5579bf9a1348070b734c295d8c0dbca08e635fd04d72dbf979c6b05b", + "scale": [ + 1, + 1 + ], + "crop": null + }, + { + "type": "arrow", + "version": 542, + "versionNonce": 407558580, + "isDeleted": false, + "id": "anAP1e8hQ4MPrGd5mJIPw", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1224.5442617137614, + "y": 539.1944379296762, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "width": 205.2903283354567, + "height": 0, + "seed": 1781230860, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1770654115841, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "triangle", + "points": [ + [ + 0, + 0 + ], + [ + 205.2903283354567, + 0 + ] + ], + "index": "ar", + "moveMidPointsWithElement": false + }, + { + "type": "line", + "version": 1174, + "versionNonce": 1814268980, + "isDeleted": false, + "id": "kVLOfrXYUm_bhK00KWUdN", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1441.5929310543029, + "y": 500.72686469852636, + "strokeColor": "#000000", + "backgroundColor": "#e9ecef", + "width": 160, + "height": 80, + "seed": 1322380172, + "groupIds": [ + "FqtwZvej4SaxiNmq7rWnL" + ], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1770653674836, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": null, + "points": [ + [ + 0, + 0 + ], + [ + -1.4896611218806584, + 80 + ], + [ + 118.51033887811936, + 79.99999999999999 + ], + [ + 158.51033887811934, + 40.000000000000014 + ], + [ + 118.51033887811934, + -7.105427357601002e-15 + ], + [ + 0, + 0 + ] + ], + "index": "as", + "polygon": false + }, + { + "type": "image", + "version": 3781, + "versionNonce": 1842079156, + "index": "at", + "isDeleted": false, + "id": "qcU_lCbEEJfrEK51E_Uwk", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": 1480.3186257099073, + "y": 513.6204399910713, + "strokeColor": "transparent", + "backgroundColor": "#ffc9c9", + "width": 61.54214099463984, + "height": 50.681763172056336, + "seed": 119839244, + "groupIds": [ + "SH9Y4XZl0F0a3RqnagE-E", + "FqtwZvej4SaxiNmq7rWnL" + ], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1770653674836, + "link": null, + "locked": false, + "status": "saved", + "fileId": "d1ee05fd41e4fad60a7372901fb1a4d206070a057f9b3a48768235e2c5775230a90d16ba6804ee1fbd79befd1ec8cbbf", + "scale": [ + 1, + 1 + ], + "crop": null + }, + { + "id": "udf33bdWTCHF0aOMjJPnP", + "type": "arrow", + "x": 1199.9999997629093, + "y": 539.6403573296411, + "width": 20, + "height": 0.46871015741385236, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "au", + "roundness": null, + "seed": 1735537332, + "version": 414, + "versionNonce": 1476357516, + "isDeleted": false, + "boundElements": [], + "updated": 1770654102092, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -20, + -0.46871015741385236 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null, + "elbowed": false + }, + { + "id": "2YeGA667pdqopGszKYHHk", + "type": "ellipse", + "x": 1201.3800898153793, + "y": 531.4518874011399, + "width": 13.508293047904658, + "height": 13.508293047904658, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "av", + "roundness": { + "type": 2 + }, + "seed": 92388404, + "version": 423, + "versionNonce": 1397873676, + "isDeleted": false, + "boundElements": [], + "updated": 1770654102092, + "link": null, + "locked": false + }, + { + "id": "Y6dgqoPj8ThOFAFcVcc8l", + "type": "text", + "x": 1194.0364892454452, + "y": 549.4106528039305, + "width": 28.195493698120117, + "height": 21.829895655804975, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "aw", + "roundness": null, + "seed": 467932596, + "version": 369, + "versionNonce": 1155449268, + "isDeleted": false, + "boundElements": [], + "updated": 1770654104858, + "link": null, + "locked": false, + "text": "MySQL\nAPI", + "fontSize": 8.73195826232199, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "top", + "containerId": null, + "originalText": "MySQL\nAPI", + "autoResize": true, + "lineHeight": 1.25 + } + ], + "appState": { + "gridSize": 20, + "gridStep": 5, + "gridModeEnabled": false, + "viewBackgroundColor": "#ffffff", + "lockedMultiSelections": {} + }, + "files": { + "2e07883d1d9f64a8e2b93ea39a266e26f42f707d5579bf9a1348070b734c295d8c0dbca08e635fd04d72dbf979c6b05b": { + "mimeType": "image/svg+xml", + "id": "2e07883d1d9f64a8e2b93ea39a266e26f42f707d5579bf9a1348070b734c295d8c0dbca08e635fd04d72dbf979c6b05b", + "dataURL": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIxMjAiIGhlaWdodD0iNjAiIHZpZXdCb3g9IjAgMCA5LjI1MiA0LjYyNiI+PGcgdHJhbnNmb3JtPSJtYXRyaXgoLjAzNzM3NiAwIDAgLjAzNzM3NiAxLjA2OTk5NCAtMS4zMTkzMzkpIiBmaWxsLXJ1bGU9ImV2ZW5vZGQiPjxwYXRoIGQ9Ik04LjUwNCAxMjguMjE1aDUuOHYtMjIuOTc3bDkuMDU4IDIwLjAzM2MxLjAyNiAyLjQwOCAyLjUgMy4zIDUuMzU0IDMuM3M0LjI0LS44OTMgNS4zLTMuM2w5LjAxMy0yMC4wMzN2MjIuOTc3aDUuODQ1di0yMi45NzdjMC0yLjIzLS44OTMtMy4zMDMtMi43NjctMy44ODMtNC40MTctMS4zMzgtNy4zNjItLjE3OC04LjcgMi44MWwtOC44NzggMTkuODEtOC41NjctMTkuODFjLTEuMjk0LTIuOTg4LTQuMjg0LTQuMTQ4LTguNzQ1LTIuODEtMS44My41OC0yLjcyMiAxLjY1Mi0yLjcyMiAzLjg4M2wtLjAwMSAyMi45Nzd6bTQ1LjE5OC0xOC42OTRoNS44NDV2MTIuNjI3Yy0uMDQ0LjcxMy4yMjMgMi4zMiAzLjQgMi4zNjMgMS42NS4wNDUgMTIuNTgyIDAgMTIuNjcgMHYtMTUuMDhoNS44NDV2MjAuNjU4YzAgNS4wODYtNi4zIDYuMi05LjIzNiA2LjI0NmgtMTguMzh2LTMuODhoMTguNDI3YzMuNzQ4LS40MDIgMy4zMDItMi4yNzUgMy4zMDItMi45di0xLjUxOGgtMTIuMzZjLTUuNzU2LS4wNDUtOS40Ni0yLjU4OC05LjUwMy01LjQ4OHYtMTMuMDN6bTEyNS4zNzQtMTQuNjM1Yy0zLjU2OC0uMDktNi4zMzYuMjY4LTguNjU2IDEuMjUtLjY2OC4yNy0xLjc0LjI3LTEuODI4IDEuMTE2LjM1Ny4zNTUuNC45MzYuNzEzIDEuNDI4LjUzNS44OTMgMS40NzMgMi4wOTYgMi4zMiAyLjcyLjkzOC43MTUgMS44NzUgMS40MjggMi44NTUgMi4wNTMgMS43NCAxLjA3IDMuNzAzIDEuNjk1IDUuMzk4IDIuNzY2Ljk4Mi42MjUgMS45NjMgMS40MjggMi45NDUgMi4wOTguNS4zNTcuODAzLjkzOCAxLjQyOCAxLjE2di0uMTM1Yy0uMzEyLS40LS40MDItLjk4LS43MTMtMS40MjgtLjQ0Ny0uNDQ1LS44OTMtLjg0OC0xLjM0LTEuMjkzLTEuMjkzLTEuNzQtMi45LTMuMjU4LTQuNjQtNC41MDYtMS40MjgtLjk4Mi00LjU1LTIuMzItNS4xMy0zLjk3bC0uMDg4LS4wOWMuOTgtLjA5IDIuMTQtLjQ0NyAzLjA3OC0uNzE1IDEuNTE4LS40IDIuOS0uMzEyIDQuNDYtLjcxMy43MTUtLjE4IDEuNDI4LS40MDIgMi4xNDMtLjYyNXYtLjRjLS44MDMtLjgwMy0xLjM4My0xLjg3NC0yLjIzLTIuNjMyLTIuMjc1LTEuOTYzLTQuNzc1LTMuODgyLTcuMzYzLTUuNDg4LTEuMzgzLS44OTItMy4xNjgtMS40NzMtNC42NC0yLjIzLS41MzctLjI2OC0xLjQyOC0uNDAyLTEuNzQtLjg0OC0uODA1LS45OC0xLjI1LTIuMjc1LTEuODMtMy40MzYtMS4yOTMtMi40NTQtMi41NDMtNS4xNzUtMy42NTgtNy43NjMtLjgwMy0xLjc0LTEuMjk1LTMuNDgtMi4yNzUtNS4wODYtNC41OTYtNy41ODUtOS41OTQtMTIuMTgtMTcuMjY4LTE2LjY4Ny0xLjY1LS45MzctMy42MTMtMS4zNC01LjctMS44M2wtMy4zNDYtLjE4Yy0uNzE1LS4zMTItMS40MjgtMS4xNi0yLjA1My0xLjU2Mi0yLjU0My0xLjYwNi05LjEwMi01LjA4Ni0xMC45NzctLjQ5LTEuMjA1IDIuOSAxLjc4NSA1Ljc1NSAyLjggNy4yMjguNzYgMS4wMjYgMS43NCAyLjE4NiAyLjI3NyAzLjM0Ni4zLjc1OC40IDEuNTYyLjcxMyAyLjM2NS43MTMgMS45NjMgMS4zODMgNC4xNSAyLjMyIDUuOTguNS45MzcgMS4wMjUgMS45MiAxLjY1IDIuNzY3LjM1Ny40OS45ODIuNzE0IDEuMTE1IDEuNTE3LS42MjUuODkzLS42NjggMi4yMy0xLjAyNSAzLjM0Ny0xLjYwNyA1LjA0Mi0uOTgyIDExLjI4OCAxLjI5MyAxNC45OS43MTUgMS4xMTUgMi40IDMuNTcgNC42ODYgMi42MzIgMi4wMDgtLjgwMyAxLjU2LTMuMzQ2IDIuMTQtNS41NzcuMTM1LS41MzUuMDQ1LS44OTIuMzEyLTEuMjV2LjA5bDEuODMgMy43MDNjMS4zODMgMi4xODYgMy43OTMgNC40NjIgNS44IDUuOTggMS4wNy44MDMgMS45MTggMi4xODcgMy4yNTYgMi42Nzd2LS4xMzVoLS4wODhjLS4yNjgtLjQtLjY3LS41OC0xLjAyNy0uODkyLS44MDMtLjgwMy0xLjY5NS0xLjc4NS0yLjMyLTIuNjc3LTEuODczLTIuNDk4LTMuNTIzLTUuMjY1LTQuOTk2LTguMTItLjcxNS0xLjM4My0xLjM0LTIuOS0xLjkxOC00LjI4My0uMjctLjUzNi0uMjctMS4zNC0uNzE1LTEuNjA2LS42Ny45OC0xLjY1IDEuODMtMi4xNDMgMy4wMzQtLjg0OCAxLjkxOC0uOTM2IDQuMjgzLTEuMjQ4IDYuNzM3LS4xOC4wNDUtLjEgMC0uMTguMDktMS40MjYtLjM1Ni0xLjkxOC0xLjgzLTIuNDUzLTMuMDc4LTEuMzM4LTMuMTY4LTEuNTYyLTguMjU0LS40MDItMTEuOTEzLjMxMi0uOTM3IDEuNjUyLTMuODgyIDEuMTE3LTQuNzc0LS4yNy0uODQ4LTEuMTYtMS4zMzgtMS42NTItMi4wMDgtLjU4LS44NDgtMS4yMDMtMS45MTgtMS42MDUtMi44NTUtMS4wNy0yLjUtMS42MDUtNS4yNjUtMi43NjYtNy43NjQtLjUzNy0xLjE2LTEuNDczLTIuMzY1LTIuMjMyLTMuNDM1LS44NDgtMS4yMDUtMS43ODMtMi4wNTMtMi40NTMtMy40OC0uMjIzLS40OS0uNTM1LTEuMjk0LS4xNzgtMS44My4wODgtLjM1Ny4yNjgtLjQ5LjYyMy0uNTguNTgtLjQ5IDIuMjMyLjEzNCAyLjgxMi40IDEuNjUuNjcgMy4wMzMgMS4yOTQgNC40MTYgMi4yMy42MjUuNDQ2IDEuMjk1IDEuMjk0IDIuMDk4IDEuNTE4aC45MzhjMS40MjguMzEyIDMuMDMzLjA5IDQuMzcuNDkgMi4zNjUuNzYgNC41MDYgMS44NzQgNi40MjYgMy4wOCA1Ljg0NCAzLjcwMyAxMC42NjQgOC45NjggMTMuOTIgMTUuMjYuNTM1IDEuMDI2Ljc1OCAxLjk2MyAxLjI1IDMuMDM0LjkzOCAyLjE4NyAyLjA5OCA0LjQxNyAzLjAzMyA2LjU2LjkzOCAyLjA5NyAxLjgzIDQuMjQgMy4xNjggNS45OC42Ny45MzcgMy4zNDYgMS40MjcgNC41NSAxLjkxOC44OTMuNCAyLjI3NS43NiAzLjA4IDEuMjUgMS41MTYuOTM3IDMuMDMzIDIuMDA4IDQuNDYgMy4wMzQuNzEzLjUzNCAyLjk0NSAxLjY1IDMuMDc4IDIuNTR6bS00NS41LTM4Ljc3MmE3LjA5IDcuMDkgMCAwIDAtMS44MjguMjIzdi4wOWguMDg4Yy4zNTcuNzE0Ljk4MiAxLjIwNSAxLjQyOCAxLjgzbDEuMDI3IDIuMTQyLjA4OC0uMDljLjYyNS0uNDQ2LjkzOC0xLjE2LjkzOC0yLjIzLS4yNjgtLjMxMi0uMzEyLS42MjUtLjUzNS0uOTM3LS4yNjgtLjQ0Ni0uODQ4LS42Ny0xLjIwNi0xLjAyNnoiIGZpbGw9IiMwMDY3OGMiLz48cGF0aCBkPSJNODUuOTE2IDEyOC4yMTVoMTYuNzc2YzEuOTYzIDAgMy44MzgtLjQgNS4zNTQtMS4xMTUgMi41NDMtMS4xNiAzLjc0OC0yLjcyIDMuNzQ4LTQuNzczdi00LjI4M2MwLTEuNjUtMS4zODMtMy4yMTMtNC4xNDgtNC4yODMtMS40MjgtLjUzNS0zLjIxMy0uODQ4LTQuOTUzLS44NDhoLTcuMDVjLTIuMzY1IDAtMy40OC0uNzE1LTMuNzkzLTIuMjc1LS4wNDQtLjE3OC0uMDQ0LS4zNTctLjA0NC0uNTM1di0yLjYzM2MwLS4xMzUgMC0uMzEyLjA0NC0uNDkuMzEyLTEuMjA1LjkzNy0xLjUxOCAzLTEuNzRoMTcuMTc3di0zLjg4M2gtMTYuMzNjLTIuMzY1IDAtMy42MTQuMTM1LTQuNzMuNDkyLTMuNDM2IDEuMDctNC45NTMgMi43NjYtNC45NTMgNS43NTR2My4zOTNjMCAyLjYzIDIuOTQ1IDQuODYzIDcuOTQyIDUuMzk4LjUzNS4wNDUgMS4xMTUuMDQ1IDEuNjk1LjA0NWg2LjAyNGMuMjIzIDAgLjQ0NSAwIC42MjMuMDQ1IDEuODMuMTc4IDIuNjMzLjQ5IDMuMTY4IDEuMTU4LjM1Ny4zNTcuNDQ3LjY3LjQ0NyAxLjA3MnYzLjM5YzAgLjQtLjI2OC45MzgtLjgwMyAxLjM4M3MtMS4zODUuNzU4LTIuNS44MDNjLS4yMjMgMC0uMzU1LjA0NS0uNTguMDQ1SDg1LjkxNnptNjIuMTk1LTYuNzM2YzAgMy45NyAzIDYuMiA4Ljk3IDYuNjQ4LjU4LjA0NSAxLjExNS4wODggMS42OTUuMDg4aDE1LjE3di0zLjg4aC0xNS4zMDNjLTMuMzkzIDAtNC42ODYtLjg0OC00LjY4Ni0yLjl2LTIwLjA3OEgxNDguMXYyMC4xMjN6bS0zMi42MTUuMTc3di0xMy44M2MwLTMuNTI1IDIuNDk4LTUuNjY4IDcuMzYzLTYuMzM2LjUzNS0uMDQ1IDEuMDctLjA5IDEuNTYtLjA5aDExLjA2NGMuNTggMCAxLjA3Mi4wNDUgMS42NTIuMDkgNC44NjMuNjY4IDcuMzE2IDIuODEgNy4zMTYgNi4zMzZ2MTMuODNjMCAyLjg1NS0xLjAyNSA0LjM3My0zLjQzNiA1LjRsNS43MSA1LjE3NGgtNi43MzZsLTQuNjQtNC4xOTMtNC42ODYuMjY4aC02LjI0NmExMy42NiAxMy42NiAwIDAgMS0zLjM5MS0uNDQ1Yy0zLjctMS4wMjgtNS41My0yLjk5LTUuNTMtNi4yMDR6bTYuMjktLjMxYzAgLjE3OC4xLjM1NS4xMzUuNTguMzEyIDEuNjA1IDEuODI4IDIuNDk4IDQuMTQ4IDIuNDk4aDUuMjY2bC00LjgxOC00LjM3M2g2LjczNmw0LjIzOCAzLjgzOGMuODA1LS40NDcgMS4yOTUtMS4wNzIgMS40NzMtMS44NzUuMDQ1LS4xNzguMDQ1LS40LjA0NS0uNTh2LTEzLjI1MmMwLS4xNzggMC0uMzU1LS4wNDUtLjUzNS0uMzEyLTEuNTE2LTEuODI4LTIuMzYzLTQuMTA0LTIuMzYzaC04Ljc5Yy0yLjU4OCAwLTQuMjgzIDEuMTE1LTQuMjgzIDIuODk4eiIgZmlsbD0iI2NlOGIyYyIvPjwvZz48L3N2Zz4=", + "created": 1770653604842 + }, + "d1ee05fd41e4fad60a7372901fb1a4d206070a057f9b3a48768235e2c5775230a90d16ba6804ee1fbd79befd1ec8cbbf": { + "mimeType": "image/svg+xml", + "id": "d1ee05fd41e4fad60a7372901fb1a4d206070a057f9b3a48768235e2c5775230a90d16ba6804ee1fbd79befd1ec8cbbf", + "dataURL": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI4NSIgaGVpZ2h0PSI3MCIgdmlld0JveD0iMCAwIDg1IDcwIiBmaWxsPSJub25lIj4KPHBhdGggZD0iTTI5LjQ1IDQ0LjQyTDQyLjc3IDEyLjkzQzQyLjgyNjkgMTIuODA1OCA0Mi44NTc4IDEyLjY3MTMgNDIuODYwNyAxMi41MzQ4QzQyLjg2MzYgMTIuMzk4MiA0Mi44Mzg2IDEyLjI2MjUgNDIuNzg3IDEyLjEzNkM0Mi43MzU1IDEyLjAwOTUgNDIuNjU4NiAxMS44OTQ5IDQyLjU2MTEgMTEuNzk5M0M0Mi40NjM1IDExLjcwMzcgNDIuMzQ3NSAxMS42MjkgNDIuMjIgMTEuNThDNDIuMDkyOCAxMS41Mjg2IDQxLjk1NzEgMTEuNTAxNSA0MS44MiAxMS41SDkuMjc5OTlDOS4wNzYzMiAxMS40OTU4IDguODc2MjEgMTEuNTUzOCA4LjcwNjQzIDExLjY2NjRDOC41MzY2NiAxMS43NzkgOC40MDUzMiAxMS45NDA3IDguMzI5OTkgMTIuMTNMMy4xMDk5OSAyNC42M0MzLjA1IDI0Ljc1NTggMy4wMTY5NSAyNC44OTI3IDMuMDEyOTggMjUuMDMyQzMuMDA5MDEgMjUuMTcxMyAzLjAzNDE5IDI1LjMwOTkgMy4wODY5MiAyNS40Mzg5QzMuMTM5NjUgMjUuNTY3OSAzLjIxODc1IDI1LjY4NDQgMy4zMTkxNiAyNS43ODExQzMuNDE5NTcgMjUuODc3NyAzLjUzOTA3IDI1Ljk1MjMgMy42Njk5OSAyNkMzLjc5MzMxIDI2LjA1MjUgMy45MjU5NCAyNi4wNzk3IDQuMDU5OTkgMjYuMDhIMTkuODJDMjAuMDg1MiAyNi4wOCAyMC4zMzk2IDI2LjE4NTQgMjAuNTI3MSAyNi4zNzI5QzIwLjcxNDYgMjYuNTYwNCAyMC44MiAyNi44MTQ4IDIwLjgyIDI3LjA4QzIwLjgxODUgMjcuMjE3MiAyMC43OTE0IDI3LjM1MjggMjAuNzQgMjcuNDhMNy41OTk5OSA1OS4wN0M3LjU0MzMyIDU5LjE5NDggNy41MTI5MiA1OS4zMjk5IDcuNTEwNjUgNTkuNDY2OUM3LjUwODM5IDU5LjYwNCA3LjUzNDMyIDU5Ljc0IDcuNTg2ODMgNTkuODY2NkM3LjYzOTM1IDU5Ljk5MzIgNy43MTczMiA2MC4xMDc3IDcuODE1OTIgNjAuMjAyOUM3LjkxNDUyIDYwLjI5OCA4LjAzMTYzIDYwLjM3MiA4LjE1OTk5IDYwLjQyQzguMjgzMzEgNjAuNDcyNSA4LjQxNTk0IDYwLjQ5OTcgOC41NDk5OSA2MC41SDQzQzQzLjIxMzIgNjAuNTE0MSA0My40MjUzIDYwLjQ1OTYgNDMuNjA1MiA2MC4zNDQ1QzQzLjc4NTIgNjAuMjI5MyA0My45MjM1IDYwLjA1OTUgNDQgNTkuODZMNDkuMjIgNDcuMjhDNDkuMjc2NyA0Ny4xNTUyIDQ5LjMwNzEgNDcuMDIwMSA0OS4zMDkzIDQ2Ljg4MzFDNDkuMzExNiA0Ni43NDYgNDkuMjg1NyA0Ni42MSA0OS4yMzMxIDQ2LjQ4MzRDNDkuMTgwNiA0Ni4zNTY4IDQ5LjEwMjcgNDYuMjQyNCA0OS4wMDQxIDQ2LjE0NzJDNDguOTA1NSA0Ni4wNTIgNDguNzg4MyA0NS45NzggNDguNjYgNDUuOTNDNDguNTM2NyA0NS44Nzc1IDQ4LjQwNCA0NS44NTAzIDQ4LjI3IDQ1Ljg1SDMwLjRDMzAuMTM0OCA0NS44NSAyOS44ODA0IDQ1Ljc0NDYgMjkuNjkyOSA0NS41NTcxQzI5LjUwNTMgNDUuMzY5NiAyOS40IDQ1LjExNTIgMjkuNCA0NC44NUMyOS4zODggNDQuNzA0OCAyOS40MDUgNDQuNTU4NiAyOS40NSA0NC40MloiIGZpbGw9IiMxMjEyMTIiLz4KPHBhdGggZD0iTTM3LjE3IDQxLjU4SDUwLjg4QzUxLjA4MzcgNDEuNTg0MyA1MS4yODM4IDQxLjUyNjIgNTEuNDUzNSA0MS40MTM2QzUxLjYyMzMgNDEuMzAxIDUxLjc1NDcgNDEuMTM5MyA1MS44MyA0MC45NUw1Ny44MyAyNi42OUM1Ny45MDUzIDI2LjUwMDcgNTguMDM2NyAyNi4zMzkgNTguMjA2NCAyNi4yMjY0QzU4LjM3NjIgMjYuMTEzOCA1OC41NzYzIDI2LjA1NTcgNTguNzggMjYuMDZINzYuNDFDNzYuNjIyIDI2LjA3NDkgNzYuODMzMiAyNi4wMjE4IDc3LjAxMyAyNS45MDg1Qzc3LjE5MjggMjUuNzk1MyA3Ny4zMzE5IDI1LjYyNzYgNzcuNDEgMjUuNDNMODIuNjMgMTIuOTNDODIuNjg1MiAxMi44MDA3IDgyLjcxMjYgMTIuNjYxMyA4Mi43MTAzIDEyLjUyMDdDODIuNzA3OSAxMi4zODAxIDgyLjY3NiAxMi4yNDE2IDgyLjYxNjYgMTIuMTE0M0M4Mi41NTcxIDExLjk4NjkgODIuNDcxNSAxMS44NzM0IDgyLjM2NTIgMTEuNzgxNEM4Mi4yNTkgMTEuNjg5MyA4Mi4xMzQ1IDExLjYyMDcgODIgMTEuNThDODEuODc2NyAxMS41Mjc1IDgxLjc0NCAxMS41MDAzIDgxLjYxIDExLjVINDguODJDNDguNjA4IDExLjQ4NTEgNDguMzk2OCAxMS41MzgyIDQ4LjIxNyAxMS42NTE1QzQ4LjAzNzEgMTEuNzY0NyA0Ny44OTgxIDExLjkzMjQgNDcuODIgMTIuMTNMMzYuMTcgNDAuMTNDMzYuMTEzMSA0MC4yNTQyIDM2LjA4MjIgNDAuMzg4NyAzNi4wNzkzIDQwLjUyNTJDMzYuMDc2MyA0MC42NjE4IDM2LjEwMTQgNDAuNzk3NSAzNi4xNTI5IDQwLjkyNEMzNi4yMDQ1IDQxLjA1MDUgMzYuMjgxNCA0MS4xNjUxIDM2LjM3ODkgNDEuMjYwN0MzNi40NzY0IDQxLjM1NjMgMzYuNTkyNSA0MS40MzEgMzYuNzIgNDEuNDhDMzYuODYxMiA0MS41NDQ3IDM3LjAxNDYgNDEuNTc4OCAzNy4xNyA0MS41OFoiIGZpbGw9IiMxMjEyMTIiLz4KPC9zdmc+", + "created": 1684733732697 + } + } +} \ No newline at end of file diff --git a/src/content/docs/integrations/mysql.mdx b/src/content/docs/integrations/mysql.mdx new file mode 100644 index 000000000..2199bb694 --- /dev/null +++ b/src/content/docs/integrations/mysql.mdx @@ -0,0 +1,50 @@ +--- +title: MySQL +--- + +import Op from '@components/see-also/Op.astro'; +import { Steps } from '@astrojs/starlight/components'; + +[MySQL](https://www.mysql.com/) is an open-source relational database management +system widely used for web applications, data warehousing, and enterprise +applications. + +![MySQL Diagram](mysql.excalidraw) + +Tenzir connects to MySQL over the network using the MySQL wire protocol. Tenzir communicates with MySQL via the host and port you specify in the from_mysql operator. This means: + +- **Network**: Tenzir and MySQL can run on the same machine (using `localhost`) + or on different machines in the same network. You just need to make sure that + Tenzir can reach the MySQL server. +- **IPC**: There is no direct inter-process communication (IPC) mechanism; all + communication uses MySQL's network protocol. +- **Co-deployment**: For best performance and security, deploy Tenzir and MySQL + in the same trusted network or use TLS for encrypted connections. + +## Examples + +These examples assume that the MySQL server is running on the same host as +Tenzir. + +### List available tables + +```tql +from_mysql show="tables", host="localhost", database="mydb" +``` + +### Execute a custom SQL query + +```tql +from_mysql sql=r"SELECT id, name, email FROM users WHERE active = 1", + host="localhost", database="mydb" +``` + +### Stream new rows + +```tql +from_mysql table="events", live=true, host="localhost", database="mydb" +``` + +## See Also + +- from_mysql diff --git a/src/content/docs/integrations/syslog.mdx b/src/content/docs/integrations/syslog.mdx index 297eef267..31a7a47b2 100644 --- a/src/content/docs/integrations/syslog.mdx +++ b/src/content/docs/integrations/syslog.mdx @@ -30,17 +30,17 @@ this = data.parse_syslog() publish "syslog" ``` -To use TCP instead of UDP, use load_tcp with +To use TCP instead of UDP, use accept_tcp with read_syslog: ```tql -load_tcp "0.0.0.0:514" { +accept_tcp "0.0.0.0:514" { read_syslog } publish "syslog" ``` -The pipeline inside `load_tcp` executes _for each accepted connection_. +The pipeline inside accept_tcp executes _for each accepted connection_. ### Parsing CEF, LEEF, or JSON Payloads @@ -125,7 +125,7 @@ original syslog line alongside the parsed fields. Use the `raw_message` parameter to store the unparsed input: ```tql -load_tcp "0.0.0.0:514" { +accept_tcp "0.0.0.0:514" { read_syslog raw_message=raw } ``` diff --git a/src/content/docs/integrations/tcp.mdx b/src/content/docs/integrations/tcp.mdx index a744be8f5..f6e1eda45 100644 --- a/src/content/docs/integrations/tcp.mdx +++ b/src/content/docs/integrations/tcp.mdx @@ -4,52 +4,42 @@ title: TCP The [Transmission Control Protocol (TCP)](https://en.wikipedia.org/wiki/Transmission_Control_Protocol) provides a -bidirectional byte stream over IP. Tenzir supports reading from and writing to -TCP sockets in both server (listening) and client (connect) mode. +bidirectional byte stream over IP. Tenzir provides operators for both sides of +a TCP conversation: connecting to remote endpoints, accepting incoming +connections, and serving data to connected clients. ![TCP](tcp.svg) Use the IP address `0.0.0.0` to listen on all available network interfaces. -:::tip[URL Support] -The URL schemes `tcp://` and `tcps://` dispatch to -load_tcp and -save_tcp for seamless URL-style use via -from and to. +:::tip[URL support] +The URL schemes `tcp://` and `tcps://` dispatch to load_tcp and save_tcp for seamless URL-style use via from and to. ::: -## SSL/TLS - -To enable TLS, use `tls=true`. You can optionally pass a PEM-encoded certificate -and private key via the `certfile` and `keyfile` options. +## Connecting to remote endpoints -For testing purposes, you can quickly generate a self-signed certificate as -follows: +Use from_tcp to connect to a remote TCP endpoint as a client and read +data from it, or to_tcp to send data to a remote endpoint. Both +operators reconnect automatically with exponential backoff on connection failure. -```bash -openssl req -x509 -newkey rsa:2048 -keyout key_and_cert.pem -out key_and_cert.pem -days 365 -nodes -``` +## Accepting incoming connections -An easy way to test a TLS connection is to try connecting via OpenSSL: +Use accept_tcp to listen on a local endpoint and accept incoming TCP +connections. Each connection spawns a nested pipeline that processes the +incoming byte stream independently. -```bash -openssl s_client 127.0.0.1:443 -``` +## Serving data to clients -## Examples +Use serve_tcp to start a TCP server that broadcasts pipeline output to +all connected clients. A nested pipeline serializes events into bytes before +sending. -### Read data by connecting to a remote TCP server +See collecting/get-data-from-the-network for practical examples. -```tql -from "tcp://127.0.0.1:443", connect=true { - read_json -} -``` - -### Read data by listen on localhost with TLS enabled +## SSL/TLS -```tql -from "tcp://127.0.0.1:443", tls=true, certfile="cert.pem", keyfile="key.pem" { - read_json -} -``` +All TCP operators support TLS via the `tls` option. Pass an empty record +(`tls={}`) for defaults, or provide specific options like `certfile` and +`keyfile`. + like `certfile` and +`keyfile`. diff --git a/src/content/docs/reference/functions/secret.mdx b/src/content/docs/reference/functions/secret.mdx index 42786297b..4129f28bf 100644 --- a/src/content/docs/reference/functions/secret.mdx +++ b/src/content/docs/reference/functions/secret.mdx @@ -20,8 +20,7 @@ to. If the secret is not found in the node, a request is made to the Tenzir Platform. Should the platform also not be able to find the secret, an error is raised. -See the [explanation page for secrets](/explanations/secrets) for more -details. +See secrets for more details. ### `name: string` @@ -42,7 +41,7 @@ We do not recommend enabling this option. ### Using secrets in an operator ```tql -load_tcp "127.0.0.1:4000" { +accept_tcp "127.0.0.1:4000" { read_ndjson } to_splunk "https://localhost:8088", hec_token=secret("splunk-hec-token") diff --git a/src/content/docs/reference/operators.mdx b/src/content/docs/reference/operators.mdx index 468b963f1..c8c68ce4c 100644 --- a/src/content/docs/reference/operators.mdx +++ b/src/content/docs/reference/operators.mdx @@ -359,10 +359,18 @@ operators: description: 'Sends and receives HTTP/1.1 requests.' example: 'from_http "0.0.0.0:8080"' path: 'reference/operators/from_http' + - name: 'from_stdin' + description: 'Reads and parses events from standard input.' + example: 'from_stdin { read_json }' + path: 'reference/operators/from_stdin' - name: 'from_kafka' description: 'Receives events from an Apache Kafka topic.' example: 'from_kafka "logs"' path: 'reference/operators/from_kafka' + - name: 'from_mysql' + description: 'Reads events from a MySQL database.' + example: 'from_mysql table="users", host="db.example.com", database="mydb"' + path: 'reference/operators/from_mysql' - name: 'from_opensearch' description: 'Receives events via Opensearch Bulk API.' example: 'from_opensearch' @@ -599,6 +607,10 @@ operators: description: 'Parses an incoming Syslog stream into events.' example: 'read_syslog' path: 'reference/operators/read_syslog' + - name: 'read_tql' + description: 'Parses an incoming byte stream of TQL-formatted records into events.' + example: 'read_tql' + path: 'reference/operators/read_tql' - name: 'read_tsv' description: 'Read TSV (Tab-Separated Values) from a byte stream.' example: 'read_tsv auto_expand=true' @@ -1774,6 +1786,14 @@ read_syslog + + +```tql +read_tql +``` + + + ```tql @@ -2175,6 +2195,14 @@ from_http "0.0.0.0:8080" + + +```tql +from_stdin { read_json } +``` + + + ```tql @@ -2183,6 +2211,14 @@ from_kafka "logs" + + +```tql +from_mysql table="users", host="db.example.com", database="mydb" +``` + + + ```tql diff --git a/src/content/docs/reference/operators/accept_tcp.mdx b/src/content/docs/reference/operators/accept_tcp.mdx new file mode 100644 index 000000000..f26e5609d --- /dev/null +++ b/src/content/docs/reference/operators/accept_tcp.mdx @@ -0,0 +1,87 @@ +--- +title: accept_tcp +category: Inputs/Events +example: 'accept_tcp "0.0.0.0:8090" { read_json }' +--- + +import Op from '@components/see-also/Op.astro'; +import Integration from '@components/see-also/Integration.astro'; + +Listens for incoming TCP or TLS connections and receives events. + +```tql +accept_tcp endpoint:string, [max_connections=int, tls=record, { … }] +``` + +## Description + +Listens on the specified endpoint for incoming TCP connections. For each +accepted connection, the operator spawns the nested pipeline and feeds it the +bytes received from that connection. + +### `endpoint: string` + +The endpoint to listen on. Must be of the form `[tcp://]:`. Use +`0.0.0.0` as the host to accept connections on all interfaces. + +### `tls = record (optional)` + +import TLSOptions from '@partials/operators/TLSOptions.mdx'; + + + +### `max_connections = int (optional)` + +The maximum number of simultaneous incoming connections to accept. Additional +connections beyond this limit are rejected. + +Defaults to `128`. + +### `{ … } (optional)` + +The pipeline to run for each individual TCP connection. If none is specified, no +transformations are applied to the output streams. Unless you are sure that +there is at most one active connection at a time, it is recommended to specify a +pipeline that parses the individual connection streams into events, for instance +`{ read_json }`. Otherwise, the output can be interleaved. + +Inside the pipeline, the `$peer` variable is available as a record with the +following fields: + +| Field | Type | Description | +| :----- | :------ | :------------------------------------ | +| `ip` | `ip` | The IP address of the connected peer | +| `port` | `int64` | The port number of the connected peer | + +## Examples + +### Accept incoming JSON over TCP + +```tql +accept_tcp "0.0.0.0:8090" { + read_json +} +``` + +### Accept incoming Syslog over TCP + +```tql +accept_tcp "0.0.0.0:514" { + read_syslog +} +``` + +### Accept connections with TLS + +```tql +accept_tcp "0.0.0.0:4443", tls={certfile: "cert.pem", keyfile: "key.pem"} { + read_json +} +``` + +## See Also + +- from_tcp +- to_tcp +- serve_tcp +- tcp diff --git a/src/content/docs/reference/operators/batch.md b/src/content/docs/reference/operators/batch.md index 186b74e7a..d50c5e8e4 100644 --- a/src/content/docs/reference/operators/batch.md +++ b/src/content/docs/reference/operators/batch.md @@ -15,12 +15,16 @@ batch [limit:int, timeout=duration] The `batch` operator takes its input and rewrites it into batches of up to the desired size. -:::caution[Expert Operator] +:::caution[Advanced feature] The `batch` operator is a lower-level building block that lets users explicitly control batching, which otherwise is controlled automatically by Tenzir's underlying pipeline execution engine. Use with caution! ::: +Note that the operator maintains separate buffers for each distinct schema. Each +buffer has independent timeout tracking and fills until reaching the `limit`, at +which point it flushes immediately. + ### `limit: int (optional)` How many events to put into one batch at most. @@ -29,5 +33,7 @@ Defaults to `65536`. ### `timeout = duration (optional)` -Specifies a maximum latency for events passing through the batch operator. When -unspecified, an infinite duration is used. +Specifies a maximum latency for events passing through the batch operator. If no +new events arrive within the timeout period, any buffered events are flushed. + +Defaults to `1min`. diff --git a/src/content/docs/reference/operators/from.mdx b/src/content/docs/reference/operators/from.mdx index f077c073a..47639293e 100644 --- a/src/content/docs/reference/operators/from.mdx +++ b/src/content/docs/reference/operators/from.mdx @@ -97,8 +97,8 @@ just plain `json`. ### The pipeline argument & its relation to the loader -Some loaders, such as the load_tcp operator, accept a sub-pipeline -directly. If the selected loader accepts a sub-pipeline, the `from` operator +Some loaders accept a sub-pipeline directly. +If the selected loader accepts a sub-pipeline, the `from` operator will dispatch decompression and parsing into that sub-pipeline. If a an explicit pipeline argument is provided it is forwarded as-is. If the loader does not accept a sub-pipeline, the decompression and parsing steps are simply performed diff --git a/src/content/docs/reference/operators/from_mysql.mdx b/src/content/docs/reference/operators/from_mysql.mdx new file mode 100644 index 000000000..24b0c5a84 --- /dev/null +++ b/src/content/docs/reference/operators/from_mysql.mdx @@ -0,0 +1,240 @@ +--- +title: from_mysql +category: Inputs/Events +example: 'from_mysql table="users", host="db.example.com", database="mydb"' +--- + +import Op from '@components/see-also/Op.astro'; +import Integration from '@components/see-also/Integration.astro'; + +Reads events from a MySQL database. + +```tql +from_mysql [uri=string], [table=string], [sql=string], [show=string], + [live=bool], [tracking_column=string], + [host=string], [port=int], [user=string], [password=string], + [database=string], [tls=bool|record] +``` + +## Description + +The `from_mysql` operator reads data from a MySQL database. You can query data +using a table name, raw SQL, or retrieve database metadata. + +The operator supports three query modes that are mutually exclusive: + +1. **Table mode**: Read all rows from a table using the `table` parameter +2. **SQL mode**: Execute a custom SQL query using the `sql` parameter +3. **Show mode**: List tables or columns using the `show` parameter + +Internal metadata queries (such as resolving tracking columns for live mode) use +MySQL prepared statements to prevent SQL injection through user-provided table +and column names. + +### `uri = string (optional)` + +A MySQL connection URI in the format: + +``` +mysql://[user[:password]@]host[:port][/database] +``` + +When provided, the URI takes precedence over individual connection parameters. +Credentials in the URI can be overridden by explicit `user` and `password` +parameters. + +### `table = string (optional)` + +The name of the table to read from. This is mutually exclusive with `sql` and +`show`. + +### `sql = string (optional)` + +A raw SQL query to execute. This is mutually exclusive with `table` and `show`. + +Use raw strings for complex queries: + +```tql +from_mysql sql=r"SELECT id, name FROM users WHERE active = 1" +``` + +### `show = string (optional)` + +Retrieve database metadata. This is mutually exclusive with `table` and `sql`. + +Supported values: + +- `"tables"`: List all tables in the database +- `"columns"`: List all columns for the table specified in `table` + +### `live = bool (optional)` + +Enables continuous polling for new rows from a table. The operator tracks +progress using a watermark on an integer column and polls every second for rows +above the last-seen value. Mutually exclusive with `sql` and `show`. Requires +`table`. + +Defaults to `false`. + +### `tracking_column = string (optional)` + +The integer column to use for watermark tracking in live mode. The operator +queries for rows where this column exceeds the last-seen watermark. + +When omitted, the tracking column is auto-detected from the table's +auto-increment primary key. Requires `live=true`. + +### `host = string (optional)` + +The hostname or IP address of the MySQL server. + +Defaults to `"localhost"`. + +### `port = int (optional)` + +The port number of the MySQL server. + +Defaults to `3306`. + +### `user = string (optional)` + +The username for authentication. Supports the `secret` function for secure +credential management. + +Defaults to `"root"`. + +### `password = string (optional)` + +The password for authentication. Supports the `secret` function for secure +credential management. + +Defaults to `""`. + +### `database = string (optional)` + +The database to connect to. + +### `tls = bool|record (optional)` + +TLS configuration for the MySQL connection. Defaults to `false` (no TLS). + +Use `tls=true` to enable TLS with default settings and certificate verification, +or provide a record to customize specific options: + +```tql +{ + skip_peer_verification: bool, // skip certificate verification. + cacert: string, // CA bundle to verify peers. + certfile: string, // client certificate to present. + keyfile: string, // private key for the client certificate. +} +``` + +## Types + +The operator maps MySQL types to types as follows: + +| MySQL Type | Tenzir Type | Notes | +| :---------------------------- | :------------ | :--------------------- | +| `TINYINT(1)` | `bool` | Boolean representation | +| `TINYINT`, `SMALLINT`, `INT` | `int64` | | +| `BIGINT` | `int64` | | +| `BIGINT UNSIGNED` | `uint64` | | +| `FLOAT`, `DOUBLE` | `double` | | +| `DECIMAL`, `NUMERIC` | `double` | May lose precision | +| `DATE`, `DATETIME` | `time` | | +| `TIMESTAMP` | `time` | | +| `TIME` | `duration` | | +| `CHAR`, `VARCHAR`, `TEXT` | `string` | | +| `BINARY`, `VARBINARY`, `BLOB` | `blob` | | +| `JSON` | `string` | | +| `ENUM` | `enumeration` | | + +## Examples + +### Read all rows from a table + +```tql +from_mysql table="users", host="db.example.com", database="mydb" +``` + +### Use a connection URI + +```tql +from_mysql uri="mysql://admin:secret@db.example.com:3306/production", table="events" +``` + +### Execute a custom SQL query + +```tql +from_mysql sql=r"SELECT id, name, created_at FROM users WHERE active = 1 LIMIT 100", + host="localhost", database="app" +``` + +### Use secure credentials + +```tql +from_mysql table="orders", + host="db.example.com", + user=secret("mysql-user"), + password=secret("mysql-password"), + database="shop" +``` + +### List all tables in a database + +```tql +from_mysql show="tables", host="localhost", database="mydb" +``` + +### List columns for a specific table + +```tql +from_mysql show="columns", table="users", host="localhost", database="mydb" +``` + +### Enable TLS with defaults + +```tql +from_mysql table="events", + host="db.example.com", + database="production", + tls=true +``` + +### Connect with TLS but skip peer verification + +```tql +from_mysql table="events", + host="db.example.com", + database="production", + tls={skip_peer_verification: true} +``` + +### Connect with TLS using a CA certificate + +```tql +from_mysql table="events", + host="db.example.com", + database="production", + tls={cacert: "/path/to/ca.pem"} +``` + +### Stream new rows from a table + +```tql +from_mysql table="events", live=true, + host="db.example.com", database="mydb" +``` + +### Stream with an explicit tracking column + +```tql +from_mysql table="events", live=true, tracking_column="event_id", + host="db.example.com", database="mydb" +``` + +## See Also + +- to_clickhouse +- mysql diff --git a/src/content/docs/reference/operators/from_stdin.mdx b/src/content/docs/reference/operators/from_stdin.mdx new file mode 100644 index 000000000..b2c63ba46 --- /dev/null +++ b/src/content/docs/reference/operators/from_stdin.mdx @@ -0,0 +1,74 @@ +--- +title: from_stdin +category: Inputs/Events +example: 'from_stdin { read_json }' +--- + +import Op from '@components/see-also/Op.astro'; + +Reads and parses events from standard input. + +```tql +from_stdin { … } +``` + +## Description + +The `from_stdin` operator reads bytes from standard input and passes them +through the provided parsing pipeline to produce events. This is useful when +piping data into the `tenzir` executable as part of a shell script or command +chain. + +### `{ … }` + +The pipeline to parse the incoming bytes into events. The pipeline receives raw +bytes and must produce events. For example, `{ read_json }` parses the input as +JSON. + +## Examples + +### Parse JSON from standard input + +```sh +echo '{"foo": 42}' | tenzir +``` + +```tql +from_stdin { + read_json +} +``` + +```tql +{ + foo: 42, +} +``` + +### Parse CSV data piped from another command + +```sh +cat data.csv | tenzir -f pipeline.tql +``` + +```tql +from_stdin { + read_csv +} +``` + +### Parse Syslog messages + +```sh +tail -f /var/log/syslog | tenzir -f pipeline.tql +``` + +```tql +from_stdin { + read_syslog +} +``` + +## See Also + +- from_file diff --git a/src/content/docs/reference/operators/from_tcp.mdx b/src/content/docs/reference/operators/from_tcp.mdx new file mode 100644 index 000000000..e8988db20 --- /dev/null +++ b/src/content/docs/reference/operators/from_tcp.mdx @@ -0,0 +1,78 @@ +--- +title: from_tcp +category: Inputs/Events +example: 'from_tcp "example.org:4000" { read_json }' +--- + +import Op from '@components/see-also/Op.astro'; +import Integration from '@components/see-also/Integration.astro'; + +Connects to a remote TCP or TLS endpoint and receives events. + +```tql +from_tcp endpoint:string, [tls=record, { … }] +``` + +## Description + +Connects to the specified TCP endpoint as a client and reads bytes from the +connection, running them through an optional nested pipeline. + +If the connection fails, the operator retries with exponential backoff. + +### `endpoint: string` + +The remote endpoint to connect to. Must be of the form +`[tcp://]:`. + +### `tls = record (optional)` + +import TLSOptions from '@partials/operators/TLSOptions.mdx'; + + + +### `{ … } (optional)` + +The pipeline to run for the TCP connection. Use this to parse the incoming byte +stream into events, for instance `{ read_json }`. + +Inside the pipeline, the `$peer` variable is available as a record with the +following fields: + +| Field | Type | Description | +| :----- | :------ | :--------------------------------- | +| `ip` | `ip` | The IP address of the remote peer | +| `port` | `int64` | The port number of the remote peer | + +## Examples + +### Connect to a remote server and read JSON + +```tql +from_tcp "example.org:4000" { + read_json +} +``` + +### Connect with TLS + +```tql +from_tcp "example.org:4443", tls={} { + read_json +} +``` + +### Connect with TLS and a custom CA certificate + +```tql +from_tcp "example.org:4443", tls={cacert: "ca.pem"} { + read_json +} +``` + +## See Also + +- to_tcp +- accept_tcp +- serve_tcp +- tcp diff --git a/src/content/docs/reference/operators/load_balance.mdx b/src/content/docs/reference/operators/load_balance.mdx index 4e00e20db..b13403a4f 100644 --- a/src/content/docs/reference/operators/load_balance.mdx +++ b/src/content/docs/reference/operators/load_balance.mdx @@ -49,8 +49,7 @@ let $cfg = ["192.168.0.30:8080", "192.168.0.30:8081"] subscribe "input" load_balance $cfg { - write_json - save_tcp $cfg + to_tcp $cfg { write_json } } ``` diff --git a/src/content/docs/reference/operators/load_tcp.mdx b/src/content/docs/reference/operators/load_tcp.mdx index 3d2549b6d..c9cb0184a 100644 --- a/src/content/docs/reference/operators/load_tcp.mdx +++ b/src/content/docs/reference/operators/load_tcp.mdx @@ -4,6 +4,14 @@ category: Inputs/Bytes example: 'load_tcp "0.0.0.0:8090" { read_json }' --- +import Op from '@components/see-also/Op.astro'; +import Integration from '@components/see-also/Integration.astro'; + +:::caution[Deprecated] +Use from_tcp for client connections or accept_tcp for server +connections instead. +::: + Loads bytes from a TCP or TLS connection. ```tql @@ -138,5 +146,9 @@ openssl s_client -connect 127.0.0.1:4000 -cert client.pem -key client-key.pem -C ## See Also +- from_tcp +- to_tcp +- accept_tcp +- serve_tcp - save_tcp - tcp diff --git a/src/content/docs/reference/operators/measure.md b/src/content/docs/reference/operators/measure.md index 749ad383e..4f6ae2065 100644 --- a/src/content/docs/reference/operators/measure.md +++ b/src/content/docs/reference/operators/measure.md @@ -7,7 +7,7 @@ example: 'measure' Replaces the input with metrics describing the input. ```tql -measure [real_time=bool, cumulative=bool] +measure [cumulative=bool] ``` ## Description @@ -31,13 +31,6 @@ type tenzir.measure.bytes = record{ } ``` -### `real_time = bool (optional)` - -Whether to emit metrics immediately with every batch, rather than buffering -until the upstream operator stalls, i.e., is idle or waiting for further input. - -The is especially useful when `measure` should emit data without latency. - ### `cumulative = bool (optional)` Whether to emit running totals for the `events` and `bytes` fields rather than diff --git a/src/content/docs/reference/operators/parallel.mdx b/src/content/docs/reference/operators/parallel.mdx index 4d79ba19b..cb1588734 100644 --- a/src/content/docs/reference/operators/parallel.mdx +++ b/src/content/docs/reference/operators/parallel.mdx @@ -7,38 +7,60 @@ example: 'parallel 4 { parsed = data.parse_json() }' Runs a subpipeline across multiple parallel workers. ```tql -parallel jobs:int { … } +parallel [jobs:int] [, route_by=any] { … } ``` ## Description -The `parallel` operator distributes incoming events across multiple parallel -instances of a subpipeline. Each event is processed by exactly one worker. +The parallel operator distributes incoming events across multiple +parallel instances of a subpipeline. Each event is processed by exactly one +worker. Use this operator to parallelize CPU-intensive transformations or I/O-bound operations that would otherwise bottleneck on a single thread. +By default, events are distributed across workers using an adaptive round-robin +strategy that keeps worker loads balanced. Use `route_by` to instead route +events deterministically by a key, ensuring that all events with the same key +value go to the same worker. + This operator may reorder the event stream since workers process events concurrently. +When used as a source operator (without upstream input), `parallel` spawns +multiple independent instances of the subpipeline. This is useful for running +the same source pipeline with concurrent connections. + :::caution[Expert Operator] The `parallel` operator is a building block for performance optimization. Use it when you have identified a specific bottleneck that benefits from parallelization. Not all operations scale linearly with parallelism. ::: -### `jobs: int` +### `jobs: int (optional)` + +The number of parallel workers to spawn. Must be greater than zero. Defaults to +the number of available CPU cores. + +### `route_by = any (optional)` -The number of parallel workers to spawn. Must be greater than zero. +An expression evaluated per event to determine which worker processes it. Events +with the same `route_by` value are always sent to the same worker. This +guarantees that related events are grouped together, which is required for +stateful subpipelines like deduplicate or summarize. + +Cannot be used when `parallel` is used as a source operator. ### `{ … }` -The subpipeline to run in parallel. The subpipeline receives events as input and -may either: +The subpipeline to run in parallel. The subpipeline may either: - Produce events as output (transformation) - End with a sink (void output) +When `parallel` is used as a source operator, the subpipeline runs as an +independent source producing events or as a full pipeline ending with a sink. + The subpipeline must not produce bytes as output. ## Examples @@ -54,6 +76,18 @@ parallel 4 { } ``` +### Route events by source IP + +Ensure events from the same source IP are always handled by the same worker, +enabling per-source deduplication: + +```tql +subscribe "events" +parallel route_by=src_ip { + deduplicate src_ip, dst_ip, dst_port +} +``` + ### Make parallel HTTP requests Send events to Google SecOps with 4 concurrent connections: diff --git a/src/content/docs/reference/operators/read_delimited_regex.mdx b/src/content/docs/reference/operators/read_delimited_regex.mdx index b29d0e68d..02d14e7d9 100644 --- a/src/content/docs/reference/operators/read_delimited_regex.mdx +++ b/src/content/docs/reference/operators/read_delimited_regex.mdx @@ -49,8 +49,9 @@ default, the separator is excluded from the results. ### Split Syslog-like events without newline terminators from a TCP input ```tql -load_tcp "0.0.0.0:514" -read_delimited_regex "(?=<[0-9]+>)" +accept_tcp "0.0.0.0:514" { + read_delimited_regex "(?=<[0-9]+>)" +} this = data.parse_syslog() ``` diff --git a/src/content/docs/reference/operators/read_gelf.mdx b/src/content/docs/reference/operators/read_gelf.mdx index 71a8159a5..3663c9181 100644 --- a/src/content/docs/reference/operators/read_gelf.mdx +++ b/src/content/docs/reference/operators/read_gelf.mdx @@ -25,8 +25,9 @@ import ParsingOptions from '@partials/operators/ParsingOptions.mdx'; ### Read a GELF stream from a TCP socket ```tql -load_tcp "0.0.0.0:54321" -read_gelf +accept_tcp "0.0.0.0:54321" { + read_gelf +} ``` ## See Also diff --git a/src/content/docs/reference/operators/read_lines.mdx b/src/content/docs/reference/operators/read_lines.mdx index 6c5553ee4..d3e175e92 100644 --- a/src/content/docs/reference/operators/read_lines.mdx +++ b/src/content/docs/reference/operators/read_lines.mdx @@ -60,16 +60,18 @@ is_error = line.starts_with("error:") Consider using read_delimited_regex for regex-based splitting: ```tql -load_tcp "0.0.0.0:514" -read_delimited_regex "(?=<[0-9]+>)" +accept_tcp "0.0.0.0:514" { + read_delimited_regex "(?=<[0-9]+>)" +} this = line.parse_syslog() ``` ::: ```tql -load_tcp "0.0.0.0:514" -read_lines split_at_regex="(?=<[0-9]+>)" +accept_tcp "0.0.0.0:514" { + read_lines split_at_regex="(?=<[0-9]+>)" +} this = line.parse_syslog() ``` diff --git a/src/content/docs/reference/operators/read_syslog.mdx b/src/content/docs/reference/operators/read_syslog.mdx index 5b6e7e906..234c96efe 100644 --- a/src/content/docs/reference/operators/read_syslog.mdx +++ b/src/content/docs/reference/operators/read_syslog.mdx @@ -176,7 +176,7 @@ When receiving syslog over TCP from systems that use RFC 6587 octet counting, the parser auto-detects and strips the length prefix: ```tql title="Pipeline" -load_tcp "0.0.0.0:514" { +accept_tcp "0.0.0.0:514" { read_syslog } ``` diff --git a/src/content/docs/reference/operators/read_tql.mdx b/src/content/docs/reference/operators/read_tql.mdx new file mode 100644 index 000000000..54dd91378 --- /dev/null +++ b/src/content/docs/reference/operators/read_tql.mdx @@ -0,0 +1,88 @@ +--- +title: read_tql +category: Parsing +example: 'read_tql' +--- + +import Op from '@components/see-also/Op.astro'; + +Parses an incoming byte stream of TQL-formatted records into events. + +```tql +read_tql [schema=string, selector=string, schema_only=bool, + merge=bool, raw=bool, unflatten_separator=string] +``` + +## Description + +Parses an incoming byte stream of TQL-formatted records into events. Each +top-level record expression in the input becomes one event. + +The input format matches the output of write_tql. This makes +`read_tql` useful for round-tripping data through TQL notation, +reading TQL-formatted files, or processing data piped from other Tenzir +pipelines. + +The parser supports all TQL literal types, including `null`, `bool`, `int64`, +`double`, `string`, `duration`, `time`, `ip`, and `subnet`, as well as nested +records and lists. + +import ParsingOptions from '@partials/operators/ParsingOptions.mdx'; + + + +## Examples + +### Read TQL records from a file + +```tql title="events.tql" +{name: "Tenzir", version: 4} +{name: "Suricata", version: 7} +``` + +```tql title="Pipeline" +from_file "events.tql" { + read_tql +} +``` + +```tql title="Output" +{ + name: "Tenzir", + version: 4, +} +{ + name: "Suricata", + version: 7, +} +``` + +### Read records with native types + +TQL notation supports types that JSON cannot represent natively, such as +durations, timestamps, IP addresses, and subnets. + +```tql title="input.tql" +{dur: 5s, ts: 2024-01-01T00:00:00.000000, addr: 192.168.1.1, net: 10.0.0.0/8} +``` + +```tql title="Pipeline" +from_file "input.tql" { + read_tql +} +``` + +```tql title="Output" +{ + dur: 5s, + ts: 2024-01-01T00:00:00Z, + addr: 192.168.1.1, + net: 10.0.0.0/8, +} +``` + +## See Also + +- write_tql +- read_json +- read_ndjson diff --git a/src/content/docs/reference/operators/save_tcp.mdx b/src/content/docs/reference/operators/save_tcp.mdx index d9b4b7a54..1f85c1f7a 100644 --- a/src/content/docs/reference/operators/save_tcp.mdx +++ b/src/content/docs/reference/operators/save_tcp.mdx @@ -4,6 +4,14 @@ category: Outputs/Bytes example: 'save_tcp "0.0.0.0:8090", tls=true' --- +import Op from '@components/see-also/Op.astro'; +import Integration from '@components/see-also/Integration.astro'; + +:::caution[Deprecated] +Use to_tcp for client connections or serve_tcp for server +connections instead. +::: + Saves bytes to a TCP or TLS connection. ```tql @@ -33,7 +41,7 @@ of a numeric port. The amount of time to wait before attempting to reconnect in case a connection attempt fails and the error is deemed recoverable. Defaults to `30s`. -### `max_retry_count = int (optional) +### `max_retry_count = int (optional)` The number of retries to attempt in case of connection errors before transitioning into the error state. Defaults to `10`. @@ -62,5 +70,6 @@ save_tcp "127.0.0.1:4000", tls=true, skip_peer_verification=true ## See Also +- from_tcp - load_tcp - tcp diff --git a/src/content/docs/reference/operators/serve_tcp.mdx b/src/content/docs/reference/operators/serve_tcp.mdx new file mode 100644 index 000000000..f87c751b9 --- /dev/null +++ b/src/content/docs/reference/operators/serve_tcp.mdx @@ -0,0 +1,75 @@ +--- +title: serve_tcp +category: Outputs/Events +example: 'serve_tcp "0.0.0.0:8090" { write_json }' +--- + +import Op from '@components/see-also/Op.astro'; +import Integration from '@components/see-also/Integration.astro'; + +Listens for incoming TCP connections and sends events to all connected clients. + +```tql +serve_tcp endpoint:string, [max_connections=int, tls=record] { … } +``` + +## Description + +The `serve_tcp` operator starts a TCP server on the given endpoint and +broadcasts pipeline output to all connected clients. Input events are run +through a nested pipeline that must produce bytes (e.g., `{ write_json }`). + +Clients that connect receive the serialized output as a continuous byte stream. +Clients that disconnect or fail to keep up are dropped with a warning. + +### `endpoint: string` + +The endpoint to listen on. Must be of the form `[tcp://]:`. Use +`0.0.0.0` as the host to accept connections on all interfaces. + +import TLSOptions from '@partials/operators/TLSOptions.mdx'; + + + +### `max_connections = int (optional)` + +The maximum number of simultaneous client connections to accept. Additional +connections beyond this limit are rejected. + +Defaults to `128`. + +### `{ … }` + +The pipeline to serialize input events into bytes. Must produce bytes as output, +for instance `{ write_json }` or `{ write_csv }`. + +## Examples + +### Serve JSON to all connected TCP clients + +```tql +export +serve_tcp "0.0.0.0:8090" { write_json } +``` + +Connect with: + +```bash +nc localhost 8090 +``` + +### Serve with TLS + +```tql +export +serve_tcp "0.0.0.0:8443", tls={certfile: "cert.pem", keyfile: "key.pem"} { + write_json +} +``` + +## See Also + +- accept_tcp +- from_tcp +- to_tcp +- tcp diff --git a/src/content/docs/reference/operators/to_tcp.mdx b/src/content/docs/reference/operators/to_tcp.mdx new file mode 100644 index 000000000..8a54dccc5 --- /dev/null +++ b/src/content/docs/reference/operators/to_tcp.mdx @@ -0,0 +1,60 @@ +--- +title: to_tcp +category: Outputs/Events +example: 'to_tcp "collector.example.com:5044" { write_json }' +--- + +import Op from '@components/see-also/Op.astro'; +import Integration from '@components/see-also/Integration.astro'; + +Connects to a remote TCP or TLS endpoint and sends events. + +```tql +to_tcp endpoint:string, [tls=record] { … } +``` + +## Description + +Connects to the specified TCP endpoint as a client and writes serialized events +to the connection. Input events are run through a nested pipeline that must +produce bytes (e.g., `{ write_json }`). + +If the connection fails, the operator reconnects automatically with exponential +backoff. + +### `endpoint: string` + +The remote endpoint to connect to. Must be of the form +`[tcp://]:`. + +import TLSOptions from '@partials/operators/TLSOptions.mdx'; + + + +### `{ … }` + +The pipeline to serialize input events into bytes. Must produce bytes as output, +for instance `{ write_json }` or `{ write_csv }`. + +## Examples + +### Send JSON to a remote server + +```tql +export +to_tcp "collector.example.com:5044" { write_json } +``` + +### Send with TLS + +```tql +export +to_tcp "collector.example.com:5044", tls={} { write_json } +``` + +## See Also + +- from_tcp +- serve_tcp +- accept_tcp +- tcp diff --git a/src/content/docs/reference/operators/write_tql.mdx b/src/content/docs/reference/operators/write_tql.mdx index b3e3aa9fc..5c256f2e9 100644 --- a/src/content/docs/reference/operators/write_tql.mdx +++ b/src/content/docs/reference/operators/write_tql.mdx @@ -100,5 +100,6 @@ write_tql strip_null_fields=true ## See Also +- read_tql - write_json - map-data-to-ocsf diff --git a/src/sidebar.ts b/src/sidebar.ts index b431bab01..654c1391a 100644 --- a/src/sidebar.ts +++ b/src/sidebar.ts @@ -388,6 +388,7 @@ export const integrations = [ "integrations/clickhouse", "integrations/elasticsearch", "integrations/graylog", + "integrations/mysql", "integrations/opensearch", "integrations/snowflake", "integrations/splunk",