diff --git a/src/content/docs/integrations/databricks.excalidraw b/src/content/docs/integrations/databricks.excalidraw new file mode 100644 index 000000000..451dea32d --- /dev/null +++ b/src/content/docs/integrations/databricks.excalidraw @@ -0,0 +1,2589 @@ +{ + "type": "excalidraw", + "version": 2, + "source": "https://app.excalidraw.com", + "elements": [ + { + "type": "rectangle", + "version": 666, + "versionNonce": 1506420900, + "index": "b0b", + "isDeleted": false, + "id": "aF7hqZtriv1we_apUhc9U", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dotted", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -680, + "y": 40, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "width": 600, + "height": 320, + "seed": 1727694709, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [ + { + "id": "ZLumu12zXzC2JMph1hVT3", + "type": "arrow" + } + ], + "updated": 1769598809021, + "link": null, + "locked": true + }, + { + "id": "p_Y9Dnn7S95RX86JHF-oG", + "type": "text", + "x": -544, + "y": -15, + "width": 8, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0c", + "roundness": null, + "seed": 1300484309, + "version": 60, + "versionNonce": 273242555, + "isDeleted": true, + "boundElements": [], + "updated": 1768547311092, + "link": null, + "locked": false, + "text": "", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "top", + "containerId": "aF7hqZtriv1we_apUhc9U", + "originalText": "", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "type": "rectangle", + "version": 613, + "versionNonce": 519921947, + "index": "b0d", + "isDeleted": false, + "id": "zXW90Zh1-YGu4HvEIsdaH", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dotted", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -680, + "y": -340, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "width": 600, + "height": 240, + "seed": 1362009652, + "groupIds": [ + "_xLfPjnwDHIRJAGdqW_47" + ], + "frameId": null, + "roundness": null, + "boundElements": [ + { + "type": "text", + "id": "TIjzFnLkYmHWxTjlpwmEv" + } + ], + "updated": 1768548596277, + "link": null, + "locked": true + }, + { + "id": "TIjzFnLkYmHWxTjlpwmEv", + "type": "text", + "x": -484.2599334716797, + "y": -335, + "width": 208.51986694335938, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "_xLfPjnwDHIRJAGdqW_47" + ], + "frameId": null, + "index": "b0dV", + "roundness": null, + "seed": 1459365179, + "version": 60, + "versionNonce": 1075936059, + "isDeleted": false, + "boundElements": [], + "updated": 1768548594126, + "link": null, + "locked": true, + "text": "Cloud Object Storage", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "top", + "containerId": "zXW90Zh1-YGu4HvEIsdaH", + "originalText": "Cloud Object Storage", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "type": "arrow", + "version": 1414, + "versionNonce": 1004824740, + "isDeleted": false, + "id": "Ac625z5YAzg2h28mIT1Fn", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -800, + "y": -219.0919918634015, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "width": 140, + "height": 0.9080081365985109, + "seed": 1671739316, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1769599066255, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "triangle", + "points": [ + [ + 0, + 0 + ], + [ + 140, + -0.9080081365985109 + ] + ], + "index": "b0e", + "moveMidPointsWithElement": false + }, + { + "id": "rzA9-QnYt3LYkiDbzTe_6", + "type": "text", + "x": -733.2000000476837, + "y": -229.54599593170076, + "width": 6.400000095367432, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0eV", + "roundness": null, + "seed": 895095452, + "version": 10, + "versionNonce": 202880036, + "isDeleted": true, + "boundElements": null, + "updated": 1769599066255, + "link": null, + "locked": false, + "text": "", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Ac625z5YAzg2h28mIT1Fn", + "originalText": "", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "type": "line", + "version": 1398, + "versionNonce": 883905013, + "isDeleted": false, + "id": "qltilJSIgUx-XwxexdAWl", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -984.500563, + "y": -260.61539355704554, + "strokeColor": "#000000", + "backgroundColor": "#e9ecef", + "width": 160, + "height": 80, + "seed": 1125884212, + "groupIds": [ + "R6jbYK4rtCSIwXIJO9zSi" + ], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1768548563084, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": null, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": null, + "points": [ + [ + 0, + 0 + ], + [ + 40, + 39.67197250080198 + ], + [ + -1.4210854715202004e-14, + 80 + ], + [ + 160, + 80 + ], + [ + 160, + 0 + ], + [ + 0, + 0 + ] + ], + "index": "b0f", + "polygon": false + }, + { + "type": "image", + "version": 3805, + "versionNonce": 773764059, + "index": "b0g", + "isDeleted": false, + "id": "3uow-JkgHfYP2NzE-UR8M", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -918.3881731857578, + "y": -246.79239872759993, + "strokeColor": "transparent", + "backgroundColor": "#ffc9c9", + "width": 61.54214099463984, + "height": 50.681763172056336, + "seed": 922311348, + "groupIds": [ + "Klvdyi4GRdxM0sCnRZFlF", + "R6jbYK4rtCSIwXIJO9zSi" + ], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1768548563084, + "link": null, + "locked": false, + "status": "saved", + "fileId": "d1ee05fd41e4fad60a7372901fb1a4d206070a057f9b3a48768235e2c5775230a90d16ba6804ee1fbd79befd1ec8cbbf", + "scale": [ + 1, + 1 + ], + "crop": null + }, + { + "id": "xtMEKRQN-7euRF6dJ8eLu", + "type": "arrow", + "x": -768.5982785071208, + "y": 290.00000000000006, + "width": 40, + "height": 0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "dotted", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0l", + "roundness": null, + "seed": 1388718644, + "version": 324, + "versionNonce": 941268027, + "isDeleted": true, + "boundElements": [], + "updated": 1768547204087, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -40, + 0 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null, + "elbowed": false + }, + { + "id": "T_9ws1m_CvhhlAB4boM3C", + "type": "ellipse", + "x": -830.4801375071208, + "y": 280, + "width": 20, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0m", + "roundness": { + "type": 2 + }, + "seed": 1651164084, + "version": 257, + "versionNonce": 1637390581, + "isDeleted": true, + "boundElements": [], + "updated": 1768547204087, + "link": null, + "locked": false + }, + { + "id": "zrRy9UT_1Brrs_ZOjCDhE", + "type": "text", + "x": -860, + "y": 307.48177083333337, + "width": 74.06393432617188, + "height": 40, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b0n", + "roundness": null, + "seed": 356527412, + "version": 155, + "versionNonce": 345070811, + "isDeleted": true, + "boundElements": [], + "updated": 1768547204087, + "link": null, + "locked": false, + "text": "Ingestion\nAPI", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "top", + "containerId": null, + "originalText": "Ingestion\nAPI", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "86gwZ_4Y1ewG6bb08bT92", + "type": "image", + "x": -578.2412227019133, + "y": -151.40500433060186, + "width": 388.8544972425565, + "height": 59.254018627437176, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b12", + "roundness": null, + "seed": 1550673812, + "version": 258, + "versionNonce": 1458270069, + "isDeleted": true, + "boundElements": [], + "updated": 1768545846169, + "link": null, + "locked": false, + "status": "saved", + "fileId": "387392f862671157744bbf8612cbdb4b1f7c68b8a840dbfc45167f2fd2bb3205fb314ac35ec097400c95584427c26a0c", + "scale": [ + 1, + 1 + ], + "crop": null + }, + { + "id": "NUfwmJZPWd-QZnVlR-_iF", + "type": "image", + "x": -200, + "y": 60, + "width": 99.99999999999997, + "height": 42.056074766355124, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b13", + "roundness": null, + "seed": 1690538907, + "version": 285, + "versionNonce": 1009666811, + "isDeleted": false, + "boundElements": [], + "updated": 1768549085911, + "link": null, + "locked": false, + "status": "saved", + "fileId": "c0ecb32feabac485eae3b9fe6c16e1923d1ee523269ee5224755b817d6804f2a0347d2288a148059cbb6e94efd58e9e1", + "scale": [ + 1, + 1 + ], + "crop": null + }, + { + "id": "zyNffx-eGINHKFEuwwAMs", + "type": "rectangle", + "x": -640, + "y": -260, + "width": 240, + "height": 100, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b14", + "roundness": { + "type": 3 + }, + "seed": 316818779, + "version": 66, + "versionNonce": 359470581, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "h9pjqAl2LsarztnV30gTq" + } + ], + "updated": 1768548611603, + "link": null, + "locked": false + }, + { + "id": "h9pjqAl2LsarztnV30gTq", + "type": "text", + "x": -582.1199493408203, + "y": -255, + "width": 124.23989868164062, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b15", + "roundness": null, + "seed": 810835573, + "version": 68, + "versionNonce": 463175323, + "isDeleted": false, + "boundElements": [], + "updated": 1768547358595, + "link": null, + "locked": false, + "text": "Staging Area", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "top", + "containerId": "zyNffx-eGINHKFEuwwAMs", + "originalText": "Staging Area", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "YmqxjeoFL1rZnX__5UrTs", + "type": "rectangle", + "x": -540, + "y": 260, + "width": 320, + "height": 60, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b16", + "roundness": { + "type": 3 + }, + "seed": 1811000091, + "version": 229, + "versionNonce": 923658907, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "-G7LbTVzutP12jf4eTGFD" + }, + { + "id": "xep9DbAS59tnPidlmQo4p", + "type": "arrow" + } + ], + "updated": 1768548987878, + "link": null, + "locked": false + }, + { + "id": "-G7LbTVzutP12jf4eTGFD", + "type": "text", + "x": -446.96995544433594, + "y": 277.5, + "width": 133.93991088867188, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b17", + "roundness": null, + "seed": 476286907, + "version": 263, + "versionNonce": 1509838939, + "isDeleted": false, + "boundElements": [], + "updated": 1768548940522, + "link": null, + "locked": false, + "text": "Unity Catalog", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "YmqxjeoFL1rZnX__5UrTs", + "originalText": "Unity Catalog", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "1PYL1pBr3cMgsHWYCGSWG", + "type": "rectangle", + "x": -480, + "y": 80, + "width": 200, + "height": 120, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b18", + "roundness": { + "type": 3 + }, + "seed": 1810401467, + "version": 151, + "versionNonce": 1603244149, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "5_VT3Qiz3Q3BtN3ohEBCg" + }, + { + "id": "ZLumu12zXzC2JMph1hVT3", + "type": "arrow" + }, + { + "id": "xep9DbAS59tnPidlmQo4p", + "type": "arrow" + } + ], + "updated": 1768548983160, + "link": null, + "locked": false + }, + { + "id": "5_VT3Qiz3Q3BtN3ohEBCg", + "type": "text", + "x": -453.68994140625, + "y": 127.5, + "width": 147.3798828125, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b19", + "roundness": null, + "seed": 1626192219, + "version": 154, + "versionNonce": 501893397, + "isDeleted": false, + "boundElements": [], + "updated": 1768548320615, + "link": null, + "locked": false, + "text": "SQL Warehouse", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "1PYL1pBr3cMgsHWYCGSWG", + "originalText": "SQL Warehouse", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "dOzyIq3tw58FRrLxWAuUN", + "type": "rectangle", + "x": -360, + "y": -260, + "width": 240, + "height": 100, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1A", + "roundness": { + "type": 3 + }, + "seed": 1566763035, + "version": 86, + "versionNonce": 483757205, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "L-wPOTVCU-5sjjgJEVsH4" + } + ], + "updated": 1768547358595, + "link": null, + "locked": false + }, + { + "id": "L-wPOTVCU-5sjjgJEVsH4", + "type": "text", + "x": -317.3399429321289, + "y": -255, + "width": 154.6798858642578, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1B", + "roundness": null, + "seed": 648359099, + "version": 101, + "versionNonce": 498653205, + "isDeleted": false, + "boundElements": [], + "updated": 1768548663472, + "link": null, + "locked": false, + "text": "Managed Tables", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "top", + "containerId": "dOzyIq3tw58FRrLxWAuUN", + "originalText": "Managed Tables", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "twz4HOOjsw4EcZGbIUe5d", + "type": "image", + "x": -154.63491919220644, + "y": -255.4234601772514, + "width": 24.244358665363933, + "height": 23.55166270349639, + "angle": 0, + "strokeColor": "transparent", + "backgroundColor": "#b2f2bb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1C", + "roundness": null, + "seed": 1296309, + "version": 643, + "versionNonce": 72718331, + "isDeleted": false, + "boundElements": [], + "updated": 1768548656519, + "link": null, + "locked": false, + "status": "saved", + "fileId": "c8f308da71e262431b6d749e8dcb0a4bc680b0dc12a84b90698c759e23ab3e416b884f49808e05672379da58ccdbf95f", + "scale": [ + 1, + 1 + ], + "crop": null + }, + { + "id": "Gndvpfdw0SEqlb01_3i87", + "type": "rectangle", + "x": -620, + "y": -220, + "width": 80, + "height": 40, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1D", + "roundness": null, + "seed": 1415629781, + "version": 28, + "versionNonce": 1476473333, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "afqnCuMMtEjt_b4L6QaS5" + } + ], + "updated": 1768547358595, + "link": null, + "locked": false + }, + { + "id": "afqnCuMMtEjt_b4L6QaS5", + "type": "text", + "x": -610.5839767456055, + "y": -210, + "width": 61.16795349121094, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1E", + "roundness": null, + "seed": 521719739, + "version": 28, + "versionNonce": 156947419, + "isDeleted": false, + "boundElements": [], + "updated": 1768547358595, + "link": null, + "locked": false, + "text": "Parquet", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Gndvpfdw0SEqlb01_3i87", + "originalText": "Parquet", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "9yXjTkHkZ1UVFb0uZvHEX", + "type": "rectangle", + "x": -500, + "y": -220, + "width": 80, + "height": 40, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1F", + "roundness": null, + "seed": 1991938843, + "version": 34, + "versionNonce": 1834036053, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "L5MJC-WyupW8E0uerQVnp" + } + ], + "updated": 1768547358595, + "link": null, + "locked": false + }, + { + "id": "L5MJC-WyupW8E0uerQVnp", + "type": "text", + "x": -490.58397674560547, + "y": -210, + "width": 61.16795349121094, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1G", + "roundness": null, + "seed": 768571323, + "version": 34, + "versionNonce": 740925563, + "isDeleted": false, + "boundElements": [], + "updated": 1768547358595, + "link": null, + "locked": false, + "text": "Parquet", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "9yXjTkHkZ1UVFb0uZvHEX", + "originalText": "Parquet", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "qsILE9p9b2S9PcOpNtCeZ", + "type": "text", + "x": -526, + "y": -213, + "width": 13.151962280273438, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1H", + "roundness": null, + "seed": 628960117, + "version": 34, + "versionNonce": 1025177781, + "isDeleted": false, + "boundElements": [], + "updated": 1768547358595, + "link": null, + "locked": false, + "text": "...", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "top", + "containerId": null, + "originalText": "...", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "Ym0daMiHBnn_ECmxiwtXo", + "type": "rectangle", + "x": -340, + "y": -220, + "width": 80, + "height": 40, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1I", + "roundness": null, + "seed": 456086485, + "version": 46, + "versionNonce": 1890965787, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "j2ZDCFFwI5yPSBaQ1rgsa" + } + ], + "updated": 1768547358595, + "link": null, + "locked": false + }, + { + "id": "j2ZDCFFwI5yPSBaQ1rgsa", + "type": "text", + "x": -321.36798095703125, + "y": -210, + "width": 42.7359619140625, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1J", + "roundness": null, + "seed": 1310724405, + "version": 50, + "versionNonce": 17046037, + "isDeleted": false, + "boundElements": [], + "updated": 1768547358595, + "link": null, + "locked": false, + "text": "Delta", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Ym0daMiHBnn_ECmxiwtXo", + "originalText": "Delta", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "F-JLz8wQt2YMvsNaojfrR", + "type": "rectangle", + "x": -220, + "y": -220, + "width": 80, + "height": 40, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1K", + "roundness": null, + "seed": 1389503125, + "version": 52, + "versionNonce": 343014843, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "mCTtRy1iVfA20q8pBWiCA" + } + ], + "updated": 1768547358595, + "link": null, + "locked": false + }, + { + "id": "mCTtRy1iVfA20q8pBWiCA", + "type": "text", + "x": -201.36798095703125, + "y": -210, + "width": 42.7359619140625, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#b2f2bb", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1L", + "roundness": null, + "seed": 234555381, + "version": 56, + "versionNonce": 570560373, + "isDeleted": false, + "boundElements": [], + "updated": 1768547358595, + "link": null, + "locked": false, + "text": "Delta", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "F-JLz8wQt2YMvsNaojfrR", + "originalText": "Delta", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "FVtifdOWH147xViOdxqoZ", + "type": "text", + "x": -246, + "y": -213, + "width": 13.151962280273438, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1M", + "roundness": null, + "seed": 832586069, + "version": 51, + "versionNonce": 41979483, + "isDeleted": false, + "boundElements": [], + "updated": 1768547358595, + "link": null, + "locked": false, + "text": "...", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "top", + "containerId": null, + "originalText": "...", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "WLQ5jYbK25_SOhbhAE0DI", + "type": "arrow", + "x": -460, + "y": -60, + "width": 0, + "height": 60, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1N", + "roundness": { + "type": 2 + }, + "seed": 1196743643, + "version": 31, + "versionNonce": 2107707419, + "isDeleted": true, + "boundElements": [], + "updated": 1768547048587, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + -60 + ] + ], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "triangle", + "elbowed": false + }, + { + "id": "fV5NTnOGQvsx0gK6B3tOF", + "type": "arrow", + "x": -440, + "y": -140, + "width": 0, + "height": 0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1O", + "roundness": { + "type": 2 + }, + "seed": 135856757, + "version": 10, + "versionNonce": 84636, + "isDeleted": true, + "boundElements": [], + "updated": 1769598697053, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 0 + ] + ], + "startBinding": { + "elementId": "zyNffx-eGINHKFEuwwAMs", + "mode": "orbit", + "fixedPoint": [ + 0.8333333333333334, + 1 + ] + }, + "endBinding": null, + "startArrowhead": "bar", + "endArrowhead": "bar", + "elbowed": false + }, + { + "id": "4Lfb18tX_LWWLO8BWGkHx", + "type": "arrow", + "x": -434.26410145119746, + "y": -103.14145426495202, + "width": 0, + "height": 0, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1P", + "roundness": { + "type": 2 + }, + "seed": 1902069781, + "version": 18, + "versionNonce": 1937980708, + "isDeleted": true, + "boundElements": [], + "updated": 1769598697053, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 0 + ] + ], + "startBinding": { + "elementId": "zXW90Zh1-YGu4HvEIsdaH", + "mode": "inside", + "fixedPoint": [ + 0.3762264975813376, + 0.9869106072293666 + ] + }, + "endBinding": null, + "startArrowhead": "bar", + "endArrowhead": "bar", + "elbowed": false + }, + { + "id": "al8Y_2JUhZ8NxIldDw8iQ", + "type": "arrow", + "x": -122.75968847268678, + "y": -38.589845249999996, + "width": 106.72971494392326, + "height": 30, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1Q", + "roundness": null, + "seed": 225204533, + "version": 516, + "versionNonce": 2144412324, + "isDeleted": true, + "boundElements": [], + "updated": 1769599003843, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 30 + ], + [ + 106.72971494392326, + 30 + ], + [ + 106.72971494392326, + 0 + ] + ], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "triangle", + "elbowed": false + }, + { + "id": "KcXunXkVoI-yB3gbvt7RP", + "type": "text", + "x": -73.79483109609258, + "y": -18.589845249999996, + "width": 8.800000190734863, + "height": 20, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1QG", + "roundness": null, + "seed": 411144603, + "version": 24, + "versionNonce": 1802636316, + "isDeleted": true, + "boundElements": [], + "updated": 1769598941984, + "link": null, + "locked": false, + "text": "", + "fontSize": 16, + "fontFamily": 8, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "al8Y_2JUhZ8NxIldDw8iQ", + "originalText": "", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "kgj9tdJljJLtLh08ueHh7", + "type": "text", + "x": -384.1982926191713, + "y": -70, + "width": 8.79998779296875, + "height": 20, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1QV", + "roundness": null, + "seed": 760125301, + "version": 10, + "versionNonce": 243793564, + "isDeleted": true, + "boundElements": [], + "updated": 1769598884987, + "link": null, + "locked": false, + "text": "", + "fontSize": 16, + "fontFamily": 8, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "al8Y_2JUhZ8NxIldDw8iQ", + "originalText": "", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "gCc5YhTwzxP5yLWqgZUkn", + "type": "text", + "x": -384.1982926191713, + "y": -70, + "width": 8.79998779296875, + "height": 20, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1R", + "roundness": null, + "seed": 978752949, + "version": 22, + "versionNonce": 2096220452, + "isDeleted": true, + "boundElements": [], + "updated": 1769598884987, + "link": null, + "locked": false, + "text": "", + "fontSize": 16, + "fontFamily": 8, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "al8Y_2JUhZ8NxIldDw8iQ", + "originalText": "", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "nbTDwUcg76UrN39G5Ex2Z", + "type": "text", + "x": -420, + "y": -80, + "width": 99, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1S", + "roundness": null, + "seed": 942409755, + "version": 27, + "versionNonce": 2134634971, + "isDeleted": true, + "boundElements": [], + "updated": 1768547146457, + "link": null, + "locked": false, + "text": "COPY INTO", + "fontSize": 20, + "fontFamily": 8, + "textAlign": "center", + "verticalAlign": "top", + "containerId": null, + "originalText": "COPY INTO", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "A_Z-iJSIUbLG4snsPKsd3", + "type": "text", + "x": -775.7132034355964, + "y": -107.82411613907041, + "width": 11, + "height": 25, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1U", + "roundness": null, + "seed": 1448560635, + "version": 9, + "versionNonce": 1494926421, + "isDeleted": true, + "boundElements": [], + "updated": 1768547275443, + "link": null, + "locked": false, + "text": "", + "fontSize": 20, + "fontFamily": 8, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "QYtE_-j7OC-_cjbNjUdft", + "originalText": "", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "QYtE_-j7OC-_cjbNjUdft", + "type": "ellipse", + "x": -766.34375, + "y": -211.80859375, + "width": 24.76856502117039, + "height": 24.76856502117039, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "SX4Wxya0WIs6Trz4-FJrk" + ], + "frameId": null, + "index": "b1V", + "roundness": null, + "seed": 1767277813, + "version": 410, + "versionNonce": 2109674396, + "isDeleted": false, + "boundElements": [], + "updated": 1769599079429, + "link": null, + "locked": false + }, + { + "id": "blCL0Pe1RkzxVe4qmbgcI", + "type": "text", + "x": -758.6631023261149, + "y": -209.13738306220046, + "width": 9.51885115120231, + "height": 21.63375261636888, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "SX4Wxya0WIs6Trz4-FJrk" + ], + "frameId": null, + "index": "b1W", + "roundness": null, + "seed": 1241788283, + "version": 562, + "versionNonce": 1548894236, + "isDeleted": false, + "boundElements": [], + "updated": 1769599079429, + "link": null, + "locked": false, + "text": "1", + "fontSize": 17.30700209309511, + "fontFamily": 8, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "1", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "type": "arrow", + "version": 1504, + "versionNonce": 1664613788, + "isDeleted": false, + "id": "ZLumu12zXzC2JMph1hVT3", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "angle": 0, + "x": -898.8294329697467, + "y": -160, + "strokeColor": "#1e1e1e", + "backgroundColor": "#ffec99", + "width": 373.94008324626, + "height": 299.8036232225811, + "seed": 1668349243, + "groupIds": [], + "frameId": null, + "roundness": null, + "boundElements": [], + "updated": 1769598980873, + "link": null, + "locked": false, + "startBinding": null, + "endBinding": { + "elementId": "VjxYWU39jl3Pps2bF9Iat", + "mode": "orbit", + "fixedPoint": [ + 0.5001, + 0.5241084173264372 + ] + }, + "lastCommittedPoint": null, + "startArrowhead": null, + "endArrowhead": "triangle", + "points": [ + [ + 0, + 0 + ], + [ + 0, + 299.01835121677436 + ], + [ + 373.94008324626, + 299.8036232225811 + ] + ], + "index": "b1X" + }, + { + "id": "RLRBxqJyF_BhuuANVGG1k", + "type": "text", + "x": -902.0294330174304, + "y": 129.01835121677436, + "width": 6.400000095367432, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1XV", + "roundness": null, + "seed": 2019453988, + "version": 3, + "versionNonce": 924707356, + "isDeleted": true, + "boundElements": null, + "updated": 1769598980873, + "link": null, + "locked": false, + "text": "", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "ZLumu12zXzC2JMph1hVT3", + "originalText": "", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "f9z_oEnf_PFNEc6PhKmzj", + "type": "arrow", + "x": -439.46451920440984, + "y": 60, + "width": 0.9472677257754754, + "height": 200, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1Y", + "roundness": null, + "seed": 1498107509, + "version": 197, + "versionNonce": 1809901732, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "Q-W6U4k0qJCeSZM6hgtn4" + } + ], + "updated": 1769599024795, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.9472677257754754, + -200 + ] + ], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "triangle", + "elbowed": false, + "moveMidPointsWithElement": false + }, + { + "id": "Q-W6U4k0qJCeSZM6hgtn4", + "type": "text", + "x": -458.62288531100455, + "y": -50, + "width": 39.263999938964844, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1Z", + "roundness": null, + "seed": 82467707, + "version": 30, + "versionNonce": 1402165916, + "isDeleted": false, + "boundElements": [], + "updated": 1769599020152, + "link": null, + "locked": false, + "text": "Read", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "f9z_oEnf_PFNEc6PhKmzj", + "originalText": "Read", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "AeRizkYMzJ9IXe14sPTM4", + "type": "text", + "x": -395.3947460051711, + "y": 352.4962052959225, + "width": 8, + "height": 25, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1a", + "roundness": null, + "seed": 1852704949, + "version": 3, + "versionNonce": 317357659, + "isDeleted": true, + "boundElements": [], + "updated": 1768548151881, + "link": null, + "locked": false, + "text": "", + "fontSize": 20, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "top", + "containerId": null, + "originalText": "", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "iaYmkVAn0UyQaVNOogtFg", + "type": "ellipse", + "x": -877.0506550232177, + "y": 148.4819080974288, + "width": 24.76856502117039, + "height": 24.76856502117039, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "r9G_zWnt66rhYEhCaXADg" + ], + "frameId": null, + "index": "b1b", + "roundness": null, + "seed": 1818106549, + "version": 515, + "versionNonce": 1353821852, + "isDeleted": false, + "boundElements": [], + "updated": 1769599057570, + "link": null, + "locked": false + }, + { + "id": "9rUAGs1L8ZKc62o-T4Z2k", + "type": "text", + "x": -869.3700073493327, + "y": 151.15311878522834, + "width": 9.514999389648438, + "height": 21.633752616368888, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "r9G_zWnt66rhYEhCaXADg" + ], + "frameId": null, + "index": "b1c", + "roundness": null, + "seed": 1603468309, + "version": 669, + "versionNonce": 259725604, + "isDeleted": false, + "boundElements": [], + "updated": 1769599057570, + "link": null, + "locked": false, + "text": "2", + "fontSize": 17.30700209309511, + "fontFamily": 8, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "2", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "ywc6VcFKPgrfKT3h-yF9c", + "type": "ellipse", + "x": -490, + "y": -52.384283, + "width": 24.76856502117039, + "height": 24.76856502117039, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "v0tgFtGPZZF9jHsSM0IRh" + ], + "frameId": null, + "index": "b1d", + "roundness": null, + "seed": 441432821, + "version": 402, + "versionNonce": 1403704860, + "isDeleted": false, + "boundElements": [], + "updated": 1769598927495, + "link": null, + "locked": false + }, + { + "id": "vwJWiGrVOGZ1VK3cWv-dt", + "type": "text", + "x": -482.319352326115, + "y": -49.71307231220046, + "width": 9.514999389648438, + "height": 21.633752616368888, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "v0tgFtGPZZF9jHsSM0IRh" + ], + "frameId": null, + "index": "b1e", + "roundness": null, + "seed": 1457951829, + "version": 557, + "versionNonce": 883012252, + "isDeleted": false, + "boundElements": [], + "updated": 1769598927495, + "link": null, + "locked": false, + "text": "3", + "fontSize": 17.30700209309511, + "fontFamily": 8, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "3", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "f9WUcpVARVYZf5-JGnqP4", + "type": "ellipse", + "x": -220, + "y": -80, + "width": 24.76856502117039, + "height": 24.76856502117039, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "_0WzzEAeQqcozEal0JIFO" + ], + "frameId": null, + "index": "b1f", + "roundness": null, + "seed": 1686413717, + "version": 294, + "versionNonce": 339891771, + "isDeleted": true, + "boundElements": [], + "updated": 1768548437795, + "link": null, + "locked": false + }, + { + "id": "LmAV_KTFeqtfRf5MRT4EQ", + "type": "text", + "x": -212.319352326115, + "y": -77.32878931220046, + "width": 9.514999389648438, + "height": 21.633752616368888, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "_0WzzEAeQqcozEal0JIFO" + ], + "frameId": null, + "index": "b1g", + "roundness": null, + "seed": 1661314805, + "version": 449, + "versionNonce": 140607221, + "isDeleted": true, + "boundElements": [], + "updated": 1768548437795, + "link": null, + "locked": false, + "text": "3", + "fontSize": 17.30700209309511, + "fontFamily": 8, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "3", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "WzsJtAywgOlyugPaqguEE", + "type": "arrow", + "x": -480, + "y": 140.46871015741385, + "width": 20, + "height": 0.46871015741385236, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "solid", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1h", + "roundness": null, + "seed": 1108493333, + "version": 267, + "versionNonce": 987752853, + "isDeleted": false, + "boundElements": [], + "updated": 1768548836972, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + -20, + -0.46871015741385236 + ] + ], + "lastCommittedPoint": null, + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null, + "elbowed": false + }, + { + "id": "VjxYWU39jl3Pps2bF9Iat", + "type": "ellipse", + "x": -514.3917774559673, + "y": 132.7600444785906, + "width": 13.508293047904658, + "height": 13.508293047904658, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1i", + "roundness": { + "type": 2 + }, + "seed": 867891573, + "version": 318, + "versionNonce": 14482267, + "isDeleted": false, + "boundElements": [ + { + "id": "ZLumu12zXzC2JMph1hVT3", + "type": "arrow" + } + ], + "updated": 1768548900345, + "link": null, + "locked": false + }, + { + "id": "58wPrnVpgLXxMfD2bYpzt", + "type": "text", + "x": -516.1881349748548, + "y": 151.42057762278392, + "width": 16.87886359602679, + "height": 21.82989565580496, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1j", + "roundness": null, + "seed": 2095199803, + "version": 262, + "versionNonce": 1146847291, + "isDeleted": false, + "boundElements": [], + "updated": 1768548896155, + "link": null, + "locked": false, + "text": "SQL\nAPI", + "fontSize": 8.73195826232199, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "top", + "containerId": null, + "originalText": "SQL\nAPI", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "m_VHjmASd1jCWS0NqdEQ7", + "type": "line", + "x": -380, + "y": 200, + "width": 0, + "height": 40, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1k", + "roundness": null, + "seed": 1260707541, + "version": 6, + "versionNonce": 739703573, + "isDeleted": true, + "boundElements": [], + "updated": 1768548931679, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0, + 40 + ] + ], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": null, + "polygon": false + }, + { + "id": "xep9DbAS59tnPidlmQo4p", + "type": "arrow", + "x": -379.43731123110206, + "y": 210.50000000000003, + "width": 0.2064388257635983, + "height": 38.99999999999997, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1l", + "roundness": null, + "seed": 168874101, + "version": 541, + "versionNonce": 1221320507, + "isDeleted": false, + "boundElements": [], + "updated": 1768548987878, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.2064388257635983, + 38.99999999999997 + ] + ], + "startBinding": { + "elementId": "1PYL1pBr3cMgsHWYCGSWG", + "mode": "orbit", + "fixedPoint": [ + 0.5067742662468302, + 0.5067742662468303 + ] + }, + "endBinding": { + "elementId": "YmqxjeoFL1rZnX__5UrTs", + "mode": "orbit", + "fixedPoint": [ + 0.5030704100580451, + 0.49692958994195446 + ] + }, + "startArrowhead": null, + "endArrowhead": null, + "elbowed": false, + "moveMidPointsWithElement": false + }, + { + "id": "Fqz1831KkQx_HwtxAx4KP", + "type": "arrow", + "x": -319.92199111084005, + "y": 60, + "width": 0.9472677257754754, + "height": 200, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1m", + "roundness": null, + "seed": 677135524, + "version": 209, + "versionNonce": 610797468, + "isDeleted": false, + "boundElements": [ + { + "type": "text", + "id": "xtlrNMN48s1lZgDbt2AGg" + } + ], + "updated": 1769599027273, + "link": null, + "locked": false, + "points": [ + [ + 0, + 0 + ], + [ + 0.9472677257754754, + -200 + ] + ], + "startBinding": null, + "endBinding": null, + "startArrowhead": null, + "endArrowhead": "triangle", + "elbowed": false, + "moveMidPointsWithElement": false + }, + { + "id": "xtlrNMN48s1lZgDbt2AGg", + "type": "text", + "x": -339.7043577667512, + "y": -50, + "width": 40.512001037597656, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "dashed", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1n", + "roundness": null, + "seed": 1429475364, + "version": 41, + "versionNonce": 1488963876, + "isDeleted": false, + "boundElements": [], + "updated": 1769599026453, + "link": null, + "locked": false, + "text": "Write", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "center", + "verticalAlign": "middle", + "containerId": "Fqz1831KkQx_HwtxAx4KP", + "originalText": "Write", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "TOZNKic0i7Dg-saikdWNA", + "type": "ellipse", + "x": -372.38428300000004, + "y": -52.384283, + "width": 24.76856502117039, + "height": 24.76856502117039, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "transparent", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "BwwxMhWhs0d6-nJiqDm5i" + ], + "frameId": null, + "index": "b1o", + "roundness": null, + "seed": 1959393188, + "version": 412, + "versionNonce": 842834588, + "isDeleted": false, + "boundElements": [], + "updated": 1769599031248, + "link": null, + "locked": false + }, + { + "id": "_6vAtd-kvlcrPlRifAido", + "type": "text", + "x": -364.70363532611503, + "y": -49.71307231220046, + "width": 9.518851280212402, + "height": 21.633752616368888, + "angle": 0, + "strokeColor": "#868e96", + "backgroundColor": "#a5d8ff", + "fillStyle": "hachure", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [ + "BwwxMhWhs0d6-nJiqDm5i" + ], + "frameId": null, + "index": "b1p", + "roundness": null, + "seed": 1332962084, + "version": 569, + "versionNonce": 390685988, + "isDeleted": false, + "boundElements": [], + "updated": 1769599031248, + "link": null, + "locked": false, + "text": "4", + "fontSize": 17.30700209309511, + "fontFamily": 8, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "4", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "le_qUuEIxwDeyN5oUld4O", + "type": "text", + "x": -845.8090522358398, + "y": 149.8661905, + "width": 156.3360137939453, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1q", + "roundness": null, + "seed": 2048010396, + "version": 279, + "versionNonce": 205093660, + "isDeleted": false, + "boundElements": null, + "updated": 1769599057570, + "link": null, + "locked": false, + "text": "Trigger COPY INTO", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Trigger COPY INTO", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "bjcPmh-ugkupX1PKR5pX0", + "type": "text", + "x": -736.7911570187989, + "y": -209.42431075, + "width": 40.512001037597656, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1r", + "roundness": null, + "seed": 517312420, + "version": 140, + "versionNonce": 1221319836, + "isDeleted": false, + "boundElements": null, + "updated": 1769599079429, + "link": null, + "locked": false, + "text": "Write", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "Write", + "autoResize": true, + "lineHeight": 1.25 + }, + { + "id": "fcQN2z3ztLvlAreB3gDA0", + "type": "text", + "x": -798.7502815, + "y": -441.5, + "width": 6.400000095367432, + "height": 20, + "angle": 0, + "strokeColor": "#1e1e1e", + "backgroundColor": "#e9ecef", + "fillStyle": "cross-hatch", + "strokeWidth": 1, + "strokeStyle": "solid", + "roughness": 1, + "opacity": 100, + "groupIds": [], + "frameId": null, + "index": "b1s", + "roundness": null, + "seed": 1507503772, + "version": 3, + "versionNonce": 67182628, + "isDeleted": true, + "boundElements": null, + "updated": 1769599114590, + "link": null, + "locked": false, + "text": "", + "fontSize": 16, + "fontFamily": 5, + "textAlign": "left", + "verticalAlign": "top", + "containerId": null, + "originalText": "", + "autoResize": true, + "lineHeight": 1.25 + } + ], + "appState": { + "gridSize": 20, + "gridStep": 5, + "gridModeEnabled": false, + "viewBackgroundColor": "#ffffff", + "lockedMultiSelections": { + "_xLfPjnwDHIRJAGdqW_47": true + } + }, + "files": { + "d1ee05fd41e4fad60a7372901fb1a4d206070a057f9b3a48768235e2c5775230a90d16ba6804ee1fbd79befd1ec8cbbf": { + "mimeType": "image/svg+xml", + "id": "d1ee05fd41e4fad60a7372901fb1a4d206070a057f9b3a48768235e2c5775230a90d16ba6804ee1fbd79befd1ec8cbbf", + "dataURL": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSI4NSIgaGVpZ2h0PSI3MCIgdmlld0JveD0iMCAwIDg1IDcwIiBmaWxsPSJub25lIj4KPHBhdGggZD0iTTI5LjQ1IDQ0LjQyTDQyLjc3IDEyLjkzQzQyLjgyNjkgMTIuODA1OCA0Mi44NTc4IDEyLjY3MTMgNDIuODYwNyAxMi41MzQ4QzQyLjg2MzYgMTIuMzk4MiA0Mi44Mzg2IDEyLjI2MjUgNDIuNzg3IDEyLjEzNkM0Mi43MzU1IDEyLjAwOTUgNDIuNjU4NiAxMS44OTQ5IDQyLjU2MTEgMTEuNzk5M0M0Mi40NjM1IDExLjcwMzcgNDIuMzQ3NSAxMS42MjkgNDIuMjIgMTEuNThDNDIuMDkyOCAxMS41Mjg2IDQxLjk1NzEgMTEuNTAxNSA0MS44MiAxMS41SDkuMjc5OTlDOS4wNzYzMiAxMS40OTU4IDguODc2MjEgMTEuNTUzOCA4LjcwNjQzIDExLjY2NjRDOC41MzY2NiAxMS43NzkgOC40MDUzMiAxMS45NDA3IDguMzI5OTkgMTIuMTNMMy4xMDk5OSAyNC42M0MzLjA1IDI0Ljc1NTggMy4wMTY5NSAyNC44OTI3IDMuMDEyOTggMjUuMDMyQzMuMDA5MDEgMjUuMTcxMyAzLjAzNDE5IDI1LjMwOTkgMy4wODY5MiAyNS40Mzg5QzMuMTM5NjUgMjUuNTY3OSAzLjIxODc1IDI1LjY4NDQgMy4zMTkxNiAyNS43ODExQzMuNDE5NTcgMjUuODc3NyAzLjUzOTA3IDI1Ljk1MjMgMy42Njk5OSAyNkMzLjc5MzMxIDI2LjA1MjUgMy45MjU5NCAyNi4wNzk3IDQuMDU5OTkgMjYuMDhIMTkuODJDMjAuMDg1MiAyNi4wOCAyMC4zMzk2IDI2LjE4NTQgMjAuNTI3MSAyNi4zNzI5QzIwLjcxNDYgMjYuNTYwNCAyMC44MiAyNi44MTQ4IDIwLjgyIDI3LjA4QzIwLjgxODUgMjcuMjE3MiAyMC43OTE0IDI3LjM1MjggMjAuNzQgMjcuNDhMNy41OTk5OSA1OS4wN0M3LjU0MzMyIDU5LjE5NDggNy41MTI5MiA1OS4zMjk5IDcuNTEwNjUgNTkuNDY2OUM3LjUwODM5IDU5LjYwNCA3LjUzNDMyIDU5Ljc0IDcuNTg2ODMgNTkuODY2NkM3LjYzOTM1IDU5Ljk5MzIgNy43MTczMiA2MC4xMDc3IDcuODE1OTIgNjAuMjAyOUM3LjkxNDUyIDYwLjI5OCA4LjAzMTYzIDYwLjM3MiA4LjE1OTk5IDYwLjQyQzguMjgzMzEgNjAuNDcyNSA4LjQxNTk0IDYwLjQ5OTcgOC41NDk5OSA2MC41SDQzQzQzLjIxMzIgNjAuNTE0MSA0My40MjUzIDYwLjQ1OTYgNDMuNjA1MiA2MC4zNDQ1QzQzLjc4NTIgNjAuMjI5MyA0My45MjM1IDYwLjA1OTUgNDQgNTkuODZMNDkuMjIgNDcuMjhDNDkuMjc2NyA0Ny4xNTUyIDQ5LjMwNzEgNDcuMDIwMSA0OS4zMDkzIDQ2Ljg4MzFDNDkuMzExNiA0Ni43NDYgNDkuMjg1NyA0Ni42MSA0OS4yMzMxIDQ2LjQ4MzRDNDkuMTgwNiA0Ni4zNTY4IDQ5LjEwMjcgNDYuMjQyNCA0OS4wMDQxIDQ2LjE0NzJDNDguOTA1NSA0Ni4wNTIgNDguNzg4MyA0NS45NzggNDguNjYgNDUuOTNDNDguNTM2NyA0NS44Nzc1IDQ4LjQwNCA0NS44NTAzIDQ4LjI3IDQ1Ljg1SDMwLjRDMzAuMTM0OCA0NS44NSAyOS44ODA0IDQ1Ljc0NDYgMjkuNjkyOSA0NS41NTcxQzI5LjUwNTMgNDUuMzY5NiAyOS40IDQ1LjExNTIgMjkuNCA0NC44NUMyOS4zODggNDQuNzA0OCAyOS40MDUgNDQuNTU4NiAyOS40NSA0NC40MloiIGZpbGw9IiMxMjEyMTIiLz4KPHBhdGggZD0iTTM3LjE3IDQxLjU4SDUwLjg4QzUxLjA4MzcgNDEuNTg0MyA1MS4yODM4IDQxLjUyNjIgNTEuNDUzNSA0MS40MTM2QzUxLjYyMzMgNDEuMzAxIDUxLjc1NDcgNDEuMTM5MyA1MS44MyA0MC45NUw1Ny44MyAyNi42OUM1Ny45MDUzIDI2LjUwMDcgNTguMDM2NyAyNi4zMzkgNTguMjA2NCAyNi4yMjY0QzU4LjM3NjIgMjYuMTEzOCA1OC41NzYzIDI2LjA1NTcgNTguNzggMjYuMDZINzYuNDFDNzYuNjIyIDI2LjA3NDkgNzYuODMzMiAyNi4wMjE4IDc3LjAxMyAyNS45MDg1Qzc3LjE5MjggMjUuNzk1MyA3Ny4zMzE5IDI1LjYyNzYgNzcuNDEgMjUuNDNMODIuNjMgMTIuOTNDODIuNjg1MiAxMi44MDA3IDgyLjcxMjYgMTIuNjYxMyA4Mi43MTAzIDEyLjUyMDdDODIuNzA3OSAxMi4zODAxIDgyLjY3NiAxMi4yNDE2IDgyLjYxNjYgMTIuMTE0M0M4Mi41NTcxIDExLjk4NjkgODIuNDcxNSAxMS44NzM0IDgyLjM2NTIgMTEuNzgxNEM4Mi4yNTkgMTEuNjg5MyA4Mi4xMzQ1IDExLjYyMDcgODIgMTEuNThDODEuODc2NyAxMS41Mjc1IDgxLjc0NCAxMS41MDAzIDgxLjYxIDExLjVINDguODJDNDguNjA4IDExLjQ4NTEgNDguMzk2OCAxMS41MzgyIDQ4LjIxNyAxMS42NTE1QzQ4LjAzNzEgMTEuNzY0NyA0Ny44OTgxIDExLjkzMjQgNDcuODIgMTIuMTNMMzYuMTcgNDAuMTNDMzYuMTEzMSA0MC4yNTQyIDM2LjA4MjIgNDAuMzg4NyAzNi4wNzkzIDQwLjUyNTJDMzYuMDc2MyA0MC42NjE4IDM2LjEwMTQgNDAuNzk3NSAzNi4xNTI5IDQwLjkyNEMzNi4yMDQ1IDQxLjA1MDUgMzYuMjgxNCA0MS4xNjUxIDM2LjM3ODkgNDEuMjYwN0MzNi40NzY0IDQxLjM1NjMgMzYuNTkyNSA0MS40MzEgMzYuNzIgNDEuNDhDMzYuODYxMiA0MS41NDQ3IDM3LjAxNDYgNDEuNTc4OCAzNy4xNyA0MS41OFoiIGZpbGw9IiMxMjEyMTIiLz4KPC9zdmc+", + "created": 1684733732697 + }, + "c0ecb32feabac485eae3b9fe6c16e1923d1ee523269ee5224755b817d6804f2a0347d2288a148059cbb6e94efd58e9e1": { + "mimeType": "image/svg+xml", + "id": "c0ecb32feabac485eae3b9fe6c16e1923d1ee523269ee5224755b817d6804f2a0347d2288a148059cbb6e94efd58e9e1", + "dataURL": "data:image/svg+xml;base64,PHN2ZyB4bWxuczppbmtzY2FwZT0iaHR0cDovL3d3dy5pbmtzY2FwZS5vcmcvbmFtZXNwYWNlcy9pbmtzY2FwZSIgeG1sbnM6c29kaXBvZGk9Imh0dHA6Ly9zb2RpcG9kaS5zb3VyY2Vmb3JnZS5uZXQvRFREL3NvZGlwb2RpLTAuZHRkIiB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHhtbG5zOnN2Zz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIxMDYuODQ5IiBoZWlnaHQ9IjQ0LjYxNzM0OCIgdmlld0JveD0iMCAwIDEwNi44NDkgNDQuNjE3MzQ4IiBmaWxsPSJub25lIiB2ZXJzaW9uPSIxLjEiIGlkPSJzdmczIiBzb2RpcG9kaTpkb2NuYW1lPSJkYXRhYnJpY2tzLWxvZ28uc3ZnIiBpbmtzY2FwZTp2ZXJzaW9uPSIxLjQuMiAoZWJmMGU5NDBkMCwgMjAyNS0wNS0wOCkiPgogIDxkZWZzIGlkPSJkZWZzMyIvPgogIDxzb2RpcG9kaTpuYW1lZHZpZXcgaWQ9Im5hbWVkdmlldzMiIHBhZ2Vjb2xvcj0iI2ZmZmZmZiIgYm9yZGVyY29sb3I9IiMwMDAwMDAiIGJvcmRlcm9wYWNpdHk9IjAuMjUiIGlua3NjYXBlOnNob3dwYWdlc2hhZG93PSIyIiBpbmtzY2FwZTpwYWdlb3BhY2l0eT0iMC4wIiBpbmtzY2FwZTpwYWdlY2hlY2tlcmJvYXJkPSIwIiBpbmtzY2FwZTpkZXNrY29sb3I9IiNkMWQxZDEiIGlua3NjYXBlOnpvb209IjMuOTU4ODkxMiIgaW5rc2NhcGU6Y3g9IjU2LjQ1NTIwMSIgaW5rc2NhcGU6Y3k9IjMzLjIxNjM3MSIgaW5rc2NhcGU6d2luZG93LXdpZHRoPSIxNzI4IiBpbmtzY2FwZTp3aW5kb3ctaGVpZ2h0PSIxMDExIiBpbmtzY2FwZTp3aW5kb3cteD0iMCIgaW5rc2NhcGU6d2luZG93LXk9IjAiIGlua3NjYXBlOndpbmRvdy1tYXhpbWl6ZWQ9IjEiIGlua3NjYXBlOmN1cnJlbnQtbGF5ZXI9InN2ZzMiLz4KICA8cGF0aCBkPSJtIDYyLjA2NDk5OSw4LjU5MSAtOC42MzEsNC44NTkgTCA0NC4xOTIsOC4yNTggNDMuNzQ3LDguNDk4IHYgMy43NyBsIDkuNjg2OTk5LDUuNDMxIDguNjMsLTQuODQgdiAxLjk5NSBsIC04LjYzLDQuODYgLTkuMjQxOTk5LC01LjE5MiAtMC40NDUsMC4yNCB2IDAuNjQ2IGwgOS42ODY5OTksNS40MzIgOS42NjgsLTUuNDMyIHYgLTMuNzY5IGwgLTAuNDQ1LC0wLjI0IC05LjIyMyw1LjE3MyBMIDQ0Ljc4NCwxMS43MzIgViA5LjczNiBsIDguNjQ5OTk5LDQuODQgOS42NjgsLTUuNDMgViA1LjQzIGwgLTAuNDgyLC0wLjI3NyAtOS4xODYsNS4xNTUgLTguMjA0OTk5LC00LjU4MiA4LjIwNDk5OSwtNC42IDYuNzQxLDMuNzg3IDAuNTkzLC0wLjMzMiBWIDQuMTE5IEwgNTMuNDMzOTk5LDAgNDMuNzQ3LDUuNDMxIHYgMC41OTIgbCA5LjY4Njk5OSw1LjQzMiA4LjYzLC00Ljg2IHoiIGZpbGw9IiNlZTNkMmMiIGlkPSJwYXRoMSIvPgogIDxnIGlkPSJnMyIgdHJhbnNmb3JtPSJ0cmFuc2xhdGUoLTI1LjE1LDI1Ljg1OTM0NikiPgogICAgPHBhdGggZD0iTSAzNy40NDksMTguNDQzIFYgMS44NTIgaCAtMi41NTYgdiA2LjIwNyBjIDAsMC4wOTMgLTAuMDU2LDAuMTY3IC0wLjE0OCwwLjIwNCBBIDAuMjMsMC4yMyAwIDAgMSAzNC41MDUsOC4yMDcgQyAzMy42MzQsNy4xOTEgMzIuMjgyLDYuNjE4IDMwLjgsNi42MTggYyAtMy4xNjcsMCAtNS42NSwyLjY2IC01LjY1LDYuMDYgMCwxLjY2MyAwLjU3NSwzLjE5NyAxLjYzLDQuMzI0IGEgNS40NCw1LjQ0IDAgMCAwIDQuMDIsMS43MzYgYyAxLjQ2MywwIDIuODE1LC0wLjYxIDMuNzA0LC0xLjY2MiAwLjA1NiwtMC4wNzQgMC4xNjcsLTAuMDkzIDAuMjQsLTAuMDc0IDAuMDkzLDAuMDM3IDAuMTUsMC4xMSAwLjE1LDAuMjAzIHYgMS4yMzggeiBtIC02LjA5MywtMi4wMTQgYyAtMi4wMzgsMCAtMy42MywtMS42NDQgLTMuNjMsLTMuNzUgMCwtMi4xMDcgMS41OTIsLTMuNzUxIDMuNjMsLTMuNzUxIDIuMDM4LDAgMy42MywxLjY0NCAzLjYzLDMuNzUgMCwyLjEwNiAtMS41OTMsMy43NSAtMy42MywzLjc1IG0gMTkuNzYyLDIuMDE2IFYgNi44OTYgSCA0OC41ODEgViA4LjA2IGMgMCwwLjA5MyAtMC4wNTYsMC4xNjYgLTAuMTQ5LDAuMjAzIEEgMC4yLDAuMiAwIDAgMSA0OC4xOTIsOC4xOSBDIDQ3LjM0LDcuMTczIDQ2LjAwNiw2LjYgNDQuNDg3LDYuNiBjIC0zLjE2NywwIC01LjY0OSwyLjY2MSAtNS42NDksNi4wNiAwLDMuNCAyLjQ4Miw2LjA2IDUuNjUsNi4wNiAxLjQ2MywwIDIuODE1LC0wLjYxIDMuNzA0LC0xLjY4IDAuMDU1LC0wLjA3NSAwLjE2NiwtMC4wOTMgMC4yNCwtMC4wNzUgMC4wOTMsMC4wMzcgMC4xNDksMC4xMTEgMC4xNDksMC4yMDQgdiAxLjI1NiBoIDIuNTM3IHogTSA0NS4wNjIsMTYuNDMgYyAtMi4wMzgsMCAtMy42MywtMS42NDUgLTMuNjMsLTMuNzUgMCwtMi4xMDcgMS41OTIsLTMuNzUxIDMuNjMsLTMuNzUxIDIuMDM4LDAgMy42MywxLjY0NCAzLjYzLDMuNzUgMCwyLjEwNiAtMS41OTMsMy43NSAtMy42MywzLjc1IG0gMjcuNzgxLDIuMDE1IFYgNi44OTYgSCA3MC4zMDUgViA4LjA2IGMgMCwwLjA5MyAtMC4wNTUsMC4xNjYgLTAuMTQ4LDAuMjAzIC0wLjA5MywwLjAzNyAtMC4xODUsMCAtMC4yNCwtMC4wNzMgQyA2OS4wNjQsNy4xNzMgNjcuNzMxLDYuNiA2Ni4yMTIsNi42IGMgLTMuMTg2LDAgLTUuNjQ5LDIuNjYxIC01LjY0OSw2LjA4IDAsMy40MTcgMi40ODIsNi4wNiA1LjY0OSw2LjA2IDEuNDYzLDAgMi44MTUsLTAuNjEgMy43MDQsLTEuNjgyIDAuMDU2LC0wLjA3NCAwLjE2NywtMC4wOTMgMC4yNDEsLTAuMDc0IDAuMDkzLDAuMDM3IDAuMTQ4LDAuMTEgMC4xNDgsMC4yMDMgdiAxLjI1NiB6IE0gNjYuNzg2LDE2LjQzIGMgLTIuMDM3LDAgLTMuNjMsLTEuNjQ1IC0zLjYzLC0zLjc1IDAsLTIuMTA3IDEuNTkzLC0zLjc1MSAzLjYzLC0zLjc1MSAyLjAzNywwIDMuNjMsMS42NDQgMy42MywzLjc1IDAsMi4xMDYgLTEuNTkzLDMuNzUgLTMuNjMsMy43NSBtIDEwLjcwNiwwLjY0NyBjIDAuMDE5LDAgMC4wNTYsLTAuMDE5IDAuMDc0LC0wLjAxOSAwLjA1NiwwIDAuMTMsMC4wMzcgMC4xNjcsMC4wNzQgMC44NywxLjAxNiAyLjIyMiwxLjU4OSAzLjcwNCwxLjU4OSAzLjE2NywwIDUuNjUsLTIuNjYgNS42NSwtNi4wNiAwLC0xLjY2MyAtMC41NzUsLTMuMTk2IC0xLjYzLC00LjMyMyBBIDUuNDQsNS40NCAwIDAgMCA4MS40MzcsNi42IGMgLTEuNDYzLDAgLTIuODE1LDAuNjEgLTMuNzA0LDEuNjYzIC0wLjA1NiwwLjA3NCAtMC4xNDgsMC4wOTIgLTAuMjQsMC4wNzQgQyA3Ny40LDguMyA3Ny4zNDQsOC4yMjYgNzcuMzQ0LDguMTMzIFYgMS44NTIgaCAtMi41NTYgdiAxNi41OSBoIDIuNTU2IFYgMTcuMjggYyAwLC0wLjA5MyAwLjA1NiwtMC4xNjYgMC4xNDgsLTAuMjAzIG0gLTAuMjYsLTQuMzk4IGMgMCwtMi4xMDYgMS41OTQsLTMuNzUgMy42MzEsLTMuNzUgMi4wMzcsMCAzLjYzLDEuNjQ0IDMuNjMsMy43NSAwLDIuMTA2IC0xLjU5MywzLjc1IC0zLjYzLDMuNzUgLTIuMDM3LDAgLTMuNjMsLTEuNjYyIC0zLjYzLC0zLjc1IE0gOTQuNDc3LDkuMjYzIGMgMC4yNCwwIDAuNDYzLDAuMDE5IDAuNjEsMC4wNTYgViA2LjY5NSBBIDIuNCwyLjQgMCAwIDAgOTQuNjYyLDYuNjU4IGMgLTEuMzM0LDAgLTIuNTU2LDAuNjg0IC0zLjIwNCwxLjc3NCAtMC4wNTYsMC4wOTIgLTAuMTQ5LDAuMTMgLTAuMjQxLDAuMDkyIEEgMC4yMiwwLjIyIDAgMCAxIDkxLjA1LDguMzIxIFYgNi44OTggaCAtMi41MzcgdiAxMS41NjYgaCAyLjU1NiB2IC01LjEgYyAwLC0yLjUzIDEuMjk2LC00LjEgMy40MDgsLTQuMSBtIDQuODE1LC0yLjM2NyBoIC0yLjU5MyB2IDExLjU2NiBoIDIuNTkzIHogTSA5Ny45NTgsMS44NyBhIDEuNTcxLDEuNTcxIDAgMSAwIDAsMy4xNDEgMS41NzEsMS41NzEgMCAxIDAgMCwtMy4xNCBtIDguOTI4LDQuNzI5IGMgLTMuNTU2LDAgLTYuMTMxLDIuNTUgLTYuMTMxLDYuMDggMCwxLjcxNyAwLjYxMiwzLjI1IDEuNzA0LDQuMzYgMS4xMTIsMS4xMDggMi42NjcsMS43MTggNC40MDgsMS43MTggMS40NDUsMCAyLjU1NiwtMC4yNzcgNC42NjgsLTEuODMgbCAtMS40NjMsLTEuNTMzIGMgLTEuMDM4LDAuNjg0IC0yLjAwMSwxLjAxNiAtMi45NDUsMS4wMTYgLTIuMTQ5LDAgLTMuNzYsLTEuNjA3IC0zLjc2LC0zLjczMiAwLC0yLjEyNSAxLjYxMSwtMy43MzIgMy43NiwtMy43MzIgMS4wMTgsMCAxLjk2MywwLjMzMyAyLjkwOCwxLjAxNiBMIDExMS42NjQsOC40MyBDIDEwOS43NTcsNi44MDQgMTA4LjAzNCw2LjYgMTA2Ljg4Niw2LjYgbSA5LjE0OSw2Ljc2MiBhIDAuMiwwLjIgMCAwIDEgMC4xNDksLTAuMDU1IGggMC4wMTggYyAwLjA1NiwwIDAuMTExLDAuMDM3IDAuMTY3LDAuMDczIGwgNC4wOTMsNS4wNjMgaCAzLjE0OSBsIC01LjI5NywtNi4zOTMgYyAtMC4wNzUsLTAuMDkyIC0wLjA3NSwtMC4yMjIgMC4wMTgsLTAuMjk1IGwgNC44NzEsLTQuODYgaCAtMy4xMyBsIC00LjIwNCw0LjIxMyBjIC0wLjA1NiwwLjA1NSAtMC4xNDgsMC4wNzQgLTAuMjQxLDAuMDU1IGEgMC4yMywwLjIzIDAgMCAxIC0wLjEzLC0wLjIwMyBWIDEuODcgaCAtMi41NzQgdiAxNi41OTEgaCAyLjU1NiB2IC00LjUwOCBjIDAsLTAuMDU1IDAuMDE4LC0wLjEzIDAuMDc0LC0wLjE2NiB6IiBmaWxsPSIjMDAwMDAwIiBpZD0icGF0aDIiLz4KICAgIDxwYXRoIGQ9Im0gMTI3Ljc3NiwxOC43MzkgYyAyLjA5MywwIDQuMjIzLC0xLjI3NSA0LjIyMywtMy42OTUgMCwtMS41ODkgLTEsLTIuNjggLTMuMDM3LC0zLjM0NCBsIC0xLjM5LC0wLjQ2MiBjIC0wLjk0NCwtMC4zMTQgLTEuMzg5LC0wLjc1OCAtMS4zODksLTEuMzY3IDAsLTAuNzAyIDAuNjMsLTEuMTgzIDEuNTE5LC0xLjE4MyAwLjg1MiwwIDEuNjExLDAuNTU1IDIuMDkzLDEuNTE1IGwgMi4wNTYsLTEuMTA4IGMgLTAuNzU5LC0xLjU1MiAtMi4zMzQsLTIuNTEzIC00LjE0OSwtMi41MTMgLTIuMjk3LDAgLTMuOTYzLDEuNDc4IC0zLjk2MywzLjQ5MiAwLDEuNjA3IDAuOTYzLDIuNjc5IDIuOTQ0LDMuMzA3IGwgMS40MjcsMC40NjIgYyAxLDAuMzE0IDEuNDI2LDAuNzIgMS40MjYsMS4zNjcgMCwwLjk4IC0wLjkwOCwxLjMzIC0xLjY4NiwxLjMzIC0xLjAzNywwIC0xLjk2MywtMC42NjUgLTIuNDA3LC0xLjc1NSBsIC0yLjA5MywxLjEwOSBjIDAuNjg1LDEuNzU1IDIuMzcsMi44NDUgNC40MjYsMi44NDUgTSA1OC4yMywxOC42MjggYyAwLjgxNSwwIDEuNTM4LC0wLjA3NCAxLjk0NSwtMC4xMyB2IC0yLjIxNiBhIDE0LDE0IDAgMCAxIC0xLjI3OCwwLjA3MyBjIC0xLjAzNywwIC0xLjgzMywtMC4xODQgLTEuODMzLC0yLjQyIFYgOS4xODcgYyAwLC0wLjEzIDAuMDkyLC0wLjIyMiAwLjIyMiwtMC4yMjIgaCAyLjUgViA2Ljg3NyBoIC0yLjUgQSAwLjIxNCwwLjIxNCAwIDAgMSA1Ny4wNjQsNi42NTYgViAzLjMzIGggLTIuNTU2IHYgMy4zNDQgYyAwLDAuMTMgLTAuMDkzLDAuMjIyIC0wLjIyMywwLjIyMiBoIC0xLjc3OCB2IDIuMDg4IGggMS43NzggYyAwLjEzLDAgMC4yMjMsMC4wOTIgMC4yMjMsMC4yMjEgdiA1LjM3NyBjIDAsNC4wNDYgMi43MDQsNC4wNDYgMy43MjIsNC4wNDYiIGZpbGw9IiMwMDAwMDAiIGlkPSJwYXRoMyIvPgogIDwvZz4KPC9zdmc+", + "created": 1768545889515 + }, + "c8f308da71e262431b6d749e8dcb0a4bc680b0dc12a84b90698c759e23ab3e416b884f49808e05672379da58ccdbf95f": { + "mimeType": "image/svg+xml", + "id": "c8f308da71e262431b6d749e8dcb0a4bc680b0dc12a84b90698c759e23ab3e416b884f49808e05672379da58ccdbf95f", + "dataURL": "data:image/svg+xml;base64,PHN2ZyB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMjAwMC9zdmciIHdpZHRoPSIzNSIgaGVpZ2h0PSIzNCIgdmlld0JveD0iMCAwIDM1IDM0IiBmaWxsPSJub25lIj4KPHBhdGggZmlsbC1ydWxlPSJldmVub2RkIiBjbGlwLXJ1bGU9ImV2ZW5vZGQiIGQ9Ik0zNC41NTkzIDI3Ljg1MjRMMTguNTM3OSAwLjQ1OTM1MkMxOC4xNzk2IC0wLjE1MzExNyAxNy4yOTQ0IC0wLjE1MzExNyAxNi45MzYxIDAuNDU5MzUyTDMuMzIwNjYgMjMuNzM4NUMzLjA4MTM5IDI0LjE0OCAzLjUxMzI3IDI0LjYyMDggMy45NDI1IDI0LjQxOTVDNC4xMTg3NyAyNC4zMzY1IDQuMzIyMiAyNC4yNTMxIDQuNTQ1NTkgMjQuMTY5OUw0LjU0MTAzIDI0LjE3OUM0LjU0MTAzIDI0LjE3OSA0LjkxMzAzIDI0LjAzNDggNS41NDE2MSAyMy44MzU2QzYuMjM0NCAyMy42MjI4IDYuOTkzNTUgMjMuNDIxMyA3LjY4MDgxIDIzLjI1MTFDOS42MDM4MiAyMi44MDcyIDEyLjEyMzcgMjIuNDQ1NyAxNC4zOTU5IDIyLjgyM0MxNi40ODcgMjMuMTcgMTcuOTQzMiAyMy43MzA2IDE5LjAwMzkgMjQuMjgxNUMxOS40OTAyIDI0LjUzNDQgMTkuOTQ5NCAyMy44OTk2IDE5LjU1NjMgMjMuNTE3NUMxOC4wMzU2IDIyLjAzOTYgMTUuNjk1NiAyMC4yNTU2IDEyLjEzNjQgMTkuNTA4QzExLjk4MDYgMTkuNDc1MyAxMS45MDI0IDE5LjI5OTUgMTEuOTgzIDE5LjE2MTlMMTcuMzQzMyA5Ljk5N0MxNy41MTk1IDkuNjk1OTQgMTcuOTU0NSA5LjY5NTk0IDE4LjEzMDggOS45OTdMMjguMzY4IDI3LjUwMDRDMjguNTgzNyAyNy44NjkgMjguNDAyOCAyOC4zMzk4IDI3Ljk5NzkgMjguNDc1NUMyNy4xOTU3IDI4Ljc0MzYgMjYuMjM3NCAyOC45NDQxIDI1LjE2NDcgMjguOTQ0MUMyNC40ODIyIDI4Ljk0NDEgMjMuNzk1NyAyOC44NjM2IDIzLjEwNzUgMjguNzAzN0MyMy4xMDc1IDI4LjcwMzcgMjEuMTIzNiAyOC4zOTA2IDE4LjQ3MiAyNi44MDg1QzE1LjgyMDQgMjUuMjI2OCAxMC43MDc4IDIzLjIwODggNC45NzA5OCAyNi40NzU3QzIuNTI2NjMgMjguMDQ2NSAwLjg5Nzk0OCAzMC4xOTQ4IDAuMDM5NzIwNiAzMS41NDM2Qy0wLjA5NDcwMDUgMzEuNzU0NyAwLjEzNzM1IDMyLjAwNjUgMC4zNTgwOTkgMzEuODg4N0MyLjMyOTIxIDMwLjgzNTYgNy42NDM3NyAyOC4zNDg3IDEyLjQzMjcgMjkuMjQyOEMxNS4yNzkxIDI5Ljc3NCAxOC41MzE5IDMxLjkyMjggMjEuOTEzMyAzMy4xNTVDMjcuNTA3MSAzNS4yMjc4IDMyLjk5MTIgMzEuNzU1IDMzLjAwMyAzMS43NDc1QzMzLjU4MjcgMzEuMzY0NCAzMy45MjMyIDMxLjA3MzIgMzMuOTgxMiAzMS4wMjI3TDM0LjA1MTIgMzAuOTYxN0MzNC45NDUgMzAuMTgxNiAzNS4xNTgzIDI4Ljg3NjMgMzQuNTU5MyAyNy44NTI0WiIgZmlsbD0iIzAwQURENCIvPgo8L3N2Zz4=", + "created": 1768546735835 + } + } +} \ No newline at end of file diff --git a/src/content/docs/integrations/databricks.mdx b/src/content/docs/integrations/databricks.mdx new file mode 100644 index 000000000..e3bd43d73 --- /dev/null +++ b/src/content/docs/integrations/databricks.mdx @@ -0,0 +1,235 @@ +--- +title: Databricks +--- + +import { Steps } from "@astrojs/starlight/components"; +import Op from "@components/see-also/Op.astro"; + +[Databricks](https://www.databricks.com/) is a unified analytics platform built +on Apache Spark, offering a Lakehouse architecture that combines data lake +flexibility with data warehouse performance. [Unity +Catalog](https://docs.databricks.com/en/data-governance/unity-catalog/index.html) +provides centralized governance for all data assets. + +![Databricks](databricks.excalidraw) + +Tenzir sends events to Databricks using the +[`to_databricks`](/reference/operators/to_databricks) operator, which writes to +managed Delta tables with full Unity Catalog governance. + +## Architecture + +The operator stages optimized Parquet files in a Unity Catalog Volume and +commits them via `COPY INTO` using the SQL Statement API. + +| Step | Who | Action | DBU Cost | +| :--: | ---------------- | ------------------------------------ | :------: | +| ① | Tenzir | Write Parquet to staging | ❌ | +| ② | Tenzir → SQL API | POST COPY INTO command | ❌ | +| ③ | SQL Warehouse | Read staged files (via UC perms) | ✅ | +| ④ | SQL Warehouse | Write to table + commit `_delta_log` | ✅ | +| ⑤ | Tenzir | Delete staging files _(not shown)_ | ❌ | + +Key characteristics: + +- Produces fully managed Delta tables with complete Unity Catalog governance +- Automatic schema evolution via `mergeSchema` +- Idempotent commits (`COPY INTO` tracks processed files) +- Data is queryable immediately after each commit + +:::tip[Cost optimization] +Each `COPY INTO` execution consumes DBUs. With Serverless SQL Warehouses at +~$0.07/DBU (list price), ingesting 1GB typically costs a few cents. Increase +`flush_interval` and `file_size` to reduce commit frequency and cost. +::: + +## Configuration + +### Authentication + +Tenzir uses OAuth machine-to-machine (M2M) authentication with a Databricks +service principal. Use a [secret store](/explanations/secrets) to manage +credentials securely. + + + +1. **Create a service principal** + + Navigate to **Account Console → User management → Service principals** and + create a new service principal. Note the Application (client) ID. + +2. **Generate an OAuth secret** + + In the service principal's **Secrets** tab, generate a new secret. Copy the + secret immediately—it's only shown once. + +3. **Grant Unity Catalog permissions** + + ```sql + -- Catalog access + GRANT USE CATALOG ON CATALOG my_catalog TO `my-service-principal`; + + -- Schema access and table creation + GRANT USE SCHEMA ON SCHEMA my_catalog.my_schema TO `my-service-principal`; + GRANT CREATE TABLE ON SCHEMA my_catalog.my_schema TO `my-service-principal`; + + -- Staging volume access + GRANT READ VOLUME, WRITE VOLUME + ON VOLUME my_catalog.my_schema.staging + TO `my-service-principal`; + ``` + +4. **Configure a SQL Warehouse** + + Create or identify a SQL Warehouse. Serverless warehouses are recommended + for ingestion due to fast startup and automatic scaling. Note the warehouse + ID from the connection details. + + + +### Staging Volume Setup + +Create a Unity Catalog Volume to stage Parquet files: + +```sql +CREATE VOLUME IF NOT EXISTS my_catalog.my_schema.tenzir_staging; +``` + +The operator writes files to this volume and deletes them after successful +`COPY INTO` commits. + +## Partitioning + +Delta tables use Hive-style partitioning, creating a directory structure like +`day=2025-01-15/class_uid=4001/`. The `partition_by` parameter accepts a list +of column names—columns must exist in the events being written. + +To partition by derived values (e.g., daily buckets from a timestamp), add the +partition column to your events before writing: + +```tql +subscribe "events" +day = time.round(1d) +to_databricks + ... + partition_by=[day, class_uid] +``` + +If the target table already exists, the operator queries Unity Catalog for the +existing partition scheme and aligns staging files accordingly. The +`partition_by` parameter is ignored for existing tables since partition columns +are immutable after table creation. + +## Write-Time Optimizations + +Tenzir applies optimizations that produce query-efficient files: + +- **Partition-aligned files**: Each output file contains data for exactly one + partition value, enabling partition pruning—queries filtering on partition + columns skip irrelevant directories without opening files. +- **Sorted rows**: Tight min/max statistics enable aggressive data skipping. +- **1GB file targets**: Optimal size for scan efficiency, reduces need for `OPTIMIZE`. +- **Zstd compression**: High compression ratio with fast decompression. + +## Examples + +### Basic ingestion to a bronze table + +```tql +let $client_id = secret("databricks-client-id") +let $client_secret = secret("databricks-client-secret") + +from_file "/var/log/app/events.json" +to_databricks + workspace="https://adb-1234567890.azuredatabricks.net", + catalog="analytics", + schema="bronze", + table="app_events", + client_id=$client_id, + client_secret=$client_secret, + warehouse_id="abc123def456" +``` + +### OCSF security events with partitioning + +```tql +let $client_id = secret("databricks-client-id") +let $client_secret = secret("databricks-client-secret") + +subscribe "ocsf" +day = time.round(1d) +to_databricks + workspace="https://dbc-a1b2c3d4-e5f6.cloud.databricks.com", + catalog="security", + schema="silver", + table="security_events", + client_id=$client_id, + client_secret=$client_secret, + warehouse_id="abc123def456", + partition_by=[day, class_uid], + sort_by=[src_endpoint.ip, dst_endpoint.ip] +``` + +### High-volume network telemetry + +Optimize for cost with larger batches and files: + +```tql +let $client_id = secret("databricks-client-id") +let $client_secret = secret("databricks-client-secret") + +subscribe "netflow" +day = time.round(1d) +to_databricks + workspace="https://1234567890123456.7.gcp.databricks.com", + catalog="network", + schema="bronze", + table="flows", + client_id=$client_id, + client_secret=$client_secret, + warehouse_id="abc123def456", + partition_by=[day], + sort_by=[src_ip, dst_ip], + flush_interval=15m, + file_size=1Gi +``` + +## Comparison with Other Approaches + +### vs. Cribl Stream + +[Cribl Stream](https://docs.cribl.io/stream/destinations-databricks/) writes +files to Unity Catalog Volumes but does not commit them to Delta tables. +Customers must separately configure Autoloader or run `COPY INTO` to make data +queryable. + +Tenzir's [`to_databricks`](/reference/operators/to_databricks) commits directly +to managed Delta tables—data is queryable immediately after each flush with no +additional setup required. + +### vs. Databricks Autoloader + +[Autoloader](https://docs.databricks.com/en/ingestion/auto-loader/index.html) +monitors cloud storage for new files and incrementally loads them. It requires +files to already exist in cloud storage, making it a "pull" model. + +Tenzir's [`to_databricks`](/reference/operators/to_databricks) is a "push" +model—data flows directly from pipelines to Databricks without intermediate +storage management. Use Autoloader when data already lands in cloud storage from +other sources. + +### vs. Delta Live Tables (DLT) + +DLT provides managed ETL pipelines within Databricks. Tenzir complements DLT +by handling data collection and initial ingestion, while DLT handles +downstream transformations within the lakehouse. + +### vs. Kafka + Databricks Connector + +The Kafka connector requires managing Kafka infrastructure. Tenzir can ingest +directly from sources, eliminating the intermediate message broker for +simpler architectures. + +## See Also + +- to_databricks diff --git a/src/content/docs/reference/operators/to_databricks.mdx b/src/content/docs/reference/operators/to_databricks.mdx new file mode 100644 index 000000000..237d219e3 --- /dev/null +++ b/src/content/docs/reference/operators/to_databricks.mdx @@ -0,0 +1,366 @@ +--- +title: to_databricks +category: Outputs/Events +example: 'to_databricks workspace="https://…' +--- + +import Integration from '@components/see-also/Integration.astro'; + +Sends events to a Databricks Unity Catalog table. + +```tql +to_databricks workspace=string, catalog=string, schema=string, table=string, + client_id=string, client_secret=string, warehouse_id=string, + [staging_volume=string], [partition_by=list], + [sort_by=list], [flush_interval=duration], [file_size=int], + [ingest_mode=string] +``` + +## Description + +The `to_databricks` operator writes events to a managed Delta table in +[Databricks Unity Catalog](https://docs.databricks.com/en/data-governance/unity-catalog/index.html). +It stages optimized Parquet files in a Unity Catalog Volume and commits them +atomically via `COPY INTO`, making data immediately queryable in Databricks +SQL, notebooks, and downstream tools. + +See the [Databricks integration overview](/integrations/databricks) for +architecture details and configuration guidance. + +### `workspace = string` + +The URL of your Databricks workspace. This is the base URL you see when logged +into Databricks. + +| Cloud | Example | +| ----- | ---------------------------------------------------- | +| AWS | `https://dbc-a1b2c3d4-e5f6.cloud.databricks.com` | +| Azure | `https://adb-1234567890123456.7.azuredatabricks.net` | +| GCP | `https://1234567890123456.7.gcp.databricks.com` | + +The operator derives all API endpoints from this URL automatically, including +the Unity Catalog REST API (`/api/2.1/unity-catalog/`) and the SQL Statement +API (`/api/2.0/sql/statements`). + +### `catalog = string` + +The Unity Catalog catalog name. Catalogs are the top-level namespace and +typically represent environments or data domains. + +Common patterns include environment-based naming (`prod`, `dev`, `staging`) or +domain-based naming (`analytics`, `security`, `finance`). New workspaces have a +default catalog named `main`. + +### `schema = string` + +The schema (also called database) within the catalog. Schemas group related +tables and control access at a finer granularity than catalogs. + +For data lake architectures, consider medallion-style naming: `bronze` for raw +ingested data, `silver` for cleaned and conformed data, `gold` for +business-ready aggregates. Alternatively, use domain-specific names like +`network_logs`, `application_events`, or `audit_trail`. + +The schema must exist before ingestion. Create it with: + +```sql +CREATE SCHEMA IF NOT EXISTS my_catalog.bronze; +``` + +### `table = string` + +The name of the target table. The operator creates the table if it does not +exist, inferring the schema from the first batch of events. + +Table naming conventions vary by use case. For raw ingestion, names often +reflect the data source: `zeek_conn`, `cloudtrail_events`, `firewall_logs`. +For normalized data, names may reflect the event type or entity: +`network_connections`, `user_authentications`, `file_operations`. + +### `client_id = string` + +The Application (client) ID of a Databricks service principal for OAuth +machine-to-machine authentication. + +Create a service principal in **Account Console → User management → Service +principals**. The Application ID displayed after creation is your `client_id`. + +### `client_secret = string` + +The OAuth secret for the service principal. Generate this in the service +principal's **Secrets** tab. The secret is shown only once at creation time. + +The service principal requires these Unity Catalog privileges: + +```sql +GRANT USE CATALOG ON CATALOG my_catalog TO `my-service-principal`; +GRANT USE SCHEMA ON SCHEMA my_catalog.bronze TO `my-service-principal`; +GRANT CREATE TABLE ON SCHEMA my_catalog.bronze TO `my-service-principal`; +GRANT READ VOLUME, WRITE VOLUME ON VOLUME my_catalog.bronze.staging + TO `my-service-principal`; +``` + +### `warehouse_id = string` + +The ID of the SQL Warehouse that executes `COPY INTO` statements. + +Find this in **SQL Warehouses → [Your Warehouse] → Connection details**. The +warehouse ID is the final segment of the HTTP Path, after `/sql/1.0/warehouses/`. + +Serverless SQL Warehouses are recommended for ingestion workloads due to fast +startup times and automatic scaling. Each `COPY INTO` execution consumes DBUs +at your warehouse's rate (typically $0.07–0.70/DBU depending on tier). + +### `staging_volume = string (optional)` + +The Unity Catalog Volume where Parquet files are staged before being committed +to the table. If not specified, the operator creates a temporary staging +location. + +Volumes provide managed storage within Unity Catalog: + +```sql +CREATE VOLUME IF NOT EXISTS my_catalog.bronze.ingestion_staging; +``` + +Staged files are automatically removed after successful commits. + +### `partition_by = list (optional)` + +List of column names to partition by. The columns must exist in the events +being written. Delta tables use Hive-style partitioning, creating a directory +structure like `day=2025-01-15/class_uid=4001/`. + +If the target table already exists, the operator queries Unity Catalog for the +existing partition scheme and ignores this parameter—partition columns are +immutable after table creation. Staging files are automatically aligned with +the target table's partitions for optimal `COPY INTO` performance. + +If the table does not exist, these columns define the partition scheme for the +newly created table. If omitted, creates an unpartitioned table. + +To partition by derived values (e.g., daily buckets from a timestamp), add the +partition column to your events before writing: + +```tql +subscribe "events" +day = time.round(1d) +to_databricks + ... + partition_by=[day, class_uid] +``` + +Avoid over-partitioning. Each partition should contain at least 1GB of data +for optimal file sizes and query performance. + +### `sort_by = list (optional)` + +Columns to sort rows by within each output file. Sorting improves query +performance by enabling data skipping: Databricks reads Parquet column +statistics (min/max values per row group) to skip files that cannot contain +matching rows. + +When `partition_by` is specified, rows are always sorted by partition columns +first to ensure partition alignment (each file contains data for exactly one +partition). The `sort_by` columns are applied as secondary sort keys. + +For best results, sort by columns that appear frequently in `WHERE` clauses: + +```tql +subscribe "events" +day = time.round(1d) +to_databricks + ... + partition_by=[day], + sort_by=[src_ip, dst_ip] +``` + +This produces files where queries like `WHERE src_ip = '10.0.0.1'` can skip +entire files based on min/max statistics. + +### `flush_interval = duration (optional)` + +Maximum time to buffer events before committing to the table. This controls +data freshness—the maximum delay before events become queryable. + +Defaults to `5m`. When this interval elapses, the operator commits whatever +data has accumulated, even if `file_size` has not been reached. + +| Interval | Data Latency | Recommended For | +| -------- | ------------ | ---------------------------------- | +| `1m` | ~1 minute | Real-time dashboards, alerting | +| `5m` | ~5 minutes | General analytics, SOC workflows | +| `15m` | ~15 minutes | Batch analytics, cost optimization | +| `1h` | ~1 hour | Archival, compliance retention | + +Shorter intervals improve freshness but may produce smaller files. The +operator optimizes file layout regardless of flush timing. + +### `file_size = int (optional)` + +Target size for output Parquet files. + +Defaults to `1Gi`. When the buffer reaches this size, the operator flushes +immediately regardless of `flush_interval`. This produces optimally-sized +files that minimize the need for later compaction via `OPTIMIZE`. + +Databricks recommends files between 256MB and 1GB for best query performance. +Larger files reduce metadata overhead and improve scan efficiency; smaller +files enable finer-grained data skipping. + +For high-volume streams, the `file_size` threshold triggers most flushes. For +low-volume streams, `flush_interval` ensures timely commits despite smaller +file sizes. + +### `ingest_mode = string (optional)` + +Controls table creation and append behavior: + +- `create_append`: Creates the table if it does not exist, otherwise appends. +- `create`: Creates the table, failing if it already exists. +- `append`: Appends to an existing table, failing if it does not exist. + +Defaults to `create_append`. + +When creating tables, the operator infers the schema from the first batch of +events. Subsequent batches with additional fields trigger automatic schema +evolution via `mergeSchema`. Fields present in the table but missing from +events are filled with `null`. + +## Write-Time Optimizations + +The operator applies several optimizations to produce query-efficient files +that minimize the need for post-write maintenance: + +**Partition-aligned files**: Each output file contains data for exactly one +partition value. This enables partition pruning—queries filtering on partition +columns skip irrelevant directories without opening files. + +**Sorted rows**: Rows are sorted by partition columns (if specified) followed +by `sort_by` columns. Sorting produces tight min/max statistics in Parquet row +groups, enabling aggressive data skipping during queries. + +**Optimal file sizing**: The `file_size` parameter (default 1GB) produces +files in Databricks' recommended size range, reducing metadata overhead and +the urgency of running `OPTIMIZE` for compaction. + +**Efficient encoding**: The operator leverages Apache Arrow's Parquet writer +with dictionary encoding, run-length encoding, and Zstd compression for +compact, fast-to-read files. + +## Examples + +### Send events to a bronze layer table + +Ingest raw Zeek connection logs into a medallion-architecture data lake: + +```tql +let $client_id = secret("databricks-client-id") +let $client_secret = secret("databricks-client-secret") + +from_file "/var/log/zeek/conn.log", read=read_zeek_tsv +day = ts.round(1d) +to_databricks + workspace="https://adb-1234567890.azuredatabricks.net", + catalog="security", + schema="bronze", + table="zeek_conn", + client_id=$client_id, + client_secret=$client_secret, + warehouse_id="abc123def456", + partition_by=[day] +``` + +### Stream OCSF events with partitioning + +Write normalized security events with OCSF-optimized partitioning: + +```tql +let $client_id = secret("databricks-client-id") +let $client_secret = secret("databricks-client-secret") + +subscribe "ocsf" +day = time.round(1d) +to_databricks + workspace="https://dbc-a1b2c3d4.cloud.databricks.com", + catalog="prod", + schema="silver", + table="security_events", + client_id=$client_id, + client_secret=$client_secret, + warehouse_id="abc123def456", + staging_volume="tenzir_staging", + partition_by=[day, class_uid] +``` + +### Low-latency ingestion for alerting + +Minimize data latency for real-time security monitoring: + +```tql +let $client_id = secret("databricks-client-id") +let $client_secret = secret("databricks-client-secret") + +subscribe "alerts" +to_databricks + workspace="https://1234567890123456.7.gcp.databricks.com", + catalog="security", + schema="realtime", + table="high_priority_alerts", + client_id=$client_id, + client_secret=$client_secret, + warehouse_id="abc123def456", + flush_interval=1m, + file_size=256Mi +``` + +### Optimized ingestion for network analytics + +High-volume network telemetry with sorting for common query patterns: + +```tql +let $client_id = secret("databricks-client-id") +let $client_secret = secret("databricks-client-secret") + +subscribe "network" +day = time.round(1d) +to_databricks + workspace="https://adb-1234567890.azuredatabricks.net", + catalog="analytics", + schema="silver", + table="network_flows", + client_id=$client_id, + client_secret=$client_secret, + warehouse_id="abc123def456", + partition_by=[day], + sort_by=[src_ip, dst_ip, dst_port], + flush_interval=5m, + file_size=1Gi +``` + +### Cost-optimized batch ingestion + +Reduce DBU consumption for high-volume archival workloads: + +```tql +let $client_id = secret("databricks-client-id") +let $client_secret = secret("databricks-client-secret") + +export +day = time.round(1d) +to_databricks + workspace="https://dbc-a1b2c3d4-e5f6.cloud.databricks.com", + catalog="archive", + schema="compliance", + table="audit_logs", + client_id=$client_id, + client_secret=$client_secret, + warehouse_id="abc123def456", + flush_interval=15m, + file_size=1Gi, + partition_by=[day, source_type] +``` + +## See Also + +- databricks diff --git a/src/sidebar.ts b/src/sidebar.ts index d4b5f90ed..868822294 100644 --- a/src/sidebar.ts +++ b/src/sidebar.ts @@ -301,6 +301,7 @@ export const integrations = [ label: "Data Tools", items: [ "integrations/clickhouse", + "integrations/databricks", "integrations/elasticsearch", "integrations/graylog", "integrations/opensearch",