From 21d69a5390b280b5ca1dd8e9d4d24b7b4f46d3ae Mon Sep 17 00:00:00 2001 From: Lorenzo Turrino Date: Mon, 20 May 2024 22:56:00 +0100 Subject: [PATCH 1/5] basic test instrumentation --- server/app.py | 39 +++++++++++++++++-- server/apps/instrumentation/__init__.py | 0 .../apps/instrumentation/instrumentation.py | 20 ++++++++++ server/apps/videos/videos.py | 2 +- 4 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 server/apps/instrumentation/__init__.py create mode 100644 server/apps/instrumentation/instrumentation.py diff --git a/server/app.py b/server/app.py index 9e372d9..75cd36a 100644 --- a/server/app.py +++ b/server/app.py @@ -1,9 +1,18 @@ +from apps.videos import videos +from apps.instrumentation import instrumentation from flask import Flask, request, render_template +from time import time + +from opentelemetry.metrics import get_meter_provider -from apps.videos import videos app = Flask(__name__) +default_meter = get_meter_provider().get_meter("default") +meter_likes = default_meter.create_counter("video_likes", description="Calls to the Like Endpoint") +meter_dislikes = default_meter.create_counter("video_dislikes", description="Calls to the Dislike Endpoint") +meter_latency_get = default_meter.create_histogram("get_video_latency_seconds", unit="s", description="Latency of Video information retrieval") + @app.get("/api/v1/video") def get_videos(): @@ -14,11 +23,28 @@ def get_videos(): @app.get("/api/v1/video/") def get_video_details(id): - return videos.get(id) # Unhandled exception on purpose + start = time() + video = videos.get(id) + end = time() + meter_latency_get.record( + end - start, + { + "id": id, + }, + ) + + return video @app.get("/api/v1/video//like") def like_video(id): + meter_likes.add( + 1, + { + "user_ip": request.remote_addr, + "id": id, + }, + ) videos.like(id) return ("OK", 204) @@ -26,6 +52,13 @@ def like_video(id): @app.get("/api/v1/video//dislike") def dislike_video(id): + meter_dislikes.add( + 1, + { + "user_ip": request.remote_addr, + "id": id, + }, + ) videos.dislike(id) return ("OK", 204) @@ -47,5 +80,5 @@ def index(): if __name__ == "__main__": - # TODO: init Open Telemetry + instrumentation.init() app.run(host="0.0.0.0") diff --git a/server/apps/instrumentation/__init__.py b/server/apps/instrumentation/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/apps/instrumentation/instrumentation.py b/server/apps/instrumentation/instrumentation.py new file mode 100644 index 0000000..926b258 --- /dev/null +++ b/server/apps/instrumentation/instrumentation.py @@ -0,0 +1,20 @@ +from opentelemetry import metrics +from opentelemetry.sdk.metrics import MeterProvider +from opentelemetry.sdk.metrics.export import ( + ConsoleMetricExporter, # Useful for debugging - exports the metrics to console + PeriodicExportingMetricReader, # Reader that batches metrics in configurable time intervals before sending it to the exporter +) + +def init(): + + # Configure the provider with the exporter and reader + exporter = ConsoleMetricExporter() + reader = PeriodicExportingMetricReader( + exporter, export_interval_millis=15000, export_timeout_millis=5000 + ) + provider = MeterProvider(metric_readers=[reader]) + + # Set the global meter provider, and create a Meter for usage + metrics.set_meter_provider(provider) + + print("OTEL Metrics successfully initialised") diff --git a/server/apps/videos/videos.py b/server/apps/videos/videos.py index e209548..57c257d 100644 --- a/server/apps/videos/videos.py +++ b/server/apps/videos/videos.py @@ -28,6 +28,7 @@ def get(id): with open(DATABASE_JSON_LOCATION) as db: videos = json.load(db) + time.sleep(math.fabs(random.gauss(mu=0, sigma=0.5))) for video in videos: if video["id"] == id: return video @@ -42,7 +43,6 @@ def like(id): if video["id"] == id: video["likes"] += 1 continue - time.sleep(math.fabs(random.gauss(mu=1, sigma=0.5))) with open(DATABASE_JSON_LOCATION, "w") as db: json.dump(videos, db, indent=2) From 77816a722c80ac041257bc75a619a7277367c2e7 Mon Sep 17 00:00:00 2001 From: Lorenzo Turrino Date: Tue, 21 May 2024 18:10:19 +0100 Subject: [PATCH 2/5] move to push OTPL metrics --- server/app.py | 4 +- .../apps/instrumentation/instrumentation.py | 9 ++- server/requirements.txt | 67 ++++++++++++++++--- 3 files changed, 67 insertions(+), 13 deletions(-) diff --git a/server/app.py b/server/app.py index 75cd36a..ae7cd5b 100644 --- a/server/app.py +++ b/server/app.py @@ -11,7 +11,7 @@ default_meter = get_meter_provider().get_meter("default") meter_likes = default_meter.create_counter("video_likes", description="Calls to the Like Endpoint") meter_dislikes = default_meter.create_counter("video_dislikes", description="Calls to the Dislike Endpoint") -meter_latency_get = default_meter.create_histogram("get_video_latency_seconds", unit="s", description="Latency of Video information retrieval") +meter_latency_get = default_meter.create_histogram("get_video_latency_milliseconds", unit="ms", description="Latency of Video information retrieval") @app.get("/api/v1/video") @@ -27,7 +27,7 @@ def get_video_details(id): video = videos.get(id) end = time() meter_latency_get.record( - end - start, + (end - start) * 1000, { "id": id, }, diff --git a/server/apps/instrumentation/instrumentation.py b/server/apps/instrumentation/instrumentation.py index 926b258..b83ad32 100644 --- a/server/apps/instrumentation/instrumentation.py +++ b/server/apps/instrumentation/instrumentation.py @@ -1,18 +1,23 @@ from opentelemetry import metrics +from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import OTLPMetricExporter from opentelemetry.sdk.metrics import MeterProvider from opentelemetry.sdk.metrics.export import ( ConsoleMetricExporter, # Useful for debugging - exports the metrics to console PeriodicExportingMetricReader, # Reader that batches metrics in configurable time intervals before sending it to the exporter ) +from opentelemetry.sdk.resources import SERVICE_NAME, Resource + def init(): + # Create a unique identifier for this App + resource = Resource(attributes={SERVICE_NAME: "video-voter"}) # Configure the provider with the exporter and reader - exporter = ConsoleMetricExporter() + exporter = OTLPMetricExporter(insecure=True) # Endpoint provided via Environment variable - OTEL_EXPORTER_OTLP_ENDPOINT reader = PeriodicExportingMetricReader( exporter, export_interval_millis=15000, export_timeout_millis=5000 ) - provider = MeterProvider(metric_readers=[reader]) + provider = MeterProvider(resource=resource, metric_readers=[reader]) # Set the global meter provider, and create a Meter for usage metrics.set_meter_provider(provider) diff --git a/server/requirements.txt b/server/requirements.txt index fd2389c..f4487ee 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,9 +1,58 @@ -blinker==1.8.2 -click==8.1.7 -flask==3.0.3 -importlib-metadata==7.1.0 -itsdangerous==2.2.0 -jinja2==3.1.4 -MarkupSafe==2.1.5 -werkzeug==3.0.3 -zipp==3.18.2 +appdirs==1.4.4 +asgiref==3.8.1 +black==22.6.0 +certifi==2022.6.15 +charset-normalizer==2.1.1 +click==8.1.3 +Deprecated==1.2.14 +distlib==0.3.1 +filelock==3.0.12 +Flask==2.2.2 +googleapis-common-protos==1.63.0 +grpcio==1.64.0 +idna==3.3 +importlib-metadata==7.0.0 +itsdangerous==2.1.2 +Jinja2==3.1.2 +MarkupSafe==2.1.1 +mypy-extensions==0.4.3 +opentelemetry-api==1.24.0 +opentelemetry-distro==0.45b0 +opentelemetry-exporter-otlp==1.24.0 +opentelemetry-exporter-otlp-proto-common==1.24.0 +opentelemetry-exporter-otlp-proto-grpc==1.24.0 +opentelemetry-exporter-otlp-proto-http==1.24.0 +opentelemetry-instrumentation==0.45b0 +opentelemetry-instrumentation-asgi==0.45b0 +opentelemetry-instrumentation-asyncio==0.45b0 +opentelemetry-instrumentation-aws-lambda==0.45b0 +opentelemetry-instrumentation-dbapi==0.45b0 +opentelemetry-instrumentation-flask==0.45b0 +opentelemetry-instrumentation-grpc==0.45b0 +opentelemetry-instrumentation-jinja2==0.45b0 +opentelemetry-instrumentation-logging==0.45b0 +opentelemetry-instrumentation-requests==0.45b0 +opentelemetry-instrumentation-sqlite3==0.45b0 +opentelemetry-instrumentation-urllib==0.45b0 +opentelemetry-instrumentation-urllib3==0.45b0 +opentelemetry-instrumentation-wsgi==0.45b0 +opentelemetry-propagator-aws-xray==1.0.1 +opentelemetry-proto==1.24.0 +opentelemetry-sdk==1.24.0 +opentelemetry-semantic-conventions==0.45b0 +opentelemetry-test-utils==0.45b0 +opentelemetry-util-http==0.45b0 +packaging==24.0 +pathspec==0.9.0 +platformdirs==2.5.2 +protobuf==4.25.3 +python-dotenv==0.20.0 +requests==2.28.1 +six==1.15.0 +tomli==2.0.1 +typing-extensions==4.3.0 +urllib3==1.26.12 +virtualenv==20.2.1 +Werkzeug==2.2.2 +wrapt==1.16.0 +zipp==3.8.1 From 7a885076fc8eebe2ffa4b6a0e6d780e09e4a055a Mon Sep 17 00:00:00 2001 From: Lorenzo Turrino Date: Tue, 21 May 2024 18:14:11 +0100 Subject: [PATCH 3/5] Add OTEL Collector and docker-compose file --- pipeline/compose.yaml | 27 ++++++++++++++++++++++++ pipeline/conf/otel-collector-config.yaml | 27 ++++++++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 pipeline/compose.yaml create mode 100644 pipeline/conf/otel-collector-config.yaml diff --git a/pipeline/compose.yaml b/pipeline/compose.yaml new file mode 100644 index 0000000..b26f6bf --- /dev/null +++ b/pipeline/compose.yaml @@ -0,0 +1,27 @@ +services: + # Video Voter Application + video-voter: + build: ../server + depends_on: + - otel-collector # Depends on otel-collector to be running first + environment: + OTEL_EXPORTER_OTLP_ENDPOINT: otel-collector:4317 + ports: + - "5000:5000" + restart: unless-stopped + + + # OpenTelemetry Collector + otel-collector: + image: otel/opentelemetry-collector:0.100.0 # Use `otel/opentelemetry-collector-contrib` to support more backends. + command: + - "--config=/conf/otel-collector-config.yaml" # Autodiscovery folder changes between versions. Better to provide explicitly. + volumes: + - ./conf/otel-collector-config.yaml:/conf/otel-collector-config.yaml + ports: + - 8888:8888 # Prometheus metrics exposed by the collector + - 8889:8889 # Prometheus exporter metrics + - 13133:13133 # health_check extension + - 4317:4317 # OTLP gRPC receiver + - 4318:4318 # OTLP http receiver + restart: unless-stopped diff --git a/pipeline/conf/otel-collector-config.yaml b/pipeline/conf/otel-collector-config.yaml new file mode 100644 index 0000000..b0dc85f --- /dev/null +++ b/pipeline/conf/otel-collector-config.yaml @@ -0,0 +1,27 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + +processors: + batch: + +exporters: + prometheus: + endpoint: 0.0.0.0:8889 + namespace: default + debug: + verbosity: detailed + +extensions: + health_check: + +service: + extensions: [health_check] + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheus] + From 6091325c89105346f78d0f1debc3190991013ce1 Mon Sep 17 00:00:00 2001 From: Lorenzo Turrino Date: Wed, 22 May 2024 14:58:48 +0100 Subject: [PATCH 4/5] set up prometheus and grafana --- pipeline/compose.yaml | 20 ++++++++++++++++++-- pipeline/conf/grafana-datasources.yaml | 10 ++++++++++ pipeline/conf/prometheus-config.yaml | 11 +++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) create mode 100644 pipeline/conf/grafana-datasources.yaml create mode 100644 pipeline/conf/prometheus-config.yaml diff --git a/pipeline/compose.yaml b/pipeline/compose.yaml index b26f6bf..9830bcc 100644 --- a/pipeline/compose.yaml +++ b/pipeline/compose.yaml @@ -8,8 +8,6 @@ services: OTEL_EXPORTER_OTLP_ENDPOINT: otel-collector:4317 ports: - "5000:5000" - restart: unless-stopped - # OpenTelemetry Collector otel-collector: @@ -25,3 +23,21 @@ services: - 4317:4317 # OTLP gRPC receiver - 4318:4318 # OTLP http receiver restart: unless-stopped + + # Prometheus + prometheus: + image: prom/prometheus:latest + volumes: + - ./conf/prometheus-config.yaml:/etc/prometheus/prometheus.yml + depends_on: + - otel-collector + ports: + - "9090:9090" + + # Grafana + grafana: + image: grafana/grafana:latest + volumes: + - ./conf/grafana-datasources.yaml:/etc/grafana/provisioning/datasources/datasources.yaml + ports: + - "3000:3000" \ No newline at end of file diff --git a/pipeline/conf/grafana-datasources.yaml b/pipeline/conf/grafana-datasources.yaml new file mode 100644 index 0000000..3d8fcb3 --- /dev/null +++ b/pipeline/conf/grafana-datasources.yaml @@ -0,0 +1,10 @@ +datasources: + - name: Prometheus + type: prometheus + uid: prometheus-1 + url: http://prometheus:9090 + access: server + +# deleteDatasources: +# - name: Prometheus +# uid: prometheus-1 \ No newline at end of file diff --git a/pipeline/conf/prometheus-config.yaml b/pipeline/conf/prometheus-config.yaml new file mode 100644 index 0000000..3d1132a --- /dev/null +++ b/pipeline/conf/prometheus-config.yaml @@ -0,0 +1,11 @@ +scrape_configs: + - job_name: "otel-collector-self-reporting" + scrape_interval: 15s + static_configs: + - targets: + - "otel-collector:8888" + - job_name: "otel-collector-exporter" + scrape_interval: 15s + static_configs: + - targets: + - "otel-collector:8889" From bf3fdf6c28f899561df0c235b58b70cfc114432a Mon Sep 17 00:00:00 2001 From: Lorenzo Turrino Date: Wed, 22 May 2024 22:43:56 +0100 Subject: [PATCH 5/5] cleanup --- server/app.py | 6 +- .../apps/instrumentation/instrumentation.py | 10 ++-- server/requirements.txt | 55 +++---------------- 3 files changed, 15 insertions(+), 56 deletions(-) diff --git a/server/app.py b/server/app.py index ae7cd5b..256d9c3 100644 --- a/server/app.py +++ b/server/app.py @@ -41,13 +41,12 @@ def like_video(id): meter_likes.add( 1, { - "user_ip": request.remote_addr, "id": id, }, ) videos.like(id) - return ("OK", 204) + return ("OK", 200) @app.get("/api/v1/video//dislike") @@ -55,13 +54,12 @@ def dislike_video(id): meter_dislikes.add( 1, { - "user_ip": request.remote_addr, "id": id, }, ) videos.dislike(id) - return ("OK", 204) + return ("OK", 200) @app.route("/") diff --git a/server/apps/instrumentation/instrumentation.py b/server/apps/instrumentation/instrumentation.py index b83ad32..60d7f58 100644 --- a/server/apps/instrumentation/instrumentation.py +++ b/server/apps/instrumentation/instrumentation.py @@ -5,19 +5,17 @@ ConsoleMetricExporter, # Useful for debugging - exports the metrics to console PeriodicExportingMetricReader, # Reader that batches metrics in configurable time intervals before sending it to the exporter ) -from opentelemetry.sdk.resources import SERVICE_NAME, Resource def init(): - # Create a unique identifier for this App - resource = Resource(attributes={SERVICE_NAME: "video-voter"}) - # Configure the provider with the exporter and reader - exporter = OTLPMetricExporter(insecure=True) # Endpoint provided via Environment variable - OTEL_EXPORTER_OTLP_ENDPOINT + exporter = OTLPMetricExporter( + insecure=True + ) # Endpoint provided via Environment variable - OTEL_EXPORTER_OTLP_ENDPOINT reader = PeriodicExportingMetricReader( exporter, export_interval_millis=15000, export_timeout_millis=5000 ) - provider = MeterProvider(resource=resource, metric_readers=[reader]) + provider = MeterProvider(metric_readers=[reader]) # Set the global meter provider, and create a Meter for usage metrics.set_meter_provider(provider) diff --git a/server/requirements.txt b/server/requirements.txt index f4487ee..ab59281 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -1,58 +1,21 @@ -appdirs==1.4.4 -asgiref==3.8.1 -black==22.6.0 -certifi==2022.6.15 -charset-normalizer==2.1.1 -click==8.1.3 +blinker==1.8.2 +click==8.1.7 Deprecated==1.2.14 -distlib==0.3.1 -filelock==3.0.12 -Flask==2.2.2 +Flask==3.0.3 googleapis-common-protos==1.63.0 grpcio==1.64.0 -idna==3.3 importlib-metadata==7.0.0 -itsdangerous==2.1.2 -Jinja2==3.1.2 -MarkupSafe==2.1.1 -mypy-extensions==0.4.3 +itsdangerous==2.2.0 +Jinja2==3.1.4 +MarkupSafe==2.1.5 opentelemetry-api==1.24.0 -opentelemetry-distro==0.45b0 -opentelemetry-exporter-otlp==1.24.0 opentelemetry-exporter-otlp-proto-common==1.24.0 opentelemetry-exporter-otlp-proto-grpc==1.24.0 -opentelemetry-exporter-otlp-proto-http==1.24.0 -opentelemetry-instrumentation==0.45b0 -opentelemetry-instrumentation-asgi==0.45b0 -opentelemetry-instrumentation-asyncio==0.45b0 -opentelemetry-instrumentation-aws-lambda==0.45b0 -opentelemetry-instrumentation-dbapi==0.45b0 -opentelemetry-instrumentation-flask==0.45b0 -opentelemetry-instrumentation-grpc==0.45b0 -opentelemetry-instrumentation-jinja2==0.45b0 -opentelemetry-instrumentation-logging==0.45b0 -opentelemetry-instrumentation-requests==0.45b0 -opentelemetry-instrumentation-sqlite3==0.45b0 -opentelemetry-instrumentation-urllib==0.45b0 -opentelemetry-instrumentation-urllib3==0.45b0 -opentelemetry-instrumentation-wsgi==0.45b0 -opentelemetry-propagator-aws-xray==1.0.1 opentelemetry-proto==1.24.0 opentelemetry-sdk==1.24.0 opentelemetry-semantic-conventions==0.45b0 -opentelemetry-test-utils==0.45b0 -opentelemetry-util-http==0.45b0 -packaging==24.0 -pathspec==0.9.0 -platformdirs==2.5.2 protobuf==4.25.3 -python-dotenv==0.20.0 -requests==2.28.1 -six==1.15.0 -tomli==2.0.1 -typing-extensions==4.3.0 -urllib3==1.26.12 -virtualenv==20.2.1 -Werkzeug==2.2.2 +typing_extensions==4.11.0 +Werkzeug==3.0.3 wrapt==1.16.0 -zipp==3.8.1 +zipp==3.18.2