From 5d6572f8ab080e96c0fd3cc1c2ecd75176cb0629 Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Fri, 20 May 2022 23:17:47 +0200 Subject: [PATCH 01/16] Consistently use dashes in Makefile targets --- .github/workflows/main.yml | 18 +++++++++--------- Makefile | 14 +++++++------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bbddf02d..297023bf 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,16 +10,16 @@ env: TEST_VERBOSITY: 2 jobs: - license-check: - name: License check + license-checks: + name: License checks runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v2 - - name: License check - run: make license_check + - name: Run checks + run: make license-checks unit-tests: name: Unit tests (${{ matrix.python-version }}/${{ matrix.os }}) @@ -133,7 +133,7 @@ jobs: TEST_DETECTION_RULES_URI: ${{ matrix.detection-rules-uri }} TEST_SIGNALS_QUERIES: ${{ matrix.signals_queries }} TEST_SIGNALS_RULES: ${{ matrix.signals_rules }} - run: make online_tests + run: make online-tests - name: Stop Elastic Stack ${{ matrix.stack-version }} run: make down @@ -142,7 +142,7 @@ jobs: name: Publish runs-on: ubuntu-latest needs: - - license-check + - license-checks - unit-tests - online-tests if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') @@ -160,13 +160,13 @@ jobs: run: make prereq - name: Build package - run: make pkg_build + run: make pkg-build - name: Install package - run: make pkg_install + run: make pkg-install - name: Try package - run: make pkg_try + run: make pkg-try - name: Publish to TestPyPI uses: pypa/gh-action-pypi-publish@v1.5.0 diff --git a/Makefile b/Makefile index 77537898..b3bd601c 100644 --- a/Makefile +++ b/Makefile @@ -16,7 +16,7 @@ lint: tests: tests/*.py $(PYTHON) -m pytest -raP tests/test_*.py -online_tests: tests/*.py +online-tests: tests/*.py $(PYTHON) -m pytest -raP tests/test_emitter_*.py up: @@ -25,7 +25,7 @@ up: down: docker compose down -license_check: +license-checks: bash scripts/license_check.sh run: @@ -33,17 +33,17 @@ run: $(PYTHON) -m geneve --help $(PYTHON) -m geneve -pkg_build: +pkg-build: $(PYTHON) -m build -pkg_install: +pkg-install: $(PYTHON) -m pip install --force-reinstall dist/geneve-*.whl -pkg_try: +pkg-try: geneve --version geneve --help geneve -package: pkg_build pkg_install pkg_try +package: pkg-build pkg-install pkg-try -.PHONY: lint tests online_tests run up down +.PHONY: lint tests online-tests run up down license-checks package From 1e2945c75bff1c00190b7ae4d234b93ff809aaa8 Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Fri, 20 May 2022 13:11:48 +0200 Subject: [PATCH 02/16] Move Docker Compose test config into tests/ --- .github/workflows/main.yml | 4 ++-- Makefile | 10 +++++----- docker-compose.yml => tests/docker-compose.yml | 0 3 files changed, 7 insertions(+), 7 deletions(-) rename docker-compose.yml => tests/docker-compose.yml (100%) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 297023bf..52264f5f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -125,7 +125,7 @@ jobs: - name: Start Elastic Stack ${{ matrix.stack-version }} env: TEST_STACK_VERSION: ${{ matrix.stack-version }} - run: make up + run: make stack-up - name: Run online tests env: @@ -136,7 +136,7 @@ jobs: run: make online-tests - name: Stop Elastic Stack ${{ matrix.stack-version }} - run: make down + run: make stack-down publish: name: Publish diff --git a/Makefile b/Makefile index b3bd601c..a2ee0c4c 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,11 @@ tests: tests/*.py online-tests: tests/*.py $(PYTHON) -m pytest -raP tests/test_emitter_*.py -up: - docker compose up --wait --quiet-pull +stack-up: + cd tests && docker compose up --wait --quiet-pull -down: - docker compose down +stack-down: + cd tests && docker compose down license-checks: bash scripts/license_check.sh @@ -46,4 +46,4 @@ pkg-try: package: pkg-build pkg-install pkg-try -.PHONY: lint tests online-tests run up down license-checks package +.PHONY: lint tests online-tests run stack-up stack-down license-checks package diff --git a/docker-compose.yml b/tests/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to tests/docker-compose.yml From ef64bd401e8a9e059bc927c7431739715a225d25 Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Thu, 19 May 2022 18:48:10 +0200 Subject: [PATCH 03/16] webapi: add version --- Makefile | 5 ++++- geneve/webapi.py | 29 +++++++++++++++++++++++++++++ requirements.txt | 1 + setup.cfg | 4 ++++ 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 geneve/webapi.py diff --git a/Makefile b/Makefile index a2ee0c4c..ae0a450d 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,9 @@ run: $(PYTHON) -m geneve --help $(PYTHON) -m geneve +flask: + FLASK_APP=geneve/webapi.py $(PYTHON) -m flask run + pkg-build: $(PYTHON) -m build @@ -46,4 +49,4 @@ pkg-try: package: pkg-build pkg-install pkg-try -.PHONY: lint tests online-tests run stack-up stack-down license-checks package +.PHONY: lint tests online-tests run flask stack-up stack-down license-checks package diff --git a/geneve/webapi.py b/geneve/webapi.py new file mode 100644 index 00000000..95f5ea62 --- /dev/null +++ b/geneve/webapi.py @@ -0,0 +1,29 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from . import version + +from flask import Flask, jsonify +app = Flask("geneve") + + +@app.route("/api/v1/version", methods=["GET"]) +def get_version(): + ret = { + "version": version + } + return jsonify(ret) diff --git a/requirements.txt b/requirements.txt index c6d04789..e8ba2e88 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ click eql>=0.9.12 elasticsearch flake8 +flask nbformat pytest pytoml diff --git a/setup.cfg b/setup.cfg index 8dcd0e9c..e06dbc5f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -35,6 +35,10 @@ install_requires = requests python_requires = >=3.8.0 +[options.extras_require] +webapi = + flask + [options.entry_points] console_scripts = geneve = geneve.cli:main From beebdef98ad47a5bd519033352a5b2e70dcfb2c8 Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Fri, 20 May 2022 11:04:15 +0200 Subject: [PATCH 04/16] webapi: add emit --- geneve/webapi.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/geneve/webapi.py b/geneve/webapi.py index 95f5ea62..e8179728 100644 --- a/geneve/webapi.py +++ b/geneve/webapi.py @@ -15,9 +15,12 @@ # specific language governing permissions and limitations # under the License. +from itertools import islice + from . import version +from .events_emitter import SourceEvents -from flask import Flask, jsonify +from flask import Flask, request, jsonify app = Flask("geneve") @@ -27,3 +30,12 @@ def get_version(): "version": version } return jsonify(ret) + + +@app.route("/api/v1/emit", methods=["GET"]) +def emit(): + query = request.args.get("query") + count = int(request.args.get("count", 1)) + se = SourceEvents.from_query(query) + docs = [event.doc for events in islice(se, count) for event in events] + return jsonify(docs) From 68e37b5fdd18e4e43494bb57c16030ac67c1b205 Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Fri, 20 May 2022 16:51:32 +0200 Subject: [PATCH 05/16] Build and check the Docker image --- .github/workflows/main.yml | 23 +++++++++++++++++++++++ Dockerfile | 29 +++++++++++++++++++++++++++++ Makefile | 14 +++++++++++++- 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 Dockerfile diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 52264f5f..a205e28d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -138,6 +138,28 @@ jobs: - name: Stop Elastic Stack ${{ matrix.stack-version }} run: make stack-down + docker-sanity: + name: Docker image sanity + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Install dependencies + run: make prereq + + - name: Build image + run: make docker + + - name: Run sanity checks + run: make docker-sanity + publish: name: Publish runs-on: ubuntu-latest @@ -145,6 +167,7 @@ jobs: - license-checks - unit-tests - online-tests + - docker-sanity if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') steps: diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..c24138b1 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,29 @@ +# syntax=docker/dockerfile:1 + +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +FROM python:alpine +WORKDIR /app + +COPY requirements.txt requirements.txt +RUN pip3 install --user -r requirements.txt + +COPY geneve geneve + +ENV FLASK_APP=geneve/webapi.py +CMD [ "python3", "-m" , "flask", "run", "--host=0.0.0.0", "-p 80" ] diff --git a/Makefile b/Makefile index ae0a450d..09eae28b 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,18 @@ stack-up: stack-down: cd tests && docker compose down +docker: GENEVE_VERSION=$(shell $(PYTHON) -c "import geneve; print(geneve.version)") +docker: + docker build -q -t geneve:$(GENEVE_VERSION) . + +docker-sanity: GENEVE_VERSION=$(shell $(PYTHON) -c "import geneve; print(geneve.version)") +docker-sanity: + docker run -p 127.0.0.1:5000:80 --name geneve-test --rm -d geneve:$(GENEVE_VERSION) + for n in `seq 5`; do \ + [ "`curl -s --fail http://localhost:5000/api/v1/version`" = '{"version":"$(GENEVE_VERSION)"}' ] && exit 0 || sleep 1; \ +done; docker container stop geneve-test; exit 1 + docker container stop geneve-test + license-checks: bash scripts/license_check.sh @@ -49,4 +61,4 @@ pkg-try: package: pkg-build pkg-install pkg-try -.PHONY: lint tests online-tests run flask stack-up stack-down license-checks package +.PHONY: lint tests online-tests run flask stack-up stack-down license-checks package docker docker-sanity From 4e1c5ff967a5e3124c2b6cc341c66545dea9d7ee Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Fri, 20 May 2022 23:57:42 +0200 Subject: [PATCH 06/16] CI: Fix Docker stack pull Fix: 1e2945c75bff1c00190b7ae4d234b93ff809aaa8. --- .github/workflows/main.yml | 2 +- Makefile | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a205e28d..de456441 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -116,7 +116,7 @@ jobs: TEST_STACK_VERSION: ${{ matrix.stack-version }} run: | mkdir ~/elastic-stack-cache - docker compose pull -q + make stack-pull docker save -o ~/elastic-stack-cache/elasticsearch-${{ matrix.stack-version }}.tar \ docker.elastic.co/elasticsearch/elasticsearch:${{ matrix.stack-version }} docker save -o ~/elastic-stack-cache/kibana-${{ matrix.stack-version }}.tar \ diff --git a/Makefile b/Makefile index 09eae28b..07c7cb85 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,9 @@ tests: tests/*.py online-tests: tests/*.py $(PYTHON) -m pytest -raP tests/test_emitter_*.py +stack-pull: + cd tests && docker compose pull -q + stack-up: cd tests && docker compose up --wait --quiet-pull From b7142b5ccda5d4aa69db194fd41d0d78d18c6ad0 Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Fri, 20 May 2022 23:53:35 +0200 Subject: [PATCH 07/16] Add package sanity checks --- .github/workflows/main.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index de456441..5549863e 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -160,6 +160,31 @@ jobs: - name: Run sanity checks run: make docker-sanity + package-sanity: + name: Python package sanity + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@v2 + + - name: Setup Python + uses: actions/setup-python@v2 + with: + python-version: "3.8" + + - name: Install dependencies + run: make prereq + + - name: Build package + run: make pkg-build + + - name: Install package + run: make pkg-install + + - name: Try package + run: make pkg-try + publish: name: Publish runs-on: ubuntu-latest @@ -168,6 +193,7 @@ jobs: - unit-tests - online-tests - docker-sanity + - package-sanity if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') steps: From ad41b59cd311cd91c4216ff75b358dfa8dfb468e Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Sat, 21 May 2022 00:16:43 +0200 Subject: [PATCH 08/16] Add missing geneve.utils to the package --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index e06dbc5f..fa849a04 100644 --- a/setup.cfg +++ b/setup.cfg @@ -27,6 +27,7 @@ license_file = LICENSE.txt packages = geneve geneve.kql + geneve.utils install_requires = click eql>=0.9.12 From 83bca9e89add376f0eb1f370b161b2e1580e3002 Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Sun, 22 May 2022 09:57:38 +0200 Subject: [PATCH 09/16] Add `make docker-push` --- Makefile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 07c7cb85..fd533836 100644 --- a/Makefile +++ b/Makefile @@ -28,18 +28,25 @@ stack-up: stack-down: cd tests && docker compose down -docker: GENEVE_VERSION=$(shell $(PYTHON) -c "import geneve; print(geneve.version)") docker: - docker build -q -t geneve:$(GENEVE_VERSION) . + docker build -q -t geneve . docker-sanity: GENEVE_VERSION=$(shell $(PYTHON) -c "import geneve; print(geneve.version)") docker-sanity: - docker run -p 127.0.0.1:5000:80 --name geneve-test --rm -d geneve:$(GENEVE_VERSION) + docker run -p 127.0.0.1:5000:80 --name geneve-test --rm -d geneve for n in `seq 5`; do \ [ "`curl -s --fail http://localhost:5000/api/v1/version`" = '{"version":"$(GENEVE_VERSION)"}' ] && exit 0 || sleep 1; \ done; docker container stop geneve-test; exit 1 docker container stop geneve-test +docker-push: GENEVE_VERSION=$(shell $(PYTHON) -c "import geneve; print(geneve.version)") +docker-push: + docker tag geneve:latest $(DOCKER_REGISTRY)/geneve:latest + docker tag geneve:latest $(DOCKER_REGISTRY)/geneve:$(GENEVE_VERSION) + docker push -q $(DOCKER_REGISTRY)/geneve:latest + docker push -q $(DOCKER_REGISTRY)/geneve:$(GENEVE_VERSION) + docker image rm $(DOCKER_REGISTRY)/geneve:latest $(DOCKER_REGISTRY)/geneve:$(GENEVE_VERSION) + license-checks: bash scripts/license_check.sh From 50e1e644d329d993c9749fb9887daad211a810ad Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Mon, 23 May 2022 14:46:17 +0200 Subject: [PATCH 10/16] Add `make docker-run` --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index fd533836..f06c7de8 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,9 @@ stack-down: docker: docker build -q -t geneve . +docker-run: + docker run -p 127.0.0.1:5000:80 --name geneve --rm geneve + docker-sanity: GENEVE_VERSION=$(shell $(PYTHON) -c "import geneve; print(geneve.version)") docker-sanity: docker run -p 127.0.0.1:5000:80 --name geneve-test --rm -d geneve From 4aa5631f99d7f0637e8a090033cf62f03cca5519 Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Mon, 23 May 2022 18:18:04 +0200 Subject: [PATCH 11/16] `make docker` -> `make docker-build` --- .github/workflows/main.yml | 2 +- Makefile | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5549863e..c61f71fa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -155,7 +155,7 @@ jobs: run: make prereq - name: Build image - run: make docker + run: make docker-build - name: Run sanity checks run: make docker-sanity diff --git a/Makefile b/Makefile index f06c7de8..3b64b8ac 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,8 @@ stack-up: stack-down: cd tests && docker compose down -docker: +docker-build: + -docker image rm geneve docker build -q -t geneve . docker-run: From 97e8cd9541b5eafd0af42e86a890239c3266fc30 Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Tue, 24 May 2022 12:25:59 +0200 Subject: [PATCH 12/16] Switch back to a high port --- Dockerfile | 4 +++- Makefile | 6 +++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index c24138b1..a6f860bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,5 +25,7 @@ RUN pip3 install --user -r requirements.txt COPY geneve geneve +EXPOSE 5000 + ENV FLASK_APP=geneve/webapi.py -CMD [ "python3", "-m" , "flask", "run", "--host=0.0.0.0", "-p 80" ] +CMD [ "python3", "-m" , "flask", "run", "--host=0.0.0.0", "-p 5000" ] diff --git a/Makefile b/Makefile index 3b64b8ac..cfacf791 100644 --- a/Makefile +++ b/Makefile @@ -33,12 +33,12 @@ docker-build: docker build -q -t geneve . docker-run: - docker run -p 127.0.0.1:5000:80 --name geneve --rm geneve + docker run -p 127.0.0.1:5000:5000 --rm --name geneve geneve docker-sanity: GENEVE_VERSION=$(shell $(PYTHON) -c "import geneve; print(geneve.version)") docker-sanity: - docker run -p 127.0.0.1:5000:80 --name geneve-test --rm -d geneve - for n in `seq 5`; do \ + docker run -p 127.0.0.1:5000:5000 --rm --name geneve-test -d geneve + for n in `seq 30`; do \ [ "`curl -s --fail http://localhost:5000/api/v1/version`" = '{"version":"$(GENEVE_VERSION)"}' ] && exit 0 || sleep 1; \ done; docker container stop geneve-test; exit 1 docker container stop geneve-test From 29b8d9ba2b692e625d73faaf7f2078dcef5643bc Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Mon, 23 May 2022 15:16:55 +0200 Subject: [PATCH 13/16] web app: load schema and rules, new endpoints, logging, etc --- Dockerfile | 1 + geneve/webapi.py | 59 ++++++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index a6f860bc..3bd31658 100644 --- a/Dockerfile +++ b/Dockerfile @@ -23,6 +23,7 @@ WORKDIR /app COPY requirements.txt requirements.txt RUN pip3 install --user -r requirements.txt +COPY etc etc COPY geneve geneve EXPOSE 5000 diff --git a/geneve/webapi.py b/geneve/webapi.py index e8179728..c701ed9c 100644 --- a/geneve/webapi.py +++ b/geneve/webapi.py @@ -15,13 +15,51 @@ # specific language governing permissions and limitations # under the License. +import sys +import logging from itertools import islice from . import version from .events_emitter import SourceEvents +from .utils import root_dir, load_schema, load_rules from flask import Flask, request, jsonify app = Flask("geneve") +app.config.from_prefixed_env("GENEVE") + +logging.basicConfig(level=logging.DEBUG) + +rule_tags = app.config.get("RULE_TAGS", "") +if rule_tags: + rule_tags = set(x.strip().lower() for x in rule_tags.split(",") if x.strip()) + if rule_tags: + app.logger.info("Rule tags: {}".format(", ".join(sorted(rule_tags)))) + +schema_uri = app.config.get("SCHEMA_URI", "./etc/ecs-8.1.0.tar.gz") +app.logger.debug(f"Loading {schema_uri}...") +schema = load_schema(schema_uri, "generated/ecs/ecs_flat.yml", root_dir) + +detection_rules_uri = app.config.get("DETECTION_RULES_URI", "./etc/detection-rules-8.1.0.tar.gz") +app.logger.debug(f"Loading {detection_rules_uri}...") +rules = load_rules(detection_rules_uri, "rules/**/*.toml", root_dir) + +source_events = SourceEvents(schema) +loaded_rules = [] +for rule in rules: + if not rule_tags or rule_tags.issubset(x.lower() for x in rule.tags): + try: + source_events.add_rule(rule) + loaded_rules.append(rule) + rule.path = str(rule.path) + except Exception as e: + app.logger.warning(f"{e}: {rule.path}") + continue + +if not source_events: + app.logger.error(f"Examined {len(rules)} rules, none was loaded.") + sys.exit(1) + +app.logger.info(f"Loaded {len(source_events)} rules") @app.route("/api/v1/version", methods=["GET"]) @@ -32,10 +70,23 @@ def get_version(): return jsonify(ret) +@app.route("/api/v1/rules", methods=["GET"]) +def get_rules(): + return jsonify([vars(x) for x in loaded_rules]) + + +@app.route("/api/v1/query", methods=["GET"]) +def query(): + query = request.args.get("query") + count = request.args.get("count", default=1, type=int) + source_events = SourceEvents(schema) + source_events.add_query(query) + docs = [event.doc for events in islice(source_events, count) for event in events] + return jsonify(docs) + + @app.route("/api/v1/emit", methods=["GET"]) def emit(): - query = request.args.get("query") - count = int(request.args.get("count", 1)) - se = SourceEvents.from_query(query) - docs = [event.doc for events in islice(se, count) for event in events] + count = request.args.get("count", default=1, type=int) + docs = [event.doc for events in islice(source_events, count) for event in events] return jsonify(docs) From 2f98d1f6bdb1e7847de58996f188eebe58651534 Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Tue, 24 May 2022 15:07:46 +0200 Subject: [PATCH 14/16] Add `make kind-up` and `make kind-down` targets --- .github/workflows/main.yml | 41 +++++++++++++++++++++++++++++++++++++- Makefile | 9 +++++++++ etc/kind-config.yml | 9 +++++++++ etc/pods/geneve.yml | 11 ++++++++++ etc/services/geneve.yml | 11 ++++++++++ 5 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 etc/kind-config.yml create mode 100644 etc/pods/geneve.yml create mode 100644 etc/services/geneve.yml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c61f71fa..66a5d34d 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -185,6 +185,45 @@ jobs: - name: Try package run: make pkg-try + kubernetes-sanity: + name: Kubernetes sanity + runs-on: ubuntu-latest + needs: + - docker-sanity + + steps: + - name: Checkout code + uses: actions/checkout@v1 + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: ">=1.16" + + - name: Install Kind + run: | + go install sigs.k8s.io/kind@latest + kind version + + - name: Build image + run: make docker-build + + - name: Create cluster + run: | + make kind-up + kubectl get nodes -o wide + kubectl get pods -o wide + kubectl get services -o wide + + - name: Run sanity checks + run: | + for n in `seq 30`; do \ + curl -s --fail http://localhost:30000/api/v1/version && exit 0 || sleep 1; \ + done; exit 1 + + - name: Destroy Kind cluster + run: make kind-down + publish: name: Publish runs-on: ubuntu-latest @@ -192,7 +231,7 @@ jobs: - license-checks - unit-tests - online-tests - - docker-sanity + - kubernetes-sanity - package-sanity if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags') diff --git a/Makefile b/Makefile index cfacf791..2b6bb405 100644 --- a/Makefile +++ b/Makefile @@ -51,6 +51,15 @@ docker-push: docker push -q $(DOCKER_REGISTRY)/geneve:$(GENEVE_VERSION) docker image rm $(DOCKER_REGISTRY)/geneve:latest $(DOCKER_REGISTRY)/geneve:$(GENEVE_VERSION) +kind-up: + kind create cluster --config=etc/kind-config.yml + kind load docker-image geneve + kubectl apply -f etc/pods/geneve.yml + kubectl apply -f etc/services/geneve.yml + +kind-down: + kind delete cluster + license-checks: bash scripts/license_check.sh diff --git a/etc/kind-config.yml b/etc/kind-config.yml new file mode 100644 index 00000000..e3af0f7d --- /dev/null +++ b/etc/kind-config.yml @@ -0,0 +1,9 @@ +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +nodes: + - role: control-plane + extraPortMappings: + - containerPort: 30000 + hostPort: 30000 + - role: worker + - role: worker diff --git a/etc/pods/geneve.yml b/etc/pods/geneve.yml new file mode 100644 index 00000000..1f6d6784 --- /dev/null +++ b/etc/pods/geneve.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: geneve + labels: + app: geneve +spec: + containers: + - name: geneve + image: geneve + imagePullPolicy: Never diff --git a/etc/services/geneve.yml b/etc/services/geneve.yml new file mode 100644 index 00000000..8e75d1ab --- /dev/null +++ b/etc/services/geneve.yml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Service +metadata: + name: geneve +spec: + type: NodePort + selector: + app: geneve + ports: + - port: 5000 + nodePort: 30000 From 809d67be96d85c3b0105becfa6454ea51cc7d58a Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Wed, 25 May 2022 18:06:06 +0200 Subject: [PATCH 15/16] Share sanity checks between Docker and Kind --- .github/workflows/main.yml | 5 +---- Makefile | 14 +++++++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 66a5d34d..23d9df07 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -216,10 +216,7 @@ jobs: kubectl get services -o wide - name: Run sanity checks - run: | - for n in `seq 30`; do \ - curl -s --fail http://localhost:30000/api/v1/version && exit 0 || sleep 1; \ - done; exit 1 + run: make sanity-checks - name: Destroy Kind cluster run: make kind-down diff --git a/Makefile b/Makefile index 2b6bb405..ac8653aa 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,11 @@ tests: tests/*.py online-tests: tests/*.py $(PYTHON) -m pytest -raP tests/test_emitter_*.py +sanity-checks: + for n in `seq 30`; do \ + curl -s --fail http://localhost:30000/api/v1/version && exit 0 || sleep 1; \ +done; exit 1 + stack-pull: cd tests && docker compose pull -q @@ -33,14 +38,13 @@ docker-build: docker build -q -t geneve . docker-run: - docker run -p 127.0.0.1:5000:5000 --rm --name geneve geneve + docker run -p 127.0.0.1:30000:5000 --rm --name geneve geneve docker-sanity: GENEVE_VERSION=$(shell $(PYTHON) -c "import geneve; print(geneve.version)") docker-sanity: - docker run -p 127.0.0.1:5000:5000 --rm --name geneve-test -d geneve - for n in `seq 30`; do \ - [ "`curl -s --fail http://localhost:5000/api/v1/version`" = '{"version":"$(GENEVE_VERSION)"}' ] && exit 0 || sleep 1; \ -done; docker container stop geneve-test; exit 1 + docker run -p 127.0.0.1:30000:5000 --rm --name geneve-test -d geneve + [ "`$(MAKE) -s sanity-checks`" = '{"version":"$(GENEVE_VERSION)"}' ] || \ + (docker container stop geneve-test; exit 1) docker container stop geneve-test docker-push: GENEVE_VERSION=$(shell $(PYTHON) -c "import geneve; print(geneve.version)") From 2219409ea5f50f80af7f24ecfcea4df05c0cc497 Mon Sep 17 00:00:00 2001 From: Domenico Andreoli Date: Thu, 26 May 2022 19:34:31 +0200 Subject: [PATCH 16/16] Run k8s sanity also on macos and windows --- .github/workflows/main.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 23d9df07..fa637a58 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -187,10 +187,15 @@ jobs: kubernetes-sanity: name: Kubernetes sanity - runs-on: ubuntu-latest + runs-on: ${{ matrix.os }} needs: - docker-sanity + strategy: + fail-fast: false + matrix: + os: ["ubuntu-latest", "macos-latest", "windows-latest"] + steps: - name: Checkout code uses: actions/checkout@v1