E2e add kafka fix

.github/workflows/pytest-tests.yml

@@ -0,0 +1,42 @@
+name: Pytest Tests
+
+env:
+  OPAL_PYTEST_REPO_OWNER: ${{vars.OPAL_PYTEST_REPO_OWNER}}
+  OPAL_PYTEST_REPO_NAME: ${{vars.OPAL_PYTEST_REPO_NAME}}
+  OPAL_PYTEST_REPO_PASSWORD: ${{vars.OPAL_PYTEST_REPO_PASSWORD}}
+  OPAL_PYTEST_SSH_KEY_PATH: ${{vars.OPAL_PYTEST_SSH_KEY_PATH}}
+  OPAL_PYTEST_SOURCE_ACCOUNT: ${{vars.OPAL_PYTEST_SOURCE_ACCOUNT}}
+  OPAL_PYTEST_SOURCE_REPO: ${{vars.OPAL_PYTEST_SOURCE_REPO}}
+  OPAL_PYTEST_WEBHOOK_SECRET: ${{vars.OPAL_PYTEST_WEBHOOK_SECRET}}
+  OPAL_PYTEST_SHOULD_FORK: ${{vars.OPAL_PYTEST_SHOULD_FORK}}
+  OPAL_PYTEST_USE_WEBHOOK: ${{vars.OPAL_PYTEST_USE_WEBHOOK}}
+  OPAL_PYTEST_WAIT_FOR_DEBUGGER: ${{vars.OPAL_PYTEST_WAIT_FOR_DEBUGGER}}
+  OPAL_PYTEST_DO_NOT_BUILD_IMAGES: ${{vars.OPAL_PYTEST_DO_NOT_BUILD_IMAGES}}
+  OPAL_PYTEST_SKIP_REBUILD_IMAGES: ${{vars.OPAL_PYTEST_SKIP_REBUILD_IMAGES}}
+  OPAL_PYTEST_KEEP_IMAGES: ${{vars.OPAL_PYTEST_KEEP_IMAGES}}
+  OPAL_PYTEST_GITHUB_PAT: ${{vars.OPAL_PYTEST_GITHUB_PAT}}
+  OPAL_PYTEST_POLICY_REPO_SSH_PRIVATE_KEY: ${{vars.OPAL_PYTEST_POLICY_REPO_SSH_PRIVATE_KEY}}
+  OPAL_PYTEST_POLICY_REPO_SSH_PUBLIC_KEY: ${{vars.OPAL_PYTEST_POLICY_REPO_SSH_PUBLIC_KEY}}
+  OPAL_AUTH_PRIVATE_KEY: ${{vars.OPAL_AUTH_PRIVATE_KEY}}
+  OPAL_AUTH_PUBLIC_KEY: ${{vars.OPAL_AUTH_PUBLIC_KEY}}
+
+on:
+  push:
+    branches:
+      - e2e-add-kafka-fix
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    environment: OPAL_pytest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Install Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.10.12"
+      - name: Install dependencies
+        run: pip install -r requirements.txt && pip install -r ./tests/requirements.txt
+      - name: Run tests
+        working-directory: ./tests
+        run: ./run.sh

app-tests/minrun.sh

@@ -64,7 +64,7 @@ else
   else
     echo "Error creating fork: $?"
   fi
-  
+
   # Update OPAL_POLICY_REPO_URL to point to the forked repo
   OPAL_POLICY_REPO_URL="$FORKED_REPO_URL"
   echo "Updated OPAL_POLICY_REPO_URL to $OPAL_POLICY_REPO_URL"

app-tests/sample_service/Dockerfile

@@ -41,4 +41,4 @@ RUN mkdir -p /var/log/nginx && \
 # Run both OpenResty and Flask
 COPY start.sh /start.sh
 RUN chmod +x /start.sh
-CMD /start.sh
+CMD /start.sh

app-tests/sample_service/app.py

@@ -1,29 +1,34 @@
-from flask import Flask, request, jsonify
-import requests
 import debugpy
+import requests
+from flask import Flask, jsonify, request
 
 app = Flask(__name__)
 
 debugpy.listen(("0.0.0.0", 5682))  # Optional, listen for debug requests on port 5678
 
 # OPAL Authorization endpoint
-OPAL_AUTH_URL = "http://opal_client:8181/v1/data/authorize"  # Adjust with actual OPAL endpoint
+OPAL_AUTH_URL = (
+    "http://opal_client:8181/v1/data/authorize"  # Adjust with actual OPAL endpoint
+)
 
-@app.route('/a')
+
+@app.route("/a")
 def a():
-    return 'Endpoint A'
+    return "Endpoint A"
+
 
-@app.route('/b')
+@app.route("/b")
 def b():
-    return 'Endpoint B'
+    return "Endpoint B"
 
-@app.route('/c')
+
+@app.route("/c")
 def c():
     # Assuming the JWT token is passed in the Authorization header
-    auth_header = request.headers.get('Authorization')
+    auth_header = request.headers.get("Authorization")
 
     debugpy.wait_for_client()
-    
+
     if not auth_header:
         return jsonify({"error": "Unauthorized, missing Authorization header"}), 401
 
@@ -46,13 +51,7 @@ def c():
         return jsonify({"error": "Unauthorized, 'sub' field not found in token"}), 401
 
     # Prepare the payload for the OPAL authorization request with the extracted user
-    payload = {
-        "input": {
-            "user": user,
-            "method": request.method,
-            "path": request.path
-        }
-    }
+    payload = {"input": {"user": user, "method": request.method, "path": request.path}}
 
     # Send the request to OPAL authorization endpoint
     try:
@@ -62,19 +61,27 @@ def c():
         if response.status_code == 200:
             opal_response = response.json()
             if opal_response.get("result") is True:
-                return 'Endpoint C - Authorized'  # Authorized access
+                return "Endpoint C - Authorized"  # Authorized access
 
             # If the result is not `true`, deny access
-            
+
             # Assuming `response` is your variable containing the response object from OPAL
-            response_data = response.get_data(as_text=True) 
-            return jsonify({"error": f"Forbidden, authorization denied! \n Response Body: {response_data}"}), 403
+            response_data = response.get_data(as_text=True)
+            return (
+                jsonify(
+                    {
+                        "error": f"Forbidden, authorization denied! \n Response Body: {response_data}"
+                    }
+                ),
+                403,
+            )
         # OPAL responded but with a non-200 status, treat as denied
         return jsonify({"error": "Forbidden, OPAL authorization failed"}), 403
 
     except requests.exceptions.RequestException as e:
         # Handle connection or other request errors
         return jsonify({"error": f"Error contacting OPAL client: {str(e)}"}), 500
 
-if __name__ == '__main__':
-    app.run()
+
+if __name__ == "__main__":
+    app.run()

app-tests/sample_service/nginx.conf

@@ -17,13 +17,13 @@ http {
             access_log /var/log/nginx/proxy_access.log;
 
             # Directly proxy to Flask without authorization
-            proxy_pass http://127.0.0.1:5000;   
+            proxy_pass http://127.0.0.1:5000;
         }
 
         # This will be enforced in the endpoint
         location /c {
             access_log /var/log/nginx/proxy_access.log;
-            
+
             proxy_pass http://127.0.0.1:5000;
         }
 
@@ -106,4 +106,4 @@ http {
             return 401 "Unauthorized";
         }
     }
-}
+}

app-tests/sample_service/openapi.yaml

@@ -37,7 +37,7 @@ paths:
     get:
       summary: Endpoint C with Authorization
       description: |
-        This endpoint requires authorization. The client must provide a JWT token in the Authorization header. 
+        This endpoint requires authorization. The client must provide a JWT token in the Authorization header.
         The endpoint checks with an OPAL server to authorize the user based on the token.
       security:
         - bearerAuth: []
@@ -85,4 +85,4 @@ components:
     bearerAuth:
       type: http
       scheme: bearer
-      bearerFormat: JWT  # Indicates the use of JWT for bearer token
+      bearerFormat: JWT  # Indicates the use of JWT for bearer token

app-tests/sample_service/policy.rego

@@ -34,4 +34,4 @@ allow = true {
     input.path = ["c"]
     input.method = "GET"
     user_role == "writer" or user_role == "reader"
-}
+}

app-tests/sample_service/supervisord.conf

@@ -5,4 +5,4 @@ nodaemon=true
 command=nginx -g "daemon off;"
 
 [program:flask]
-command=flask run --host=0.0.0.0 --port=5000
+command=flask run --host=0.0.0.0 --port=5000

docker/Dockerfile.client

@@ -61,10 +61,10 @@ CMD ["./start.sh"]
     # uvicorn config ------------------------------------
     # install the opal-client package
     RUN cd ./packages/opal-client && python setup.py install
-    
+
     # WARNING: do not change the number of workers on the opal client!
     # only one worker is currently supported for the client.
-    
+
     # number of uvicorn workers
     ENV UVICORN_NUM_WORKERS=1
     # uvicorn asgi app
@@ -73,46 +73,46 @@ CMD ["./start.sh"]
     ENV UVICORN_PORT=7000
     # disable inline OPA
     ENV OPAL_INLINE_OPA_ENABLED=false
-    
+
     # expose opal client port
     EXPOSE 7000
     USER opal
-    
+
     RUN mkdir -p /opal/backup
     VOLUME /opal/backup
-    
-    
+
+
     # IMAGE to extract OPA from official image ----------
     # ---------------------------------------------------
     FROM alpine:latest AS opa-extractor
     USER root
-    
+
     RUN apk update && apk add skopeo tar
     WORKDIR /opal
-    
+
     # copy opa from official docker image
     ARG opa_image=openpolicyagent/opa
     ARG opa_tag=latest-static
     RUN skopeo copy "docker://${opa_image}:${opa_tag}" docker-archive:./image.tar && \
       mkdir image && tar xf image.tar -C ./image && cat image/*.tar | tar xf - -C ./image -i && \
       find image/ -name "opa*" -type f -executable -print0 | xargs -0 -I "{}" cp {} ./opa && chmod 755 ./opa && \
       rm -r image image.tar
-    
-    
+
+
     # OPA CLIENT IMAGE ----------------------------------
     # Using standalone image as base --------------------
     # ---------------------------------------------------
     FROM client-standalone AS client
-    
+
     # Temporarily move back to root for additional setup
     USER root
-    
+
     # copy opa from opa-extractor
     COPY --from=opa-extractor /opal/opa ./opa
-    
+
     # enable inline OPA
     ENV OPAL_INLINE_OPA_ENABLED=true
     # expose opa port
     EXPOSE 8181
-    
-    USER opal
+
+    USER opal

docker/docker-compose-local.yml

@@ -102,4 +102,4 @@ services:
 
 volumes:
   opa_backup:
-  gitea_data:  # Data volume for Gitea
+  gitea_data:  # Data volume for Gitea

packages/opal-client/opal_client/main.py

@@ -3,9 +3,10 @@
 client = OpalClient()
 
 import debugpy
-#debugpy.listen(("0.0.0.0", 5678))
+
+# debugpy.listen(("0.0.0.0", 5678))
 print("Waiting for debugger attach...")
-#debugpy.wait_for_client()  # Optional, wait for debugger to attach before continuing
+# debugpy.wait_for_client()  # Optional, wait for debugger to attach before continuing
 
 # expose app for Uvicorn
 app = client.app

packages/opal-server/opal_server/main.py

@@ -1,8 +1,9 @@
-
 import debugpy
-#debugpy.listen(("0.0.0.0", 5678))
+
+# debugpy.listen(("0.0.0.0", 5678))
 print("Waiting for debugger attach...")
-#debugpy.wait_for_client()  # Optional, wait for debugger to attach before continuing
+# debugpy.wait_for_client()  # Optional, wait for debugger to attach before continuing
+
 
 def create_app(*args, **kwargs):
     from opal_server.server import OpalServer

tests/.env.example

@@ -1,5 +1,21 @@
 #!/usr/bin/env bash
 
-export OPAL_POLICY_REPO_URL=''
-export POLICY_REPO_BRANCH=''
-export OPAL_POLICY_REPO_SSH_KEY=''
+OPAL_PYTEST_REPO_OWNER=
+OPAL_PYTEST_REPO_NAME=
+OPAL_PYTEST_REPO_PASSWORD=
+OPAL_PYTEST_SSH_KEY_PATH=
+OPAL_PYTEST_SOURCE_ACCOUNT=
+OPAL_PYTEST_SOURCE_REPO=
+OPAL_PYTEST_WEBHOOK_SECRET=
+OPAL_PYTEST_SHOULD_FORK=
+OPAL_PYTEST_USE_WEBHOOK=
+OPAL_PYTEST_WAIT_FOR_DEBUGGER=
+OPAL_PYTEST_DO_NOT_BUILD_IMAGES=
+OPAL_PYTEST_SKIP_REBUILD_IMAGES=
+OPAL_PYTEST_KEEP_IMAGES=
+OPAL_PYTEST_GITHUB_PAT=
+OPAL_PYTEST_POLICY_REPO_SSH_PRIVATE_KEY=
+OPAL_PYTEST_POLICY_REPO_SSH_PUBLIC_KEY=
+
+OPAL_AUTH_PRIVATE_KEY=
+OPAL_AUTH_PUBLIC_KEY=

tests/README.md

@@ -79,4 +79,4 @@ Refer to the [OPAL API Documentation](https://opal-v2.permit.io/redoc#tag/Bundle
 
 ---
 
-Let me know if you'd like to include specific code examples or any other details!
+Let me know if you'd like to include specific code examples or any other details!