diff --git a/.devcontainer/Dockerfile.dev b/.devcontainer/Dockerfile.dev new file mode 100644 index 00000000..b0cd2ba9 --- /dev/null +++ b/.devcontainer/Dockerfile.dev @@ -0,0 +1,45 @@ +FROM python:3.12-slim-trixie + +ENV INSTALL_PATH /opt/certego +RUN mkdir -p $INSTALL_PATH +WORKDIR $INSTALL_PATH + +# Install system dependencies +RUN sed -i 's/main/main non-free/g' /etc/apt/sources.list.d/debian.sources \ + && apt-get update \ + && apt-get -y install \ + gcc \ + curl \ + git \ + vim \ + && rm -rf /var/lib/apt/lists/* + +# Copy requirements and install Python packages +COPY buffalogs/ /opt/certego/buffalogs/ +RUN pip install --no-cache-dir -r buffalogs/requirements.txt -r buffalogs/requirements_dev.txt + +# Install additional development packages +RUN pip install --no-cache-dir \ + django-debug-toolbar \ + django-extensions \ + ipython \ + pytest-django \ + pytest-cov \ + black==25.1.0 \ + pylint==3.3.5 \ + flake8==7.1.2 \ + debugpy + +RUN chmod +x /opt/certego/buffalogs/run.sh +RUN chmod +x /opt/certego/buffalogs/run_worker.sh +RUN chmod +x /opt/certego/buffalogs/run_beat.sh + +WORKDIR /opt/certego/buffalogs/ + +# Create socket directory for uwsgi +RUN mkdir -p /var/run/nginx-sockets + +# Expose port for development server +EXPOSE 8000 + +CMD ["sleep", "infinity"] \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..31ccf35e --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,70 @@ +{ + "name": "BuffaLogs Development", + "dockerComposeFile": "docker-compose.yaml", + "service": "buffalogs", + "workspaceFolder": "/opt/certego/buffalogs", + // Stop all running containers on close + "shutdownAction": "stopCompose", + "remoteUser": "root", + "postCreateCommand": "python manage.py migrate", + "forwardPorts": [ + 8000, + 5433, + 5672, + 15672, + 5679 + ], + "portsAttributes": { + "8000": { + "label": "Django Dev Server", + "onAutoForward": "notify" + }, + "5433": { + "label": "PostgreSQL", + "onAutoForward": "notify" + }, + "5672": { + "label": "RabbitMQ", + "onAutoForward": "silent" + }, + "15672": { + "label": "RabbitMQ Management", + "onAutoForward": "notify" + }, + "5679": { + "label": "Celery Worker", + "onAutoForward": "notify" + } + }, + "customizations": { + "vscode": { + "settings": { + "python.defaultInterpreterPath": "/usr/local/bin/python", + "python.linting.enabled": true, + "python.linting.pylintEnabled": true, + "python.formatting.provider": "black", + "python.testing.pytestEnabled": true, + "python.testing.unittestEnabled": false, + "files.exclude": { + "**/__pycache__": true, + "**/*.pyc": true + }, + "django.debugJustMyCode": false + }, + "extensions": [ + "ms-python.python", + "ms-python.debugpy", + "ms-python.black-formatter", + "ms-python.pylint", + "batisteo.vscode-django", + "ms-vscode.vscode-json", + "redhat.vscode-yaml" + ] + } + }, + "features": { + "ghcr.io/devcontainers/features/git:1": {}, + "ghcr.io/devcontainers/features/github-cli:1": {} + }, + "overrideCommand": false +} \ No newline at end of file diff --git a/.devcontainer/docker-compose.yaml b/.devcontainer/docker-compose.yaml new file mode 100644 index 00000000..c125d28d --- /dev/null +++ b/.devcontainer/docker-compose.yaml @@ -0,0 +1,97 @@ +services: + # Development override - extends original docker-compose.yaml + buffalogs_postgres: + extends: + file: ../docker-compose.yaml + service: buffalogs_postgres + volumes: + - buffalogs_postgres_data:/var/lib/postgresql/data + ports: + - "5433:5432" + + buffalogs_rabbitmq: + extends: + file: ../docker-compose.yaml + service: buffalogs_rabbitmq + ports: + - "5672:5672" + - "15672:15672" + + + + # Override buffalogs service for development + buffalogs: + extends: + file: ../docker-compose.yaml + service: buffalogs + build: + context: .. + dockerfile: .devcontainer/Dockerfile.dev + volumes: + - ../buffalogs:/opt/certego/buffalogs + - ../config:/opt/certego/config:ro + - ../.vscode:/opt/certego/buffalogs/.vscode + - buffalogs_django_static:/var/www + - buffalogs_nginx_sockets:/var/run/nginx-sockets + # For development, we need to expose port 8000 for Django runserver and 5678 for debugpy + ports: + - "8000:8000" + - "5678:5678" + environment: + - DEBUG=1 + - CERTEGO_BUFFALOGS_DEBUG=True + - USE_DEBUGPY=1 + command: python -m debugpy --listen 0.0.0.0:5678 manage.py runserver 0.0.0.0:8000 + depends_on: + buffalogs_postgres: + condition: service_healthy + buffalogs_rabbitmq: + condition: service_healthy + + # Development Celery worker + buffalogs_celery: + extends: + file: ../docker-compose.yaml + service: buffalogs_celery + build: + context: .. + dockerfile: .devcontainer/Dockerfile.dev + volumes: + - ../buffalogs:/opt/certego/buffalogs + - ../config:/opt/certego/config:ro + ports: + - "5679:5679" + command: python -m debugpy --listen 0.0.0.0:5679 --wait-for-client -m celery -A buffalogs worker -c 1 --loglevel=info + environment: + - DEBUG=1 + + # Development Celery beat + buffalogs_celery_beat: + extends: + file: ../docker-compose.yaml + service: buffalogs_celery_beat + build: + context: .. + dockerfile: .devcontainer/Dockerfile.dev + volumes: + - ../buffalogs:/opt/certego/buffalogs + - ../config:/opt/certego/config:ro + environment: + - DEBUG=1 + + + +volumes: + buffalogs_postgres_data: + driver: local + buffalogs_nginx_sockets: + driver: local + buffalogs_django_static: + driver: local + buffalogs_rabbitmq_data: + driver: local + +networks: + buffalogs_network: + driver: bridge + name: buffalogs_network \ No newline at end of file diff --git a/README.md b/README.md index 4766fbef..f26a9b7d 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,11 @@ git clone git@github.com:certego/BuffaLogs.git ``` Or download the application directly from the [Docker Hub](https://hub.docker.com/r/certego/buffalogs), with the `sudo docker pull certego/buffalogs:`. -After that, there are two ways of running BuffaLogs, depending on your system configurations: +After that, there are three ways of running BuffaLogs, depending on your system configurations: +* **DevContainers (Only for backend development)**: + * Use [Development with DevContainers](docs/guides/development/development_with_devcontainers.md) for a complete containerized development environment + * All dependencies (PostgreSQL, RabbitMQ, Elasticsearch, Kibana) are pre-configured + * Includes debugging tools and development utilities * if you already have an elastic cluster: * Configure the connection in `config/buffalogs/ingestion.json` (see [Elasticsearch Configuration Guide](docs/ingestion/elasticsearch.md) for detailed instructions) * Set the Elasticsearch URL, credentials, and index patterns diff --git a/docs/guides/development/complete_guide_for_developers.md b/docs/guides/development/complete_guide_for_developers.md index 4fb33974..cad55411 100644 --- a/docs/guides/development/complete_guide_for_developers.md +++ b/docs/guides/development/complete_guide_for_developers.md @@ -6,6 +6,8 @@ Steps workflow for Developers: 1. All the developers have to set the enrivornment correctly → Follow [the System Configuration guide](#system-configuration) + **Alternative Setup**: For a containerized **backend** development environment with all dependencies pre-configured, see [Development with DevContainers](development_with_devcontainers.md). + 1a. Then, if you'd like to make front-end changes, so you need only to query the APIs → Upload [the fixture data](#fixture) and read the [API's section](docs/rest-apis.md) for more details about them 1b. Otherwise, if you want to analyze the back-end development generating new data, please check [the BuffaLogs detection](#run-buffalogs-detection) diff --git a/docs/guides/development/development_with_devcontainers.md b/docs/guides/development/development_with_devcontainers.md new file mode 100644 index 00000000..a827e907 --- /dev/null +++ b/docs/guides/development/development_with_devcontainers.md @@ -0,0 +1,148 @@ +# Overview +DevContainers provide a consistent development environment with debuggers that eliminates the need for local setup of dependencies, databases, and services. This guide will help you set up BuffaLogs backend development environment using DevContainers. + +## Prerequisites +Before starting, ensure you have: +- [Visual Studio Code](https://code.visualstudio.com/) (or any other IDE which support the Dev Containers) installed +- [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) installed in your IDE +- Docker and Docker compose installed and running + +# Quick Start +1. **Clone the repository** + ```bash + git clone https://github.com/certego/BuffaLogs.git + cd BuffaLogs + ``` + +2. **Open in DevContainer** + - Open the project in VS Code + - Before starting development with dev containers, ensure that no `buffalogs` containers are currently running on your system. This prevents port conflicts and any errors that could interfere with the dev container environment. + - To see current docker containers running on your system, execute: `docker ps`. There shouldn't be any running containers. + - Press `Ctrl+Shift+P` + - Select or type "Dev Containers: Reopen in Container" + - Wait for the container to build and start (first time may take few minutes) + +3. **Verify setup** + The DevContainer will automatically build and run following containers: + - PostgreSQL database + - RabbitMQ message broker + - Celery and Celery beat workers + - Run Django migrations + - Runs `debugpy` on port `5678` for debugging + - Start the Django development server on port 8000 + +# Development Environment +The DevContainer provides the following services: + +## Core Services +| **Service** | **Port** | **Description** | **Access** | +|---|---|---|---| +| Django Dev Server | 8000 | Main BuffaLogs application | http://localhost:8000 | +| PostgreSQL | 5433 | Primary database | localhost:5433 | +| RabbitMQ | 5672 | Message broker | localhost:5672 | +| RabbitMQ Management | 15672 | Web management UI | http://localhost:15672 | +| Celery Worker | 5679 | Celery Worker debugging | localhost:5679 | +| Debug Server | 5678 | Python debugging | localhost:5678 | + +## Development Tools +The container includes: +- **Python 3.12** with all BuffaLogs dependencies +- **Django Debug Toolbar** for enhanced debugging +- **pytest** for testing +- **black** for code formatting +- **pylint** for code linting +- **debugpy** for remote debugging +- **Git** and **GitHub CLI** for version control + +# Working with the DevContainer + +## Django Management Commands +All Django management commands work as expected inside the devcontainer terminal: + +```bash +# Create superuser +python manage.py createsuperuser + +# Run migrations +python manage.py migrate + +# Start shell +python manage.py shell + +# Collect static files +python manage.py collectstatic +``` + +## Database Access +The PostgreSQL database is automatically configured and accessible: + +```bash +# Connect to database +psql -h localhost -p 5433 -U default_user -d buffalogs +``` + +Default credentials: +- **Username**: default_user +- **Password**: password +- **Database**: buffalogs + + +# Debugging +The DevContainer is configured for remote debugging: + +## Django Debugging + +Configure `launch.json` in VS Code for debugging. +``` +{ + "name": "Python: Django Debug Service", + "type": "debugpy", + "request": "attach", + "connect": { + "host": "localhost", + "port": 5678 + }, + "pathMappings": [ + { + "localRoot": "${workspaceFolder}", + "remoteRoot": "/opt/certego/buffalogs" + } + ], + "justMyCode": false, + "django": true, + "presentation": { + "hidden": false, + "group": "Docker Debug" + } +} +``` + +1. Set breakpoints in your code +2. Go to Run and Debug panel in VS Code +3. Select "Python: Django" configuration +4. Start debugging + +## Celery Worker Debugging +The Celery worker is configured for debugging on port 5679: +1. Set breakpoints in Celery tasks +2. Create a debug configuration for port 5679 +3. Go to Run and Debug panel in VS Code +4. Select "Python: Celery Worker" configuration + +Note: For these configurations to appear in the Run and Debug panel in VS Code, you will have to create a `launch.json` file in a `.vscode folder` and manually add these configurations. In case of further assistance, you may reach out. + +# Development Workflow + +## Making Changes +1. Edit files in the `buffalogs/` directory +2. Django development server auto-reloads on changes + +## Environment Variables +Key environment variables for development: +- `DEBUG=1` - Enables Django debug mode +- `CERTEGO_BUFFALOGS_DEBUG=True` - BuffaLogs-specific debug settings +- `USE_DEBUGPY=1` - Enables debugpy for remote debugging + +For closing the devcontainers, go to the bottom left of your screen (in VS Code) and click on `Reopen folder locally`. This will stop all running buffalogs containers. + +This DevContainer setup provides a consistent, reproducible development environment that matches the production infrastructure while offering enhanced debugging and development capabilities. \ No newline at end of file diff --git a/docs/guides/development/project_architecture.md b/docs/guides/development/project_architecture.md index b823c84c..de06a0bd 100644 --- a/docs/guides/development/project_architecture.md +++ b/docs/guides/development/project_architecture.md @@ -38,4 +38,8 @@ Message queueing allows web servers to respond to requests quickly instead of be ### Celery Beat [Celery Beat](https://docs.celeryq.dev/en/stable/reference/celery.beat.html) is the scheduler, so it keeps track of when tasks should be executed by Celery. -Celery would consume the tasks instantly, instead with Celery Beat the consumption of tasks is periodic. \ No newline at end of file +Celery would consume the tasks instantly, instead with Celery Beat the consumption of tasks is periodic. + +## Development Environment Setup + +For a complete containerized **backend** development environment that includes all the components mentioned above, see [Development with DevContainers](development_with_devcontainers.md). This provides a pre-configured setup with all services running and ready for development. \ No newline at end of file diff --git a/docs/guides/development/step_by_step_example.md b/docs/guides/development/step_by_step_example.md index 42cebb87..5af0b471 100644 --- a/docs/guides/development/step_by_step_example.md +++ b/docs/guides/development/step_by_step_example.md @@ -1,4 +1,7 @@ ## A Step by step Example + +**Alternative Setup**(optional): For a simplified development experience with all services pre-configured, consider using [Development with DevContainers](development_with_devcontainers.md) instead of manual setup. + It is possible to create random logs to test the project just by running the commands below: 1. Launch Elasticsearch and Kibana with the `docker compose -f docker-compose.yaml -f docker-compose.elastic.yaml up -d elasticsearch kibana` command` 2. Load the Elastic Common Schema template running the `./load_templates.sh` script