Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 45 additions & 0 deletions .devcontainer/Dockerfile.dev
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
FROM python:3.12-slim-trixie

ENV INSTALL_PATH /opt/certego
RUN mkdir -p $INSTALL_PATH
WORKDIR $INSTALL_PATH

# Install system dependencies
RUN sed -i 's/main/main non-free/g' /etc/apt/sources.list.d/debian.sources \
&& apt-get update \
&& apt-get -y install \
gcc \
curl \
git \
vim \
&& rm -rf /var/lib/apt/lists/*

# Copy requirements and install Python packages
COPY buffalogs/ /opt/certego/buffalogs/
RUN pip install --no-cache-dir -r buffalogs/requirements.txt -r buffalogs/requirements_dev.txt

# Install additional development packages
RUN pip install --no-cache-dir \
django-debug-toolbar \
django-extensions \
ipython \
pytest-django \
pytest-cov \
black==25.1.0 \
pylint==3.3.5 \
flake8==7.1.2 \
debugpy

RUN chmod +x /opt/certego/buffalogs/run.sh
RUN chmod +x /opt/certego/buffalogs/run_worker.sh
RUN chmod +x /opt/certego/buffalogs/run_beat.sh

WORKDIR /opt/certego/buffalogs/

# Create socket directory for uwsgi
RUN mkdir -p /var/run/nginx-sockets

# Expose port for development server
EXPOSE 8000

CMD ["sleep", "infinity"]
70 changes: 70 additions & 0 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
{
"name": "BuffaLogs Development",
"dockerComposeFile": "docker-compose.yaml",
"service": "buffalogs",
"workspaceFolder": "/opt/certego/buffalogs",
// Stop all running containers on close
"shutdownAction": "stopCompose",
"remoteUser": "root",
"postCreateCommand": "python manage.py migrate",
"forwardPorts": [
8000,
5433,
5672,
15672,
5679
],
"portsAttributes": {
"8000": {
"label": "Django Dev Server",
"onAutoForward": "notify"
},
"5433": {
"label": "PostgreSQL",
"onAutoForward": "notify"
},
"5672": {
"label": "RabbitMQ",
"onAutoForward": "silent"
},
"15672": {
"label": "RabbitMQ Management",
"onAutoForward": "notify"
},
"5679": {
"label": "Celery Worker",
"onAutoForward": "notify"
}
},
"customizations": {
"vscode": {
"settings": {
"python.defaultInterpreterPath": "/usr/local/bin/python",
"python.linting.enabled": true,
"python.linting.pylintEnabled": true,
"python.formatting.provider": "black",
"python.testing.pytestEnabled": true,
"python.testing.unittestEnabled": false,
"files.exclude": {
"**/__pycache__": true,
"**/*.pyc": true
},
"django.debugJustMyCode": false
},
"extensions": [
"ms-python.python",
"ms-python.debugpy",
"ms-python.black-formatter",
"ms-python.pylint",
"batisteo.vscode-django",
"ms-vscode.vscode-json",
"redhat.vscode-yaml"
]
}
},
"features": {
"ghcr.io/devcontainers/features/git:1": {},
"ghcr.io/devcontainers/features/github-cli:1": {}
},
"overrideCommand": false
}
97 changes: 97 additions & 0 deletions .devcontainer/docker-compose.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
services:
# Development override - extends original docker-compose.yaml
buffalogs_postgres:
extends:
file: ../docker-compose.yaml
service: buffalogs_postgres
volumes:
- buffalogs_postgres_data:/var/lib/postgresql/data
ports:
- "5433:5432"

buffalogs_rabbitmq:
extends:
file: ../docker-compose.yaml
service: buffalogs_rabbitmq
ports:
- "5672:5672"
- "15672:15672"



# Override buffalogs service for development
buffalogs:
extends:
file: ../docker-compose.yaml
service: buffalogs
build:
context: ..
dockerfile: .devcontainer/Dockerfile.dev
volumes:
- ../buffalogs:/opt/certego/buffalogs
- ../config:/opt/certego/config:ro
- ../.vscode:/opt/certego/buffalogs/.vscode
- buffalogs_django_static:/var/www
- buffalogs_nginx_sockets:/var/run/nginx-sockets
# For development, we need to expose port 8000 for Django runserver and 5678 for debugpy
ports:
- "8000:8000"
- "5678:5678"
environment:
- DEBUG=1
- CERTEGO_BUFFALOGS_DEBUG=True
- USE_DEBUGPY=1
command: python -m debugpy --listen 0.0.0.0:5678 manage.py runserver 0.0.0.0:8000
depends_on:
buffalogs_postgres:
condition: service_healthy
buffalogs_rabbitmq:
condition: service_healthy

# Development Celery worker
buffalogs_celery:
extends:
file: ../docker-compose.yaml
service: buffalogs_celery
build:
context: ..
dockerfile: .devcontainer/Dockerfile.dev
volumes:
- ../buffalogs:/opt/certego/buffalogs
- ../config:/opt/certego/config:ro
ports:
- "5679:5679"
command: python -m debugpy --listen 0.0.0.0:5679 --wait-for-client -m celery -A buffalogs worker -c 1 --loglevel=info
environment:
- DEBUG=1

# Development Celery beat
buffalogs_celery_beat:
extends:
file: ../docker-compose.yaml
service: buffalogs_celery_beat
build:
context: ..
dockerfile: .devcontainer/Dockerfile.dev
volumes:
- ../buffalogs:/opt/certego/buffalogs
- ../config:/opt/certego/config:ro
environment:
- DEBUG=1



volumes:
buffalogs_postgres_data:
driver: local
buffalogs_nginx_sockets:
driver: local
buffalogs_django_static:
driver: local
buffalogs_rabbitmq_data:
driver: local

networks:
buffalogs_network:
driver: bridge
name: buffalogs_network
6 changes: 5 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,11 @@ git clone git@github.com:certego/BuffaLogs.git
```
Or download the application directly from the [Docker Hub](https://hub.docker.com/r/certego/buffalogs), with the `sudo docker pull certego/buffalogs:<release_tag>`.

After that, there are two ways of running BuffaLogs, depending on your system configurations:
After that, there are three ways of running BuffaLogs, depending on your system configurations:
* **DevContainers (Only for backend development)**:
* Use [Development with DevContainers](docs/guides/development/development_with_devcontainers.md) for a complete containerized development environment
* All dependencies (PostgreSQL, RabbitMQ, Elasticsearch, Kibana) are pre-configured
* Includes debugging tools and development utilities
* if you already have an elastic cluster:
* Configure the connection in `config/buffalogs/ingestion.json` (see [Elasticsearch Configuration Guide](docs/ingestion/elasticsearch.md) for detailed instructions)
* Set the Elasticsearch URL, credentials, and index patterns
Expand Down
2 changes: 2 additions & 0 deletions docs/guides/development/complete_guide_for_developers.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ Steps workflow for Developers:

1. All the developers have to set the enrivornment correctly &rarr; Follow [the System Configuration guide](#system-configuration)

**Alternative Setup**: For a containerized **backend** development environment with all dependencies pre-configured, see [Development with DevContainers](development_with_devcontainers.md).

1a. Then, if you'd like to make front-end changes, so you need only to query the APIs &rarr; Upload [the fixture data](#fixture) and read the [API's section](docs/rest-apis.md) for more details about them

1b. Otherwise, if you want to analyze the back-end development generating new data, please check [the BuffaLogs detection](#run-buffalogs-detection)
Expand Down
148 changes: 148 additions & 0 deletions docs/guides/development/development_with_devcontainers.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,148 @@
# Overview
DevContainers provide a consistent development environment with debuggers that eliminates the need for local setup of dependencies, databases, and services. This guide will help you set up BuffaLogs backend development environment using DevContainers.

## Prerequisites
Before starting, ensure you have:
- [Visual Studio Code](https://code.visualstudio.com/) (or any other IDE which support the Dev Containers) installed
- [Dev Containers extension](https://marketplace.visualstudio.com/items?itemName=ms-vscode-remote.remote-containers) installed in your IDE
- Docker and Docker compose installed and running

# Quick Start
1. **Clone the repository**
```bash
git clone https://github.com/certego/BuffaLogs.git
cd BuffaLogs
```

2. **Open in DevContainer**
- Open the project in VS Code
- Before starting development with dev containers, ensure that no `buffalogs` containers are currently running on your system. This prevents port conflicts and any errors that could interfere with the dev container environment.
- To see current docker containers running on your system, execute: `docker ps`. There shouldn't be any running containers.
- Press `Ctrl+Shift+P`
- Select or type "Dev Containers: Reopen in Container"
- Wait for the container to build and start (first time may take few minutes)

3. **Verify setup**
The DevContainer will automatically build and run following containers:
- PostgreSQL database
- RabbitMQ message broker
- Celery and Celery beat workers
- Run Django migrations
- Runs `debugpy` on port `5678` for debugging
- Start the Django development server on port 8000

# Development Environment
The DevContainer provides the following services:

## Core Services
| **Service** | **Port** | **Description** | **Access** |
|---|---|---|---|
| Django Dev Server | 8000 | Main BuffaLogs application | http://localhost:8000 |
| PostgreSQL | 5433 | Primary database | localhost:5433 |
| RabbitMQ | 5672 | Message broker | localhost:5672 |
| RabbitMQ Management | 15672 | Web management UI | http://localhost:15672 |
| Celery Worker | 5679 | Celery Worker debugging | localhost:5679 |
| Debug Server | 5678 | Python debugging | localhost:5678 |

## Development Tools
The container includes:
- **Python 3.12** with all BuffaLogs dependencies
- **Django Debug Toolbar** for enhanced debugging
- **pytest** for testing
- **black** for code formatting
- **pylint** for code linting
- **debugpy** for remote debugging
- **Git** and **GitHub CLI** for version control

# Working with the DevContainer

## Django Management Commands
All Django management commands work as expected inside the devcontainer terminal:

```bash
# Create superuser
python manage.py createsuperuser

# Run migrations
python manage.py migrate

# Start shell
python manage.py shell

# Collect static files
python manage.py collectstatic
```

## Database Access
The PostgreSQL database is automatically configured and accessible:

```bash
# Connect to database
psql -h localhost -p 5433 -U default_user -d buffalogs
```

Default credentials:
- **Username**: default_user
- **Password**: password
- **Database**: buffalogs


# Debugging
The DevContainer is configured for remote debugging:

## Django Debugging

Configure `launch.json` in VS Code for debugging.
```
{
"name": "Python: Django Debug Service",
"type": "debugpy",
"request": "attach",
"connect": {
"host": "localhost",
"port": 5678
},
"pathMappings": [
{
"localRoot": "${workspaceFolder}",
"remoteRoot": "/opt/certego/buffalogs"
}
],
"justMyCode": false,
"django": true,
"presentation": {
"hidden": false,
"group": "Docker Debug"
}
}
```

1. Set breakpoints in your code
2. Go to Run and Debug panel in VS Code
3. Select "Python: Django" configuration
4. Start debugging

## Celery Worker Debugging
The Celery worker is configured for debugging on port 5679:
1. Set breakpoints in Celery tasks
2. Create a debug configuration for port 5679
3. Go to Run and Debug panel in VS Code
4. Select "Python: Celery Worker" configuration

Note: For these configurations to appear in the Run and Debug panel in VS Code, you will have to create a `launch.json` file in a `.vscode folder` and manually add these configurations. In case of further assistance, you may reach out.

# Development Workflow

## Making Changes
1. Edit files in the `buffalogs/` directory
2. Django development server auto-reloads on changes

## Environment Variables
Key environment variables for development:
- `DEBUG=1` - Enables Django debug mode
- `CERTEGO_BUFFALOGS_DEBUG=True` - BuffaLogs-specific debug settings
- `USE_DEBUGPY=1` - Enables debugpy for remote debugging

For closing the devcontainers, go to the bottom left of your screen (in VS Code) and click on `Reopen folder locally`. This will stop all running buffalogs containers.

This DevContainer setup provides a consistent, reproducible development environment that matches the production infrastructure while offering enhanced debugging and development capabilities.
Loading