Move Makefile

.github/workflows/continuous_integration.yml

@@ -14,9 +14,6 @@ jobs:
   validate-models:
     name: CI
     runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: dev/
 
     steps:
       - uses: actions/checkout@v6
@@ -27,7 +24,7 @@ jobs:
           python-version: ${{ env.PYTHON_VERSION }}
 
       - name: Install requirements
-        run: pip install -r requirements.txt
+        run: pip install -r dev/requirements.txt
 
       - name: Check black
         if: always()
@@ -54,11 +51,11 @@ jobs:
         run: make validate-models
 
       - name: Generate models.yml from collections
-        run: python src/join_models_yml.py
+        run: python dev/src/join_models_yml.py
 
       - name: Check if models.yml is up-to-date
         run: |
-          if git diff --exit-code ../models.yml; then
+          if git diff --exit-code ./models.yml; then
             echo "✅ models.yml is up-to-date"
           else
             echo "❌ models.yml is out of sync with collections/"

.github/workflows/test-sql.yml

@@ -0,0 +1,59 @@
+name: Test SQL
+on: [pull_request]
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    # TODO: What container should be used? This is from the github example
+    container: node:20-bookworm-slim
+
+    services:
+      postgres:
+        image: postgres:15
+        env:
+          POSTGRES_PASSWORD: password
+          POSTGRES_USER: openslides
+          POSTGRES_DB: openslides
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@v4
+
+      - name: Install psql
+        run: |
+          apt-get update
+          apt-get install --yes --no-install-recommends postgresql-client
+
+      - name: apply schema
+        shell: bash
+        run: bash dev/scripts/apply_db_schema.sh
+        env:
+          DATABASE_HOST: postgres
+          DATABASE_PORT: 5432
+          DATABASE_USER: openslides
+          DATABASE_NAME: openslides
+          PGPASSWORD: password
+
+      - name: insert base data
+        shell: bash
+        run: bash dev/scripts/apply_data.sh base_data.sql
+        env:
+          DATABASE_HOST: postgres
+          DATABASE_PORT: 5432
+          DATABASE_USER: openslides
+          DATABASE_NAME: openslides
+          PGPASSWORD: password
+
+      - name: insert test data
+        shell: bash
+        run: bash dev/scripts/apply_data.sh test_data.sql
+        env:
+          DATABASE_HOST: postgres
+          DATABASE_PORT: 5432
+          DATABASE_USER: openslides
+          DATABASE_NAME: openslides
+          PGPASSWORD: password

.gitignore

@@ -0,0 +1,34 @@
+# Patterns of intentionally untracked files that Git should ignore
+
+# Private development data
+private/
+
+# Python compiled files
+__pycache__/
+
+# Python virtual environment target directory
+.virtualenv/
+.venv/
+
+# Mypy (static type checker) cache
+.mypy_cache/
+
+# Pytest, pytest-cov and Coverage.py cache
+.pytest_cache/
+.coverage
+htmlcov/
+
+# Specific editor files
+.vscode/*
+.vimrc
+
+# coverage
+
+migrations/*.json
+migrations/*.sql
+
+# secrets
+secrets/
+
+# code-workspaces
+*.code-workspace

.sqlfluffignore

@@ -0,0 +1 @@
+schema_relational.sql

Makefile

@@ -0,0 +1,69 @@
+# Commands inside the container
+paths = dev/src/ dev/tests/
+
+all: pyupgrade black autoflake isort flake8 mypy sqlfluff
+
+pyupgrade:
+	pyupgrade --py310-plus --exit-zero-even-if-changed $$(find . -name '*.py')
+
+check-pyupgrade:
+	pyupgrade --py310-plus $$(find . -name '*.py')
+
+black:
+	black $(paths)
+
+check-black:
+	black --check --diff $(paths)
+
+autoflake:
+	autoflake $(paths)
+
+isort:
+	isort $(paths)
+
+check-isort:
+	isort --check-only --diff $(paths)
+
+flake8:
+	flake8 $(paths)
+
+mypy:
+	mypy dev/src/
+
+sqlfluff:
+	sqlfluff fix --dialect postgres --verbose
+
+validate-models:
+	python -m dev.src.validate
+
+generate-relational-schema:
+	python -m dev.src.generate_sql_schema
+
+drop-database:
+	dropdb -f -e --if-exists -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USER} ${DATABASE_NAME}
+
+create-database:
+	createdb -e -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USER} ${DATABASE_NAME}
+
+apply-db-schema:
+	dev/scripts/apply_db_schema.sh
+
+apply-test-data:
+	dev/scripts/apply_data.sh base_data.sql
+	dev/scripts/apply_data.sh test_data.sql
+
+create-database-with-schema: drop-database create-database apply-db-schema
+
+create-test-data: create-database-with-schema apply-test-data
+
+run-psql:
+	psql -h ${DATABASE_HOST} -p ${DATABASE_PORT} -U ${DATABASE_USER} -d ${DATABASE_NAME}
+
+# Docker manage commands
+
+run-dev:
+	USER_ID=$$(id -u $${USER}) GROUP_ID=$$(id -g $${USER}) docker compose -f dev/docker-compose.yml up -d --build
+	docker compose -f dev/docker-compose.yml exec models bash --rcfile /etc/bash_completion
+
+stop-dev:
+	docker compose -f dev/docker-compose.yml down

collections/agenda_item.yml

@@ -48,6 +48,11 @@ weight:
 
 content_object_id:
   type: generic-relation
+  reference:
+  - motion
+  - motion_block
+  - assignment
+  - topic
   to:
     collections:
     - motion
@@ -61,6 +66,7 @@ content_object_id:
   constant: true
 parent_id:
   type: relation
+  reference: agenda_item
   to: agenda_item/child_ids
   equal_fields: meeting_id
   restriction_mode: A
@@ -82,6 +88,7 @@ projection_ids:
   restriction_mode: A
 meeting_id:
   type: relation
+  reference: meeting
   to: meeting/agenda_item_ids
   required: true
   restriction_mode: A

collections/assignment.yml

@@ -32,6 +32,7 @@ number_poll_candidates:
   restriction_mode: A
 sequential_number:
   type: number
+  sequence_scope: meeting_id
   description: The (positive) serial number of this model in its meeting. This number is auto-generated and read-only.
   read_only: true
   required: true
@@ -82,6 +83,7 @@ projection_ids:
   restriction_mode: A
 meeting_id:
   type: relation
+  reference: meeting
   to: meeting/assignment_ids
   required: true
   restriction_mode: A

collections/assignment_candidate.yml

@@ -11,18 +11,21 @@ weight:
 
 assignment_id:
   type: relation
+  reference: assignment
   to: assignment/candidate_ids
   equal_fields: meeting_id
   restriction_mode: A
   required: true
   constant: true
 meeting_user_id:
   type: relation
+  reference: meeting_user
   to: meeting_user/assignment_candidate_ids
   restriction_mode: A
   constant: true
 meeting_id:
   type: relation
+  reference: meeting
   to: meeting/assignment_candidate_ids
   required: true
   restriction_mode: A

collections/chat_group.yml

@@ -12,7 +12,6 @@ weight:
   type: number
   default: 10000
   restriction_mode: A
-
 chat_message_ids:
   type: relation-list
   to: chat_message/chat_group_id
@@ -31,6 +30,7 @@ write_group_ids:
   restriction_mode: A
 meeting_id:
   type: relation
+  reference: meeting
   to: meeting/chat_group_ids
   required: true
   restriction_mode: A

collections/chat_message.yml

@@ -14,17 +14,20 @@ created:
   restriction_mode: A
 meeting_user_id:
   type: relation
+  reference: meeting_user
   to: meeting_user/chat_message_ids
   restriction_mode: A
   constant: true
 chat_group_id:
   type: relation
+  reference: chat_group
   to: chat_group/chat_message_ids
   restriction_mode: A
   required: true
   constant: true
 meeting_id:
   type: relation
+  reference: meeting
   to: meeting/chat_message_ids
   required: true
   restriction_mode: A

collections/committee.yml

@@ -23,13 +23,37 @@ meeting_ids:
 default_meeting_id:
   type: relation
   to: meeting/default_meeting_for_committee_id
+  reference: meeting
   restriction_mode: A
 user_ids:
   type: relation-list
   to: user/committee_ids
   restriction_mode: A
   read_only: true
-  description: "Calculated field."
+  description: "Calculated field: All users which are in a group of a meeting, belonging to the committee or beeing manager of the committee"
+  sql: |
+    (
+      SELECT array_agg(DISTINCT user_id ORDER BY user_id)
+      FROM (
+        -- Select user_ids from committees meetings
+        SELECT mu.user_id
+        FROM meeting_t AS m
+        INNER JOIN meeting_user_t AS mu ON mu.meeting_id = m.id
+        WHERE m.committee_id = c.id
+
+        UNION
+
+        -- Select user_ids from committee managers
+        SELECT cmu.user_id
+        FROM nm_committee_manager_ids_user_t cmu
+        WHERE cmu.committee_id = c.id
+
+        UNION
+
+        -- Select user_id from home committees
+        SELECT u.id FROM user_t u WHERE u.home_committee_id = c.id
+      ) _
+    ) AS user_ids
 manager_ids:
   type: relation-list
   to: user/committee_management_ids
@@ -39,6 +63,7 @@ parent_id:
   type: relation
   to: committee/child_ids
   restriction_mode: A
+  reference: committee
 child_ids:
   type: relation-list
   to: committee/parent_id
@@ -69,6 +94,7 @@ organization_tag_ids:
   restriction_mode: A
 organization_id:
   type: relation
+  reference: organization
   to: organization/committee_ids
   required: true
   restriction_mode: A

collections/gender.yml

@@ -14,8 +14,10 @@ organization_id:
   required: true
   to: organization/gender_ids
   restriction_mode: A
+  reference: organization
 user_ids:
   type: relation-list
   to: user/gender_id
   restriction_mode: A
+  reference: user
 

collections/group.yml

@@ -122,22 +122,27 @@ poll_ids:
   restriction_mode: A
 used_as_motion_poll_default_id:
   type: relation
+  reference: meeting
   to: meeting/motion_poll_default_group_ids
   restriction_mode: A
 used_as_assignment_poll_default_id:
   type: relation
+  reference: meeting
   to: meeting/assignment_poll_default_group_ids
   restriction_mode: A
 used_as_topic_poll_default_id:
   type: relation
+  reference: meeting
   to: meeting/topic_poll_default_group_ids
   restriction_mode: A
 used_as_poll_default_id:
   type: relation
+  reference: meeting
   to: meeting/poll_default_group_ids
   restriction_mode: A
 meeting_id:
   type: relation
+  reference: meeting
   to: meeting/group_ids
   required: true
   restriction_mode: A