Implement semaphore around Gitea calls to avoid overloading Gitea.

roles/ocp4_workload_tenant_gitea_user/defaults/main.yml

@@ -23,3 +23,10 @@ ocp4_workload_tenant_gitea_user_repositories: []
 # - name: my-repo
 #   repo: https://github.com/org/my-repo
 #   private: false
+
+# Semaphore for serializing Gitea operations across concurrent runs
+ocp4_workload_tenant_gitea_user_semaphore_name: gitea-operations-lock
+ocp4_workload_tenant_gitea_user_semaphore_retries: 60
+ocp4_workload_tenant_gitea_user_semaphore_delay_min: 5
+ocp4_workload_tenant_gitea_user_semaphore_delay_max: 20
+ocp4_workload_tenant_gitea_user_semaphore_stale_timeout: 600

roles/ocp4_workload_tenant_gitea_user/tasks/acquire_semaphore.yml

@@ -0,0 +1,62 @@
+---
+- name: Check for existing operations lock
+  kubernetes.core.k8s_info:
+    host: "{{ ocp4_workload_tenant_gitea_user_openshift_api_url }}"
+    api_key: "{{ ocp4_workload_tenant_gitea_user_openshift_api_token }}"
+    validate_certs: false
+    api_version: v1
+    kind: Secret
+    name: "{{ ocp4_workload_tenant_gitea_user_semaphore_name }}"
+    namespace: "{{ ocp4_workload_tenant_gitea_user_namespace }}"
+  register: r_existing_lock
+
+- name: Remove stale operations lock
+  when:
+  - r_existing_lock.resources | length > 0
+  - >-
+    (now(utc=true) - (r_existing_lock.resources[0].metadata.annotations['lock-acquired-at']
+    | to_datetime('%Y-%m-%dT%H:%M:%SZ'))).total_seconds()
+    > ocp4_workload_tenant_gitea_user_semaphore_stale_timeout | int
+  kubernetes.core.k8s:
+    host: "{{ ocp4_workload_tenant_gitea_user_openshift_api_url }}"
+    api_key: "{{ ocp4_workload_tenant_gitea_user_openshift_api_token }}"
+    validate_certs: false
+    state: absent
+    api_version: v1
+    kind: Secret
+    name: "{{ ocp4_workload_tenant_gitea_user_semaphore_name }}"
+    namespace: "{{ ocp4_workload_tenant_gitea_user_namespace }}"
+
+# Use k8s API because k8s module would be idempotent. We need to know if creating the semaphore secret fails because it already exists.
+- name: Acquire operations lock
+  ansible.builtin.uri:
+    url: >-
+      {{ ocp4_workload_tenant_gitea_user_openshift_api_url }}/api/v1/namespaces/{{
+      ocp4_workload_tenant_gitea_user_namespace }}/secrets
+    method: POST
+    headers:
+      Authorization: "Bearer {{ ocp4_workload_tenant_gitea_user_openshift_api_token }}"
+      Content-Type: application/json
+    body_format: json
+    body:
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: "{{ ocp4_workload_tenant_gitea_user_semaphore_name }}"
+        namespace: "{{ ocp4_workload_tenant_gitea_user_namespace }}"
+        annotations:
+          lock-acquired-at: "{{ now(utc=true).strftime('%Y-%m-%dT%H:%M:%SZ') }}"
+    validate_certs: false
+    status_code:
+    - 201
+    - 409
+  register: r_lock_attempt
+  until: r_lock_attempt.status == 201
+  retries: "{{ ocp4_workload_tenant_gitea_user_semaphore_retries }}"
+  delay: >-
+    {{ range(ocp4_workload_tenant_gitea_user_semaphore_delay_min | int,
+    ocp4_workload_tenant_gitea_user_semaphore_delay_max | int + 1) | random }}
+
+- name: Record lock acquisition
+  ansible.builtin.set_fact:
+    _ocp4_workload_tenant_gitea_user_lock_acquired: true

roles/ocp4_workload_tenant_gitea_user/tasks/remove_workload.yml

@@ -13,20 +13,29 @@
 - name: Clean up Gitea user and repositories
   when: r_gitea.resources | length > 0
   block:
-    - name: Validate Gitea admin setup is complete
-      ansible.builtin.assert:
-        that:
-          - r_gitea.resources[0].status.adminSetupComplete | bool
-        fail_msg: Gitea admin setup is not complete yet.
+  - name: Validate Gitea admin setup is complete
+    ansible.builtin.assert:
+      that:
+      - r_gitea.resources[0].status.adminSetupComplete | bool
+      fail_msg: Gitea admin setup is not complete yet.
 
-    - name: Set Gitea connection facts
+  - name: Set Gitea connection facts
+    ansible.builtin.set_fact:
+      _ocp4_workload_tenant_gitea_user_admin_user: >-
+        {{ r_gitea.resources[0].spec.giteaAdminUser }}
+      _ocp4_workload_tenant_gitea_user_admin_password: >-
+        {{ r_gitea.resources[0].status.adminPassword }}
+      _ocp4_workload_tenant_gitea_user_route: >-
+        {{ r_gitea.resources[0].status.giteaRoute }}
+
+  - name: Gitea API operations with semaphore
+    block:
+    - name: Initialize semaphore tracking
       ansible.builtin.set_fact:
-        _ocp4_workload_tenant_gitea_user_admin_user: >-
-          {{ r_gitea.resources[0].spec.giteaAdminUser }}
-        _ocp4_workload_tenant_gitea_user_admin_password: >-
-          {{ r_gitea.resources[0].status.adminPassword }}
-        _ocp4_workload_tenant_gitea_user_route: >-
-          {{ r_gitea.resources[0].status.giteaRoute }}
+        _ocp4_workload_tenant_gitea_user_lock_acquired: false
+
+    - name: Acquire Gitea operations semaphore
+      ansible.builtin.include_tasks: acquire_semaphore.yml
 
     - name: Get all repositories for Gitea user
       ansible.builtin.uri:
@@ -39,7 +48,7 @@
         force_basic_auth: true
         validate_certs: false
         status_code:
-          - 200
+        - 200
       register: r_gitea_user_repos
 
     - name: Delete all repositories for Gitea user
@@ -54,8 +63,8 @@
         force_basic_auth: true
         validate_certs: false
         status_code:
-          - 204
-          - 404
+        - 204
+        - 404
       loop: "{{ r_gitea_user_repos.json.data | default([]) }}"
       loop_control:
         label: "{{ item.name }}"
@@ -71,5 +80,18 @@
         force_basic_auth: true
         validate_certs: false
         status_code:
-          - 204
-          - 404
+        - 204
+        - 404
+
+    always:
+    - name: Release Gitea operations semaphore
+      when: _ocp4_workload_tenant_gitea_user_lock_acquired | default(false) | bool
+      kubernetes.core.k8s:
+        host: "{{ ocp4_workload_tenant_gitea_user_openshift_api_url }}"
+        api_key: "{{ ocp4_workload_tenant_gitea_user_openshift_api_token }}"
+        validate_certs: false
+        state: absent
+        api_version: v1
+        kind: Secret
+        name: "{{ ocp4_workload_tenant_gitea_user_semaphore_name }}"
+        namespace: "{{ ocp4_workload_tenant_gitea_user_namespace }}"

roles/ocp4_workload_tenant_gitea_user/tasks/workload.yml

@@ -26,43 +26,16 @@
     _ocp4_workload_tenant_gitea_user_route: >-
       {{ r_gitea.resources[0].status.giteaRoute }}
 
-- name: Check if Gitea user exists
-  ansible.builtin.uri:
-    url: >-
-      {{ _ocp4_workload_tenant_gitea_user_route }}/api/v1/users/{{ ocp4_workload_tenant_gitea_user_username }}
-    method: GET
-    url_username: "{{ _ocp4_workload_tenant_gitea_user_admin_user }}"
-    url_password: "{{ _ocp4_workload_tenant_gitea_user_admin_password }}"
-    force_basic_auth: true
-    validate_certs: false
-    status_code:
-    - 200
-    - 404
-  register: r_gitea_user
+- name: Gitea API operations with semaphore
+  block:
+  - name: Initialize semaphore tracking
+    ansible.builtin.set_fact:
+      _ocp4_workload_tenant_gitea_user_lock_acquired: false
 
-- name: Create Gitea user
-  when: r_gitea_user.status == 404
-  ansible.builtin.uri:
-    url: "{{ _ocp4_workload_tenant_gitea_user_route }}/api/v1/admin/users"
-    method: POST
-    url_username: "{{ _ocp4_workload_tenant_gitea_user_admin_user }}"
-    url_password: "{{ _ocp4_workload_tenant_gitea_user_admin_password }}"
-    force_basic_auth: true
-    body_format: json
-    body:
-      email: "{{ ocp4_workload_tenant_gitea_user_email }}"
-      login_name: "{{ ocp4_workload_tenant_gitea_user_username }}"
-      must_change_password: false
-      password: "{{ ocp4_workload_tenant_gitea_user_password }}"
-      username: "{{ ocp4_workload_tenant_gitea_user_username }}"
-    validate_certs: false
-    status_code:
-    - 201
+  - name: Acquire Gitea operations semaphore
+    ansible.builtin.include_tasks: acquire_semaphore.yml
 
-- name: Migrate repositories
-  when: ocp4_workload_tenant_gitea_user_repositories | length > 0
-  block:
-  - name: Get Gitea user uid
+  - name: Check if Gitea user exists
     ansible.builtin.uri:
       url: >-
         {{ _ocp4_workload_tenant_gitea_user_route }}/api/v1/users/{{ ocp4_workload_tenant_gitea_user_username }}
@@ -71,59 +44,108 @@
       url_password: "{{ _ocp4_workload_tenant_gitea_user_admin_password }}"
       force_basic_auth: true
       validate_certs: false
-    register: r_gitea_user_info
-
-  - name: Check if repositories already exist
-    ansible.builtin.uri:
-      url: >-
-        {{ _ocp4_workload_tenant_gitea_user_route }}/api/v1/repos/{{
-        ocp4_workload_tenant_gitea_user_username }}/{{ item.name }}
-      method: GET
-      url_username: "{{ _ocp4_workload_tenant_gitea_user_admin_user }}"
-      url_password: "{{ _ocp4_workload_tenant_gitea_user_admin_password }}"
-      force_basic_auth: true
-      validate_certs: false
       status_code:
       - 200
       - 404
-    register: r_gitea_repos
-    loop: "{{ ocp4_workload_tenant_gitea_user_repositories }}"
-    loop_control:
-      label: "{{ item.name }}"
+    register: r_gitea_user
 
-  - name: Migrate repositories that do not exist yet
-    when: item.status == 404
+  - name: Create Gitea user
+    when: r_gitea_user.status == 404
     ansible.builtin.uri:
-      url: "{{ _ocp4_workload_tenant_gitea_user_route }}/api/v1/repos/migrate"
+      url: "{{ _ocp4_workload_tenant_gitea_user_route }}/api/v1/admin/users"
       method: POST
       url_username: "{{ _ocp4_workload_tenant_gitea_user_admin_user }}"
       url_password: "{{ _ocp4_workload_tenant_gitea_user_admin_password }}"
       force_basic_auth: true
       body_format: json
-      body: >-
-        {{
-          {
-            'clone_addr': item.item.repo,
-            'description': '',
-            'issues': false,
-            'milestones': false,
-            'mirror': false,
-            'private': item.item.private | default(false),
-            'repo_name': item.item.name,
-            'uid': r_gitea_user_info.json.id | int
-          }
-        }}
+      body:
+        email: "{{ ocp4_workload_tenant_gitea_user_email }}"
+        login_name: "{{ ocp4_workload_tenant_gitea_user_username }}"
+        must_change_password: false
+        password: "{{ ocp4_workload_tenant_gitea_user_password }}"
+        username: "{{ ocp4_workload_tenant_gitea_user_username }}"
       validate_certs: false
       status_code:
       - 201
-      timeout: 60
-    register: r_migrate_result
-    until: r_migrate_result.status == 201
-    retries: 24
-    delay: 5
-    loop: "{{ r_gitea_repos.results }}"
-    loop_control:
-      label: "{{ item.item.name }}"
+
+  - name: Migrate repositories
+    when: ocp4_workload_tenant_gitea_user_repositories | length > 0
+    block:
+    - name: Get Gitea user uid
+      ansible.builtin.uri:
+        url: >-
+          {{ _ocp4_workload_tenant_gitea_user_route }}/api/v1/users/{{ ocp4_workload_tenant_gitea_user_username }}
+        method: GET
+        url_username: "{{ _ocp4_workload_tenant_gitea_user_admin_user }}"
+        url_password: "{{ _ocp4_workload_tenant_gitea_user_admin_password }}"
+        force_basic_auth: true
+        validate_certs: false
+      register: r_gitea_user_info
+
+    - name: Check if repositories already exist
+      ansible.builtin.uri:
+        url: >-
+          {{ _ocp4_workload_tenant_gitea_user_route }}/api/v1/repos/{{
+          ocp4_workload_tenant_gitea_user_username }}/{{ item.name }}
+        method: GET
+        url_username: "{{ _ocp4_workload_tenant_gitea_user_admin_user }}"
+        url_password: "{{ _ocp4_workload_tenant_gitea_user_admin_password }}"
+        force_basic_auth: true
+        validate_certs: false
+        status_code:
+        - 200
+        - 404
+      register: r_gitea_repos
+      loop: "{{ ocp4_workload_tenant_gitea_user_repositories }}"
+      loop_control:
+        label: "{{ item.name }}"
+
+    - name: Migrate repositories that do not exist yet
+      when: item.status == 404
+      ansible.builtin.uri:
+        url: "{{ _ocp4_workload_tenant_gitea_user_route }}/api/v1/repos/migrate"
+        method: POST
+        url_username: "{{ _ocp4_workload_tenant_gitea_user_admin_user }}"
+        url_password: "{{ _ocp4_workload_tenant_gitea_user_admin_password }}"
+        force_basic_auth: true
+        body_format: json
+        body: >-
+          {{
+            {
+              'clone_addr': item.item.repo,
+              'description': '',
+              'issues': false,
+              'milestones': false,
+              'mirror': false,
+              'private': item.item.private | default(false),
+              'repo_name': item.item.name,
+              'uid': r_gitea_user_info.json.id | int
+            }
+          }}
+        validate_certs: false
+        status_code:
+        - 201
+        timeout: 60
+      register: r_migrate_result
+      until: r_migrate_result.status == 201
+      retries: 24
+      delay: 5
+      loop: "{{ r_gitea_repos.results }}"
+      loop_control:
+        label: "{{ item.item.name }}"
+
+  always:
+  - name: Release Gitea operations semaphore
+    when: _ocp4_workload_tenant_gitea_user_lock_acquired | default(false) | bool
+    kubernetes.core.k8s:
+      host: "{{ ocp4_workload_tenant_gitea_user_openshift_api_url }}"
+      api_key: "{{ ocp4_workload_tenant_gitea_user_openshift_api_token }}"
+      validate_certs: false
+      state: absent
+      api_version: v1
+      kind: Secret
+      name: "{{ ocp4_workload_tenant_gitea_user_semaphore_name }}"
+      namespace: "{{ ocp4_workload_tenant_gitea_user_namespace }}"
 
 - name: Save Gitea user information
   agnosticd.core.agnosticd_user_info: